diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b1de36c0b..672a963b2 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -91,6 +91,61 @@ jobs: - name: Install Python dependencies run: pip install -r requirements.txt + # A fresh install's schema, from the catalog init created above before + # any test ran, against the committed file. On a difference the job + # fails with the diff and uploads the regenerated file to commit. + - name: Check the current schema + run: | + devtools/schema-dump postgres postgresql://postgres:devpass@127.0.0.1:5432 \ + extenddb_catalog extenddb > schema-postgres.txt + if ! diff -u docs/schema/postgres.txt schema-postgres.txt; then + echo "docs/schema/postgres.txt differs from a fresh install: commit the schema-postgres artifact as docs/schema/postgres.txt" + exit 1 + fi + + - name: Upload the regenerated schema + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: schema-postgres + path: schema-postgres.txt + if-no-files-found: ignore + + # PgBouncer 1.26.1 in transaction mode in front of the job's server, for + # the lock script's pass through a connection pooler. It logs every role + # in through auth_query as the superuser, so the script's own role needs + # no entry of its own. + - name: Start PgBouncer + run: >- + docker run -d --name pgbouncer --network host + -e DB_HOST=127.0.0.1 -e DB_PORT=5432 + -e DB_USER=postgres -e DB_PASSWORD=devpass -e AUTH_USER=postgres + -e AUTH_TYPE=scram-sha-256 -e POOL_MODE=transaction + -e LISTEN_ADDR=127.0.0.1 -e LISTEN_PORT=6432 + edoburu/pgbouncer@sha256:8638aa563607256d3c7440437534195f1066a92b68015d89ae9d6b935b444a6d + + # The migration lock and the serving registration between serve and + # migrate, on a deployment of its own (port 18452), then the commands + # and serve with the catalog reached through PgBouncer. + - name: Check serve and migrate on one catalog + env: + POOLER_URL: postgresql://127.0.0.1:6432 + run: >- + devtools/check-postgres-catalog-lock ./target/release/extenddb + postgresql://postgres:devpass@127.0.0.1:5432 18452 + + # The last 0.1.x image's verify, migrate --yes and serve against + # catalogs this binary created and migrated: what each writes, and that + # this binary then reports WrittenByOlderRelease and --reconverge + # restores Current. + - name: Check the last 0.1.x release against migrated catalogs + run: | + docker pull extenddb/extenddb-postgres:0.1.14 + devtools/check-postgres-older-binary ./target/release/extenddb \ + postgresql://postgres:devpass@127.0.0.1:5432 \ + crates/storage-postgres/tests/fixtures/released/0.1.14 \ + extenddb/extenddb-postgres:0.1.14 18453 + - name: Run integration tests env: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 @@ -124,6 +179,21 @@ jobs: - name: Build release (SQLite backend) run: cargo build --release -p extenddb --no-default-features --features sqlite + # The migration lock between serve and migrate, on a catalog of its own + # (port 18450), before the suite's server starts. + - name: Check serve and migrate on one catalog + run: devtools/check-sqlite-catalog-lock ./target/release/extenddb 18450 + + # The last 0.1.x dev image's serve and migrate --yes against catalogs + # this binary migrated: what each writes, and that this binary then + # reports WrittenByOlderRelease and --reconverge restores Current. + - name: Check the last 0.1.x release against migrated catalogs + run: | + docker pull extenddb/extenddb-dev:0.1.14 + devtools/check-sqlite-older-binary ./target/release/extenddb \ + crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql \ + extenddb/extenddb-dev:0.1.14 18451 + - name: Initialize ExtendDB id: init run: | @@ -132,6 +202,24 @@ jobs: password=$(echo "$output" | grep -oP 'Password: \K\S+') echo "admin_password=$password" >> "$GITHUB_OUTPUT" + # A fresh install's schema against the committed file. On a difference + # the job fails with the diff and uploads the regenerated file to commit. + - name: Check the current schema + run: | + devtools/schema-dump sqlite extenddb.sqlite > schema-sqlite.txt + if ! diff -u docs/schema/sqlite.txt schema-sqlite.txt; then + echo "docs/schema/sqlite.txt differs from a fresh install: commit the schema-sqlite artifact as docs/schema/sqlite.txt" + exit 1 + fi + + - name: Upload the regenerated schema + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: schema-sqlite + path: schema-sqlite.txt + if-no-files-found: ignore + - name: Start ExtendDB run: | # --write-pid-file so devtools/run-tests can restart the server with @@ -240,6 +328,9 @@ jobs: steps: - uses: actions/checkout@v6 + with: + # migrations_tag_blobs reads the release tags. + fetch-depth: 0 - uses: dtolnay/rust-toolchain@stable @@ -311,7 +402,10 @@ jobs: # cases need catalog rows the current binary would not write; all need # direct access. The online GSI build tests reorder queue rows by hand and # pause writers with a test hook. They build their own throwaway databases; - # the connection string is the server, not a database. + # the connection string is the server, not a database. `--tests` runs + # every target of the crate, so a new test file needs no edit here, and + # EXTENDDB_TEST_PG_REQUIRED turns a test's skip for a missing connection + # string into a failure. - name: Run PostgreSQL storage-level tests # table_snapshot: the SnapshotEngine consistency check; it needs the # shared page-size hook, compiled only into the crate's own test @@ -320,11 +414,7 @@ jobs: EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 # table_snapshot fails rather than skips without the string above. EXTENDDB_TEST_PG_REQUIRED: "1" - run: >- - cargo test --release -p extenddb-storage-postgres - --test vector_control_plane --test put_create_race --test twi_conflict - --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test backup_restore - --test gsi_online_build --test gsi_stale_writer --test table_snapshot + run: cargo test --release -p extenddb-storage-postgres --tests # The daemonized server logs to syslog; dump it so server-side failures # are diagnosable from the job log. @@ -403,16 +493,24 @@ jobs: devtools/run-tests --extenddb --rust-integration --release --filter vector_index_unsupported + # The migration tests on a server without pgvector: the released vector + # states skip, naming pgvector, and every other case runs. The other + # storage-level tests build extension-free scratch databases and behave + # the same on either image, so the job above covers them. + - name: Run the PostgreSQL migration tests without pgvector + env: + EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_PG_REQUIRED: "1" + run: >- + cargo test --release -p extenddb-storage-postgres + --test migrations_fresh --test migrations_released --test migrations_crash + --test migrations_golden --test migrations_guards --test migrations_runs + # The daemonized server logs to syslog; dump it so server-side failures # are diagnosable from the job log. - name: Dump server syslog on failure if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true - - # No storage-level step here on purpose. Those tests build their own - # extension-free scratch databases, so they behave identically on either - # image and the job above already runs them; repeating them here would buy - # a second release build and no coverage. run-rust-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 469774ead..a7cc8a65d 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -164,6 +164,25 @@ jobs: echo "sha=$SHA" } >> "$GITHUB_OUTPUT" + - name: Check out the tagged commit for the migration gate + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ steps.meta.outputs.sha }} + persist-credentials: false + + - name: Gate - the release freezes the catalog migrations it ships + # The platform packages ship the sqlite,dev-mode build of the tag, + # whose dev server applies SQLite migrations to a catalog behind it. + # As in release-image.yml: RELEASED_THROUGH must reach the last + # migration id. A tag before the migration framework passes. + run: | + set -euo pipefail + if [[ -e crates/storage/src/migrations/registry.rs ]]; then + python3 devtools/check-released-through + else + echo "the tagged commit has no catalog migration registry" + fi + build: needs: gate # The matrix lives in npm-build-matrix.yml, shared with the pull-request diff --git a/.github/workflows/release-dev-image.yml b/.github/workflows/release-dev-image.yml index a85e8ed25..7a97bf9ff 100644 --- a/.github/workflows/release-dev-image.yml +++ b/.github/workflows/release-dev-image.yml @@ -137,6 +137,17 @@ jobs: cargo install --locked --version 0.9.0 --features cli cargo-about ./devtools/generate-dev-license-notices --check + - name: Gate - the release freezes the catalog migrations it ships + # As in release-image.yml: RELEASED_THROUGH must reach the last + # migration id. A tag before the migration framework passes. + run: | + set -euo pipefail + if [[ -e crates/storage/src/migrations/registry.rs ]]; then + python3 devtools/check-released-through + else + echo "the tagged commit has no catalog migration registry" + fi + build: needs: gate strategy: diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 37775b3d7..955651f18 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -210,6 +210,18 @@ jobs: ./devtools/generate-software-license-notices --check \ --backend '${{ steps.meta.outputs.license_backend }}' + - name: Gate - the release freezes the catalog migrations it ships + # RELEASED_THROUGH must reach the last migration id, so each + # backend's checksum test pins every body this tag ships. A tag + # before the migration framework has no registry and passes. + run: | + set -euo pipefail + if [[ -e crates/storage/src/migrations/registry.rs ]]; then + python3 devtools/check-released-through + else + echo "the tagged commit has no catalog migration registry" + fi + build: needs: gate # If one architecture fails, cancel the other: nothing is published either diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index c76b06cac..043d4caf1 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -1,15 +1,18 @@ name: Scripts test # Unit tests for the JavaScript under .github/scripts (Node's built-in runner, -# no dependencies to install). +# no dependencies to install), and for the Python helpers under devtools that +# the catalog check scripts report through (the standard library's unittest). on: pull_request: paths: - '.github/scripts/**' + - 'devtools/**' - '.github/workflows/scripts-test.yml' push: branches: [main] paths: - '.github/scripts/**' + - 'devtools/**' - '.github/workflows/scripts-test.yml' permissions: @@ -33,3 +36,18 @@ jobs: - name: Run tests # The shell expands the glob, so this works on every Node version. run: node --test .github/scripts/*.test.js + + devtools: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out devtools + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: devtools + persist-credentials: false + + - name: Run tests + # A check that passed on its exit code alone, or on its output + # alone, would let the catalog check scripts pass a broken binary. + run: python3 -m unittest discover -s devtools -p '*_test.py' -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20cd1f7bf..4c92f7f46 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,6 +98,471 @@ write code: scan, multi-tenancy, error fidelity, and operational correctness), plus the conformance stress tests reviewers will run. +## Adding a catalog migration + +Every change to a catalog's schema, or to data the catalog already holds, is a +numbered migration. The numbers are the same on every backend. Each store +records the migrations it applied in a ledger table, `extenddb_migrations`, +and `extenddb migrate --yes` applies the missing ones in order. +[ADR-0023](docs/adr/0023-catalog-migrations.md) records the design. + +SQLite and PostgreSQL migrate through this framework. Cassandra and MongoDB +still migrate through their bootstrappers: a Cassandra change is a numbered +CQL file listed in `crates/storage-cassandra/src/migrations.rs`, and a MongoDB +change is code in `crates/storage-mongodb/src/bootstrapper.rs`. + +### Procedure + +1. Add the id. Append an entry to the `migration_ids!` list in + `crates/storage/src/migrations/registry.rs` with the next number, a + snake_case name and a one-line description, which `extenddb migrate --list` + prints. The list generates `MigrationId`, `MigrationId::ALL` and each id's + name and description, and `MigrationId::LATEST` is its last entry. A number + is never reused, reordered or removed. + +2. Add an arm for the id to each backend's table, `migration()` in + `crates/storage-sqlite/src/migrations/mod.rs` and in + `crates/storage-postgres/src/migrations/mod.rs`. Both are exhaustive + `match`es with no wildcard arm, so the workspace does not compile until + every table has the arm. The arm is one of: + + - `Migration::Applies { kind, convergent, bodies }`, with one body for each + kind of store the migration changes: `StoreKind::Catalog`, and on + PostgreSQL also `StoreKind::Data`. The runner applies the catalog body + first, and it writes a catalog ledger row for every id, including an id + whose only body runs in the data database. + - `Migration::NotApplicable("reason")`, when the change has no effect on + that backend. The reason says why. The catalog ledger records it as + `not_applicable`, and it counts as additive and convergent. + + On PostgreSQL, also add the id to `runs_in()` in the same file. Every id + runs in the runner's transaction (`RunsIn::Transaction`) except one whose + code body builds indexes with `CREATE INDEX CONCURRENTLY`, which runs + statement by statement (`RunsIn::Autocommit`). + +3. Write each body, as a file or as code. + + A file body is SQL that the runner executes in one transaction with the + store's ledger row. Name it `NNNN_.sql`, where `NNNN` is the id in + four digits and `` is the id's name. PostgreSQL catalog bodies go in + `crates/storage-postgres/migrations/` and data bodies in + `crates/storage-postgres/data_migrations/`, and a SQLite file goes in + `crates/storage-sqlite/migrations/`. The backend's arm in `mod.rs` embeds + the file with the `file_body!` macro of `extenddb-storage`, + `file_body!(store, "", "NNNN_.sql")`, which names the + file once; the directory starts at the backend crate's root, and both SQL + backends use the same `file_body!`. The runner owns the transaction and writes the + version, so a file must not contain a `BEGIN`, `COMMIT`, `END`, + `ROLLBACK`, `SAVEPOINT` or `RELEASE` statement, and must not name + `catalog_version` outside a comment. Write every statement with + `IF NOT EXISTS` or `IF EXISTS`, so that a second run changes nothing. + + A code body is Rust, for what a file cannot express: a data rewrite, a + step that depends on what the store holds, or an index build outside a + transaction. Put it in its own module, `src/migrations/mNNNN_.rs` in + the backend's crate, and declare the module with `#[rustfmt::skip]` and + `#[allow(clippy::all, deprecated)]`, so that neither tool can require an + edit to a released body. The module defines: + + - `SOURCE`, the module's own text through `include_str!`, followed by the + text of any SQL file the body executes. The checksum test pins its + SHA-256. + - `FINGERPRINT`, which the ledger records. It starts with `NNNN_/`, + which the checksum test checks: a backend's own body uses + `"NNNN_//1"`, and a body shared by every backend, such as + migration 2's, `"NNNN_/1"`. A released body keeps its fingerprint. + - An implementation of `CodeBody` (`crates/storage/src/migrations/code.rs`). + `detect` is read-only and returns the items that would stop the body. + The runner calls it for every pending code body before the first write + of a run, and stops with exit 5 when any item comes back. `run` is + idempotent and safe to stop at any point, and returns `Done { notes }`, + which the ledger records and `migrate` and `verify` print. + + `detect` sees the stores as they are before the run's first pending + migration, so it may read only objects that no earlier pending migration + changes. A code body may call application code such as `ManagementStore`. + A function that defines what the migration does belongs in the module, + so that the module's checksum covers it, as `rewrite_policy_document` is in + `crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs`. + + Every code body has a golden test on each backend it runs on, apart from + migration 1, which the runner checks against + `0001_baseline.expected.json`. The test + runs the body on a committed released fixture and compares what the body + changed, and its ledger notes, with `tests/fixtures/golden/NNNN_.txt` + in the backend's crate. SQLite's golden tests are in + `crates/storage-sqlite/src/migrations/golden_tests.rs`, and PostgreSQL's + in `crates/storage-postgres/tests/migrations_golden.rs`. With + `EXTENDDB_BLESS_EXPECTED=1` the test writes the golden file instead of + comparing; read the file before you commit it. + + On PostgreSQL, a body that runs `CREATE INDEX CONCURRENTLY` builds every + index through `IndexHelper` in + `crates/storage-postgres/src/migrations/index_helper.rs`. `ensure` drops + an index that a cancelled build left invalid and builds it again, and + `finish` checks that every index the body asked for is valid before the + ledger row is written. `CREATE INDEX CONCURRENTLY IF NOT EXISTS` on its own + skips an invalid index and leaves it invalid. + +4. Declare the kind. A migration is `Kind::Additive` when the previous + release reads and writes correctly against the result, and everything the + previous release writes reads back through the new release the same as + what the new release writes. Otherwise it is `Kind::Breaking`. New tables, + nullable or defaulted columns and non-unique catalog indexes can be + additive. A column that the new release needs to read back what it writes + is breaking, unless the new release derives the value when it is missing. + Unique indexes and constraints, and changes to per-table or per-account + objects, are never additive. A 1.x binary serves a catalog whose unknown + migrations are all additive, and refuses one with a breaking migration it + does not know (exit 4), so the kind decides whether a rollback can run the + previous binary or has to restore a backup. + + Declaring `Kind::Additive` on a backend requires the old-binary job to + pass on that backend. In that job the previous release's binary and its + wire tests run against a catalog the new binary migrated, with writes from + both binaries interleaved, and every object the old binary writes is read + back through the new binary and compared with the same object written by + the new binary. The job does not exist yet; the first PR that declares an + additive migration adds it, and deletes the tests that refuse + `Kind::Additive` until then (`no_migration_is_additive_before_the_old_binary_job` + in the SQLite crate, and its PostgreSQL counterpart in `crates/bin`). + + Set `convergent: true` when every body is safe to run again on a store at + any later migration, over data that clients wrote through the API since it + was applied. A body that rewrites such data is not convergent: migration 2 + rewrites stored policies, and running it again would rewrite a policy + written after it and change that policy's decision. + `extenddb migrate --yes --reconverge`, which repairs a catalog that a + release before 1.0 wrote to, runs the convergent migrations again and + skips a recorded migration that is not convergent, with a line that names + it. + +5. Handle per-table objects. CreateTable and CreateAccount create objects at + request time with the binary's own DDL: the `_ddb_` tables and their + indexes on PostgreSQL, the index tables on SQLite, and on Cassandra the + `items_` tables and each account's keyspace. Their shape depends on + the release that created them. A migration that changes them is breaking. + It lists the objects from the catalog rows, never by name in the data + store, where PostgreSQL's vector tables share the `_ddb_` prefix. It skips + a catalog row whose table is `DELETING` or whose physical table is + missing, and adds a note for it. The same PR changes the DDL that creates + new objects, in `crates/storage-sqlite/src/data/ddl.rs` and + `crates/storage-postgres/src/data/ddl.rs`. Migration 4, + `m0004_index_table_base_key.rs` in each backend's `src/migrations/`, does + this. + +6. Commit the current schema. Each backend's integration job regenerates + `docs/schema/.txt` from a fresh install. When the result differs + from the committed file, the job fails, prints the diff and uploads the + regenerated file as an artifact. Commit that file with the migration; it + is where a reader finds a backend's current schema. To regenerate it + locally, dump a fresh install with `devtools/schema-dump`: `devtools/schema-dump + sqlite `, or `devtools/schema-dump postgres + ` for a deployment `init` created on that server. + The dump lists the kinds of object `devtools/schema-dump`'s docstring + names; a migration that creates another kind adds a query for it there. + +7. Leave the checksum literals to the release. Each backend's `RELEASED` + table pins the checksum, kind and convergent flag of every released body, + which is every migration at or below `RELEASED_THROUGH` in + `crates/storage/src/migrations/registry.rs`. The test + `the_released_bodies_match_their_checksums` refuses a released body that + changed. A migration above `RELEASED_THROUGH` carries no literal, so its + body can change during review with nothing to paste. The release PR that + raises `RELEASED_THROUGH` to the last id pastes the lines the test + prints; the release workflows' gate jobs run + `devtools/check-released-through` on the tagged commit and refuse a tag + whose bound is below the last id: + + ```bash + cargo test -p extenddb-storage-sqlite --lib the_released_bodies -- --nocapture + cargo test -p extenddb-storage-postgres --lib the_released_bodies -- --nocapture + ``` + + A released body does not change. If one has to, for example because a + dependency upgrade changes a code body's source, add a second `RELEASED` + entry for it with the new checksum and `rebless: Some("")`. A + code body keeps its fingerprint, and its golden test shows that its output + did not change. A later migration leaves migration 1's file and + `0001_baseline.expected.json` as they are: they define the v1.0 schema, + which the runner checks a store against the first time migration 1 runs on + it. + +8. Clear a changed migration from your development database. When a + migration's body changes after it ran on your database, `extenddb migrate` + reports `Changed` and exits 5. `extenddb migrate --forget ` deletes + that migration's ledger rows, and the next `extenddb migrate --yes` runs + it again. `--forget` refuses a migration at or below `RELEASED_THROUGH`. + +### Example: a nullable column on `tables` + +Migration 5, `table_note`, adds a nullable `note TEXT` column to the catalog's +`tables` table on SQLite and PostgreSQL. + +In `crates/storage/src/migrations/registry.rs`, at the end of the +`migration_ids!` list: + +```rust + /// Adds a nullable `note` column to the `tables` catalog table. + TableNote = 5, "table_note", + "add a nullable note column to the tables catalog"; +``` + +PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file, +`crates/storage-postgres/migrations/0005_table_note.sql`: + +```sql +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 5, table_note, PostgreSQL catalog database: a nullable note +-- column on tables. IF NOT EXISTS makes a second run change nothing. + +ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT; +``` + +In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`: + +```rust + MigrationId::TableNote => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Catalog, + "migrations", + "0005_table_note.sql" + )], + }, +``` + +and the id in the transactional arm of `runs_in()`: + +```rust + MigrationId::Baseline + | MigrationId::ResourceTagConditionKeys + | MigrationId::SortKeyDefinitions + | MigrationId::TableNote => RunsIn::Transaction, +``` + +SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain +`ADD COLUMN` fails when it runs a second time, so the SQLite body is code +that checks for the column first. +`crates/storage-sqlite/src/migrations/m0005_table_note.rs`: + +```rust +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 5, `table_note`, SQLite: a nullable `note` column on `tables`. +//! +//! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's +//! columns first and adds the column only when it is missing. A second run +//! changes nothing, which makes the migration convergent. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; + +use super::SqliteStores; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0005_table_note.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0005_table_note/sqlite/1"; + +pub(crate) struct TableNote; + +#[async_trait] +impl CodeBody for TableNote { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + let has_note: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pragma_table_info('tables') WHERE name = 'note')", + ) + .fetch_one(&stores.pool) + .await + .map_err(|e| MigrationError::store("read the columns of tables", e))?; + if !has_note { + sqlx::query("ALTER TABLE tables ADD COLUMN note TEXT") + .execute(&stores.pool) + .await + .map_err(|e| MigrationError::store("add tables.note", e))?; + } + Ok(Done::default()) + } +} +``` + +In `crates/storage-sqlite/src/migrations/mod.rs`, the module declaration +beside the others: + +```rust +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0005_table_note; +``` + +and the arm in `migration()`: + +```rust + MigrationId::TableNote => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0005_table_note::TableNote, + m0005_table_note::SOURCE, + )], + }, +``` + +The golden test, at the end of +`crates/storage-sqlite/src/migrations/golden_tests.rs`, runs the migrations +on the fixture of the v0.1.14 dev image and records the new column and the +rows of `tables`: + +```rust +#[tokio::test] +async fn migration_5_adds_the_note_column_to_tables() { + let db = temp_db("m5"); + load_dump(&db, DEV_0_1_14).await; + run(&db).await.expect("migrate"); + let mut text = dump( + &db, + "SELECT name, type, \"notnull\", dflt_value FROM pragma_table_info('tables') \ + WHERE name = 'note'", + ) + .await; + text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await); + golden("0005_table_note", &text); +} +``` + +`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_5` +writes `crates/storage-sqlite/tests/fixtures/golden/0005_table_note.txt`, one +tab-separated line per row under each query: + +``` +-- SELECT name, type, "notnull", dflt_value FROM pragma_table_info('tables') WHERE name = 'note' +note TEXT 0 NULL +-- SELECT table_name, note FROM tables ORDER BY table_name +cu_orders NULL +``` + +The PostgreSQL body is a file, so it has no golden test. + +The example declares `Kind::Breaking`. A nullable column can be additive by +the rule in step 4, but declaring it additive needs the old-binary job, which +this example does not add. As a breaking migration it makes a 1.x binary +without migration 5 refuse a catalog that has it, and a rollback restores the +backup. + +The same PR changes the code that reads and writes `tables.note` in each +backend's catalog store, and commits the `docs/schema/sqlite.txt` and +`docs/schema/postgres.txt` that the integration jobs regenerate. It changes +no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED` +entry. `extenddb migrate --list` then shows migration 5, and on a catalog at +migration 4 `extenddb migrate` lists migration 5 as pending and exits 3. + +### Running the migration tests + +The framework's and SQLite's tests need no database: + +```bash +cargo test -p extenddb-storage -p extenddb-storage-sqlite +``` + +They cover the checksum rules, the golden tests, an upgrade of every +committed released SQLite catalog compared with a fresh install, and a +migrator killed inside a body and then run again. + +PostgreSQL's unit tests need no database either: +`cargo test -p extenddb-storage-postgres --lib`. Its migration integration +tests need a PostgreSQL server and a role that can create databases; each +test creates its own databases and drops them. The vector fixtures need the +pgvector package, which the `pgvector/pgvector:pg16` image that CI uses has: + +```bash +docker run -d --name extenddb-test-pg -p 127.0.0.1:5432:5432 \ + -e POSTGRES_PASSWORD=devpass pgvector/pgvector:pg16 +export EXTENDDB_TEST_PG_CONNECTION_STRING=postgresql://postgres:devpass@127.0.0.1:5432 +cargo test -p extenddb-storage-postgres --tests +docker rm -f -v extenddb-test-pg +``` + +`--tests` runs every target of the crate, as CI does. The connection +string names the server, with no database. Without +`EXTENDDB_TEST_PG_CONNECTION_STRING` each test prints a skip line and +passes, so check the output for skips; CI sets `EXTENDDB_TEST_PG_REQUIRED=1`, +which turns a missing connection string into a failure. `EXTENDDB_TEST_PGVECTOR_CONNECTION_STRING` +points the vector fixtures at a second server when the first has no +pgvector. `migrations_tag_blobs` needs the release tags in your clone +(`git fetch --tags`). + +Two scripts check SQLite end to end with a release binary, as CI's +`run-integration-sqlite` job runs them: + +```bash +cargo build --release -p extenddb --no-default-features --features sqlite +devtools/check-sqlite-catalog-lock ./target/release/extenddb 18450 +docker pull extenddb/extenddb-dev:0.1.14 +devtools/check-sqlite-older-binary ./target/release/extenddb \ + crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql \ + extenddb/extenddb-dev:0.1.14 18451 +``` + +`check-sqlite-catalog-lock` checks that `migrate --yes` and `serve` exclude +each other, that the `migrate` preview and `verify` run beside a server, +that `verify` exits 3 on a catalog behind the binary, and that +`migrate --yes` refuses a missing database without creating it. +`check-sqlite-older-binary` runs the last 0.1.x dev image against catalogs +this binary migrated, and checks that the binary then reports +`WrittenByOlderRelease` and that `extenddb migrate --yes --reconverge` +restores `Current`; with `DEV_BIN` set to a `sqlite,dev-mode` build it also +checks the repair a dev server without a config file advises. + +Two scripts check PostgreSQL the same way, as CI's `run-integration` job +runs them, against a server whose superuser URL (with no database) they are +given: + +```bash +cargo build --release -p extenddb +devtools/check-postgres-catalog-lock ./target/release/extenddb \ + postgresql://postgres:devpass@127.0.0.1:5432 18452 +docker pull extenddb/extenddb-postgres:0.1.14 +devtools/check-postgres-older-binary ./target/release/extenddb \ + postgresql://postgres:devpass@127.0.0.1:5432 \ + crates/storage-postgres/tests/fixtures/released/0.1.14 \ + extenddb/extenddb-postgres:0.1.14 18453 +``` + +`check-postgres-catalog-lock` checks the migration lock and the serving +registration between `serve` and `migrate`; `check-postgres-older-binary` +runs the 0.1.14 image against catalogs this binary created and migrated. +With `POOLER_URL` set to a connection pooler in front of the same server +(`postgresql://127.0.0.1:6432` for the PgBouncer that CI starts in +transaction mode, logging roles in through `auth_query` as the superuser), +`check-postgres-catalog-lock` also runs `verify`, the preview, +`migrate --yes`, `--reconverge`, `--forget` and `serve` through it; without +it the script prints a skip line. `check-postgres-older-binary` names the +image's containers `$CONTAINER` (default `extenddb-older-binary`). + +The four scripts are Python and need Python 3.10 or later. +`check-sqlite-catalog-lock`, `check-sqlite-older-binary` and +`check-postgres-catalog-lock` also call `curl` to read a server's `/health` +endpoint, so `curl` must be on the `PATH`. The scripts import +`devtools/catalog_checks.py`, which runs a command with a time limit, checks +its exit code and prints one `ok` or `FAIL` line per check; a new check +script imports it too. + ## License By contributing, you agree that your contributions will be licensed under the diff --git a/Cargo.lock b/Cargo.lock index c59cdea4e..90aa1a9e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1690,6 +1690,9 @@ dependencies = [ "extenddb-storage-mongodb", "extenddb-storage-postgres", "extenddb-storage-sqlite", + "sqlx", + "tokio", + "uuid", ] [[package]] @@ -1876,6 +1879,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", + "sha2 0.10.9", "thiserror", "time", "tokio", @@ -1989,6 +1993,8 @@ dependencies = [ "extenddb-storage-sqlite", "futures", "libc", + "libsqlite3-sys", + "log", "rand 0.9.4", "serde", "serde_json", @@ -1997,6 +2003,7 @@ dependencies = [ "tokio", "toml", "tracing", + "tracing-subscriber", "uuid", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index b8bc187ee..8ba393e8e 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,11 @@ moka = { version = "0.12", features = ["future"] } # Database cdrs-tokio = { version = "9.0", features = ["derive"] } sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls-aws-lc-rs", "postgres", "json", "time", "uuid", "bigdecimal", "derive"] } +# SQLite's C API, for the online backup API that sqlx does not expose. The +# version is the one sqlx-sqlite depends on: the crate declares +# `links = "sqlite3"`, so one build holds one version of it, and the features +# are sqlx's. +libsqlite3-sys = { version = "0.30.1", default-features = false } # Vector column type for the PostgreSQL backend. The `sqlx` feature gives a typed # `Vector` that encodes and decodes pgvector's binary format, so embeddings keep # f32 bit-exactness instead of going through a hand-written text parser. diff --git a/README.md b/README.md index 20e8a0cf7..c6d931af8 100755 --- a/README.md +++ b/README.md @@ -149,11 +149,12 @@ unreachable at startup does stop the server from listening. There is no separate ### One instance per database Run one `extenddb serve` per catalog. On SQLite this is enforced: the server takes an exclusive lock on `.lock` -next to the database file and a second server on the same file will refuse to start (`init`, `migrate`, and `destroy` take -the same lock while they run). On PostgreSQL, MongoDB and Cassandra nothing prevents a second ExtendDB instance from -running, but running more than one against the same catalog is not supported in this release: every instance runs every -background worker, credential and policy caches are per instance with no cross-instance invalidation (a revoked key -stays valid on the other instance for up to the cache TTL), and `/health` does not check the database. +next to the database file and a second server on the same file will refuse to start (`init`, `migrate --yes`, +`migrate --forget` and `destroy` take the same lock while they run; the `migrate` preview and `verify` take none). On +PostgreSQL, MongoDB and Cassandra nothing prevents a second ExtendDB instance from running, but running more than one +against the same catalog is not supported in this release: every instance runs every background worker, credential and +policy caches are per instance with no cross-instance invalidation (a revoked key stays valid on the other instance for +up to the cache TTL), and `/health` does not check the database. See the [deployment guide](docs/manuals/11-deployment-guide.md#multi-instance-considerations). To make the generated self-signed certificate valid for the name clients use — an in-cluster service DNS name, for example — pass `--tls-san` to `init` (repeatable): diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index bd4b77644..674ffe010 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -144,8 +144,12 @@ fn discover_docs_dir() -> Option { None } +/// How long `init` waits for the catalog's migration lock, which another +/// `init` or a `migrate` may hold. +const INIT_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + /// Returns exit code: 0 = success, 255 = existing config preserved. -pub async fn run(args: InitArgs) -> anyhow::Result { +pub async fn run(args: InitArgs, release: &str) -> anyhow::Result { // The installed backend is authoritative — there is exactly one compiled // into this binary. An explicit --backend (or a config file's backend key) // is validated against it rather than driving dispatch, so a mismatch is a @@ -209,8 +213,8 @@ pub async fn run(args: InitArgs) -> anyhow::Result { )?; // Extract bind_addr from CLI args - let bind_addr = - extract_arg(&cli_args, "--bind-addr").unwrap_or_else(|| "127.0.0.1".to_string()); + let bind_addr = extenddb_storage::bootstrapper::helpers::extract_arg(&cli_args, "--bind-addr") + .unwrap_or_else(|| "127.0.0.1".to_string()); // Generate the self-signed TLS certificate if it isn't already present, // covering the bind address plus any --tls-san values so it matches the URLs @@ -248,29 +252,32 @@ pub async fn run(args: InitArgs) -> anyhow::Result { .await .map_err(crate::cmd_migrate::op_err)?; - // Take the same lock `migrate` uses around the schema work below, so an - // init cannot race a migrate running on another replica and fail on a - // duplicate pg_type entry from concurrent `CREATE TABLE IF NOT EXISTS`. - // Two concurrent inits cannot get this far: the second aborts earlier, at - // `create_catalog_db`, because the database already exists. The catalog - // database does exist by this point, so the lock connection can be opened. - // - // The lock is held through the bootstrap rows as well, not just the - // schema: a server that starts after the schema exists but before the - // encryption key is written fails with MissingEncryptionKey, and on - // SQLite this same lock is what keeps `serve` out until init is done. - bootstrapper - .acquire_migration_lock() - .await - .map_err(crate::cmd_migrate::op_err)?; - let bootstrap_result = run_init_bootstrap(bootstrapper.as_ref()).await; - if let Err(e) = bootstrapper.release_migration_lock().await { - tracing::warn!( - "Failed to release migration lock: {}", - crate::cmd_migrate::op_err(e) - ); - } - let admin_result = bootstrap_result?; + let admin_result = if let Some(migrations) = extenddb_storage::migrations::installed() { + init_on_runner(bootstrapper.as_ref(), migrations, release).await? + } else { + // Take the same lock `migrate` uses around the schema work below, so an + // init cannot race a migrate running on another replica and fail on a + // duplicate pg_type entry from concurrent `CREATE TABLE IF NOT EXISTS`. + // Two concurrent inits cannot get this far: the second aborts earlier, at + // `create_catalog_db`, because the database already exists. The catalog + // database does exist by this point, so the lock connection can be opened. + // + // The lock is held through the bootstrap rows as well, not just the + // schema: a server that starts after the schema exists but before the + // encryption key is written fails with MissingEncryptionKey. + bootstrapper + .acquire_migration_lock() + .await + .map_err(crate::cmd_migrate::op_err)?; + let bootstrap_result = run_init_bootstrap(bootstrapper.as_ref()).await; + if let Err(e) = bootstrapper.release_migration_lock().await { + tracing::warn!( + "Failed to release migration lock: {}", + crate::cmd_migrate::op_err(e) + ); + } + bootstrap_result? + }; if admin_result.already_existed { // Already printed by the bootstrap store. @@ -324,6 +331,43 @@ pub async fn run(args: InitArgs) -> anyhow::Result { Ok(0) } +/// `init` on a backend on the migration runner, all under the migration +/// lock: the catalog's status, which must be `Uninitialized` (another `init` +/// may have created the catalog while this one waited), then every +/// migration, then the rows a server needs before it can start, then the +/// data connection rows. Holding the lock through the rows keeps a server +/// from starting on a catalog that has no encryption key yet. The target +/// carries what the bootstrapper resolved from the command line, so the +/// backend reads no flag itself. +async fn init_on_runner( + bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, + migrations: &dyn extenddb_storage::migrations::CatalogMigrations, + release: &str, +) -> anyhow::Result { + use extenddb_storage::migrations::{RunKind, run_locked}; + + let target = bootstrapper.init_target(); + let lock = migrations + .lock(&target, INIT_LOCK_TIMEOUT) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + println!("--- Creating the catalog through its migrations..."); + let report = run_locked(migrations, &target, &lock, release, RunKind::Create) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + for note in &report.notes { + println!(" {note}"); + } + crate::util::print_applied(&report, " ", "Applied migration"); + let admin = write_bootstrap_rows(bootstrapper).await?; + bootstrapper + .record_data_connection() + .await + .map_err(crate::cmd_migrate::op_err)?; + drop(lock); + Ok(admin) +} + /// Apply the catalog and data schema while the migration lock is held. Split out /// of `run` so that the lock is released on every path, including errors. /// Everything `init` does under the migration lock: the schema, then the @@ -333,7 +377,14 @@ async fn run_init_bootstrap( bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, ) -> anyhow::Result { run_init_migrations(bootstrapper).await?; + write_bootstrap_rows(bootstrapper).await +} +/// The rows a server needs before it can start: encryption key, default +/// account, admin user. +async fn write_bootstrap_rows( + bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, +) -> anyhow::Result { bootstrapper .bootstrap_encryption_key() .await @@ -386,11 +437,6 @@ async fn run_init_migrations( Ok(()) } -/// Extract a CLI argument value by flag name. -fn extract_arg(args: &[String], flag: &str) -> Option { - args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone()) -} - #[cfg(test)] mod tests { use super::InitArgs; diff --git a/crates/app/src/cmd_migrate.rs b/crates/app/src/cmd_migrate.rs index 2c9ea1be3..dfc2b3255 100755 --- a/crates/app/src/cmd_migrate.rs +++ b/crates/app/src/cmd_migrate.rs @@ -3,12 +3,15 @@ //! `extenddb migrate` — apply catalog schema migrations (REQ-CAT-014). //! -//! Reads current catalog version, runs pending migrations, and reports the result. -//! -//! Exit status: 0 when nothing is pending or the migrations were applied; -//! [`EXIT_PENDING`] (3) when work is pending and `--yes` was not given (the -//! preview); 1 on any error. A scripted upgrade can run the preview under -//! `set -e` by checking for 3 explicitly. +//! A backend on the migration runner (PostgreSQL, SQLite) takes +//! `cmd_migrate_runner`'s path, whose exit statuses are 0, 1, 3, 4, 5 and 6 +//! (see that module). This module is the bootstrapper's path, for Cassandra +//! and MongoDB: it reads the current catalog version, runs pending +//! migrations, and reports the result. Exit status: 0 when nothing is +//! pending or the migrations were applied; [`EXIT_PENDING`] (3) when work is +//! pending and `--yes` was not given (the preview); 1 on any error. A +//! scripted upgrade can run the preview under `set -e` by checking for 3 +//! explicitly. use clap::Args; @@ -18,22 +21,30 @@ use extenddb_storage::management_store::OpError; /// Exit status of a preview (`migrate` without `--yes`) that found work to do. /// Distinct from 1 so a script can tell "pending" from "failed". -pub const EXIT_PENDING: u8 = 3; +pub const EXIT_PENDING: u8 = extenddb_storage::migrations::exit::PENDING; + +/// The config file `migrate` and `verify` read when `--config` is not given. +pub(crate) const DEFAULT_CONFIG: &str = "extenddb.toml"; + +/// Seconds `--yes` and `--forget` wait for the migration lock by default. +pub(crate) const DEFAULT_LOCK_TIMEOUT_SECS: u64 = 60; #[derive(Args)] pub struct MigrateArgs { - /// Path to configuration file - #[arg(short, long, default_value = "extenddb.toml")] - config: String, + /// Path to configuration file (default: extenddb.toml; a dev-mode build + /// without one reads the EXTENDDB__ environment variables, as serve does) + #[arg(short, long)] + pub(crate) config: Option, - /// `PostgreSQL` admin user (for catalog migrations) + /// `PostgreSQL` admin role for the pgvector extension attempt; schema + /// work runs as the application role in the config file #[arg(long)] - pg_user: Option, + pub(crate) pg_user: Option, /// `PostgreSQL` admin password. /// Prefer `EXTENDDB_PG_PASSWORD` to keep the value out of process arguments. #[arg(long)] - pg_pass: Option, + pub(crate) pg_pass: Option, /// `PostgreSQL` data database to migrate (PostgreSQL only; ignored on /// other backends). Normally not needed: the name `init` recorded in the @@ -41,7 +52,7 @@ pub struct MigrateArgs { /// `_catalog` suffix. Pass it for a catalog that has no recorded name and /// a data database named otherwise. #[arg(long)] - data_db: Option, + pub(crate) data_db: Option, /// Cassandra admin user (the role `init` used); without it the /// connection is attempted as `$USER` @@ -56,7 +67,34 @@ pub struct MigrateArgs { /// Apply the pending migrations. Without it, migrate only reports what is /// pending and exits 3 if there is any (0 if everything is up to date) #[arg(long)] - yes: bool, + pub(crate) yes: bool, + + /// Print every migration this binary carries, with its kind on this + /// backend, and exit without opening the catalog + #[arg(long, conflicts_with_all = ["yes", "forget", "reconverge"])] + pub(crate) list: bool, + + /// With --yes: run every convergent migration again, to repair a catalog + /// a release before 1.0 wrote to after it was migrated + #[arg(long, requires = "yes", conflicts_with = "forget")] + pub(crate) reconverge: bool, + + /// Delete the ledger rows of an unreleased migration whose body changed on + /// a development database, so the next `migrate --yes` runs it again + #[arg(long, value_name = "ID", conflicts_with = "yes")] + pub(crate) forget: Option, + + /// Seconds --yes and --forget wait for a server or another migration to + /// release the catalog before exiting 6 (default 60, at most 86400) + #[arg(long, value_name = "SECONDS", value_parser = clap::value_parser!(u64).range(0..=86_400))] + pub(crate) lock_timeout: Option, +} + +impl MigrateArgs { + /// The config file named by `--config`, else the default. + pub(crate) fn config_path(&self) -> &str { + self.config.as_deref().unwrap_or(DEFAULT_CONFIG) + } } /// The message of a bootstrapper error, without the variant name that `{e:?}` @@ -67,7 +105,8 @@ pub(crate) fn op_err(e: OpError) -> anyhow::Error { | OpError::AlreadyExists(m) | OpError::NotFound(m) | OpError::HasDependents(m) - | OpError::Internal(m) => anyhow::anyhow!(m), + | OpError::Internal(m) + | OpError::Database { message: m, .. } => anyhow::anyhow!(m), } } @@ -80,22 +119,34 @@ pub(crate) fn storage_err(e: StorageError) -> anyhow::Error { } } -/// Run `extenddb migrate`; returns the process exit status (see the module docs). -pub async fn run(args: MigrateArgs) -> anyhow::Result { - if !std::path::Path::new(&args.config).exists() { +/// Run `extenddb migrate`; returns the process exit status (see the module +/// docs). A backend on the migration runner takes `cmd_migrate_runner`'s +/// path; this one is the bootstrapper's. +pub async fn run(args: MigrateArgs, release: &str) -> anyhow::Result { + if let Some(migrations) = extenddb_storage::migrations::installed() { + return crate::cmd_migrate_runner::run(&args, migrations, release).await; + } + if args.list || args.reconverge || args.forget.is_some() || args.lock_timeout.is_some() { anyhow::bail!( - "Config file '{}' not found. Run 'extenddb init' to set up a deployment, \ + "--list, --reconverge, --forget and --lock-timeout need the migration ledger, and \ + the {} backend still migrates through its bootstrapper", + extenddb_storage::backend_name().unwrap_or("installed") + ); + } + let config_path = args.config_path(); + if !std::path::Path::new(config_path).exists() { + anyhow::bail!( + "Config file '{config_path}' not found. Run 'extenddb init' to set up a deployment, \ or use --config to specify a different location.", - args.config, ); } // Load the config for validation and to reach the catalog for the policy // rewrite: migrate drives the schema steps through the bootstrapper, but a // malformed config should fail here rather than midway through a migration. - let app_config = config::load(&args.config)?; + let app_config = config::load(config_path)?; println!("=== extenddb migrate ==="); - println!("Config: {}", args.config); + println!("Config: {config_path}"); println!(); // Collect CLI args for backend-specific parsing. The environment-sourced @@ -107,7 +158,7 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result { )?; // Create bootstrapper via registry - let bootstrap = extenddb_storage::bootstrapper::create_bootstrapper(&args.config, &cli_args) + let bootstrap = extenddb_storage::bootstrapper::create_bootstrapper(config_path, &cli_args) .await .map_err(storage_err)?; @@ -127,7 +178,7 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result { /// The argument vector the backend bootstrapper parses: `argv`, plus /// `--pg-pass` from `EXTENDDB_PG_PASSWORD` and `--cassandra-pass` from /// `EXTENDDB_CASSANDRA_PASSWORD` when the flag itself was not given. -fn backend_args( +pub(crate) fn backend_args( argv: Vec, pg_pass_env: Option, cassandra_pass_env: Option, @@ -203,7 +254,8 @@ async fn apply_migrations( // schema connection. println!("--- Checking resource-tag condition keys..."); let (rtag_detect, marker_present) = rtag.detect().await?; - if rtag_detect.rewritten.is_empty() && rtag_detect.needs_attention.is_empty() { + let rtag_attention = rtag_detect.needs_attention.len() + rtag_detect.unreadable.len(); + if rtag_detect.rewritten.is_empty() && rtag_attention == 0 { println!(" No dynamodb:ResourceTag condition keys found."); } else { for r in &rtag_detect.rewritten { @@ -219,6 +271,7 @@ async fn apply_migrations( n.account_id, n.principal, n.policy_name ); } + print_unreadable(&rtag_detect); } if rtag_detect.errors > 0 { println!( @@ -238,7 +291,7 @@ async fn apply_migrations( && !marker_pending { println!(); - if detected.needs_attention.is_empty() && rtag_detect.needs_attention.is_empty() { + if detected.needs_attention.is_empty() && rtag_attention == 0 { println!( "Everything is up to date (catalog version {expected}). No migrations needed." ); @@ -249,7 +302,7 @@ async fn apply_migrations( println!( "No migrations needed (catalog version {expected}), but {} item(s) above \ need manual attention.", - detected.needs_attention.len() + rtag_detect.needs_attention.len() + detected.needs_attention.len() + rtag_attention ); } return Ok(0); @@ -284,11 +337,10 @@ async fn apply_migrations( rtag_detect.errors )); } - if !rtag_detect.needs_attention.is_empty() { + if rtag_attention > 0 { what.push(format!( - "{} policy(ies) listed as NEEDS ATTENTION, which must be fixed by hand before \ - the migration can complete", - rtag_detect.needs_attention.len() + "{rtag_attention} policy(ies) listed as NEEDS ATTENTION, which must be fixed by \ + hand before the migration can complete" )); } // A preview, not a failure: report and exit EXIT_PENDING so a script @@ -341,6 +393,7 @@ async fn apply_migrations( n.account_id, n.principal, n.policy_name ); } + print_unreadable(&rtag_applied); if rtag_applied.errors > 0 { anyhow::bail!( "resource-tag migration incomplete: {} policy lookup(s) failed. \ @@ -362,6 +415,16 @@ async fn apply_migrations( rtag_applied.needs_attention.len() ); } + // A policy whose document cannot be decoded may name the dynamodb: form + // too, so the marker is withheld for it as for a collision. + if !rtag_applied.unreadable.is_empty() { + anyhow::bail!( + "resource-tag migration incomplete: {} policy(ies) above have a stored document \ + that cannot be decoded and were left unchanged. Replace or delete each, then \ + re-run `extenddb migrate --yes`; the migration marker was not recorded.", + rtag_applied.unreadable.len() + ); + } // Record the completion marker only after a clean rewrite. serve reads this // row and refuses to start without it. rtag.record_marker().await?; @@ -377,6 +440,18 @@ async fn apply_migrations( Ok(0) } +/// One `NEEDS ATTENTION` line per policy of `report` whose stored document +/// cannot be decoded, with the decode error. +fn print_unreadable(report: &extenddb_storage::policy_migration::MigrationReport) { + for u in &report.unreadable { + println!( + " NEEDS ATTENTION: account {} {} policy {} left unchanged (its stored document \ + cannot be decoded: {})", + u.account_id, u.principal, u.policy_name, u.error + ); + } +} + /// Boxed future returned by [`ResourceTagStep`]. type StepFuture<'a, T> = std::pin::Pin + Send + 'a>>; @@ -479,240 +554,8 @@ fn refuse_newer_catalog(current: Option<&str>, expected: &str) -> anyhow::Result } #[cfg(test)] -mod preview_tests { - //! The preview's exit status and that it changes nothing (it used to - //! exit 1 with "Error: --yes is required", which stops a - //! scripted upgrade under `set -e`). - use std::sync::Mutex; - - use super::{ResourceTagStep, StepFuture}; - - use async_trait::async_trait; - use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair}; - use extenddb_storage::management_store::{OpError, OpResult}; - - use super::{EXIT_PENDING, MigrateArgs, apply_migrations, op_err, storage_err}; - - struct Fake { - version: Mutex>, - data_pending: Mutex>, - catalog_runs: Mutex, - data_runs: Mutex, - } - - impl Fake { - fn new(version: Option<&str>, data_pending: &[&str]) -> Self { - Self { - version: Mutex::new(version.map(str::to_owned)), - data_pending: Mutex::new(data_pending.iter().map(|s| (*s).to_owned()).collect()), - catalog_runs: Mutex::new(0), - data_runs: Mutex::new(0), - } - } - fn runs(&self) -> (u32, u32) { - ( - *self.catalog_runs.lock().unwrap(), - *self.data_runs.lock().unwrap(), - ) - } - } - - fn unused() -> OpResult { - Err(OpError::Internal("not used by migrate".to_owned())) - } - - #[async_trait] - impl Bootstrapper for Fake { - async fn ensure_app_user(&self) -> OpResult<()> { - unused() - } - async fn grant_app_role_to_admin(&self) -> OpResult<()> { - unused() - } - async fn create_catalog_db(&self) -> OpResult<()> { - unused() - } - async fn create_data_db(&self) -> OpResult<()> { - unused() - } - async fn run_catalog_migrations(&self) -> OpResult<()> { - *self.catalog_runs.lock().unwrap() += 1; - *self.version.lock().unwrap() = Some(self.expected_catalog_version()); - Ok(()) - } - async fn run_data_migrations(&self) -> OpResult<()> { - *self.data_runs.lock().unwrap() += 1; - self.data_pending.lock().unwrap().clear(); - Ok(()) - } - async fn repair_lost_sort_key_definitions( - &self, - _apply: bool, - ) -> OpResult { - Ok(KeyDefinitionRepair::default()) - } - async fn pending_data_migrations(&self) -> OpResult> { - Ok(self.data_pending.lock().unwrap().clone()) - } - async fn record_data_connection(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_encryption_key(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_default_account(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_admin_user( - &self, - _env_user: Option<&str>, - _env_password: Option<&str>, - ) -> OpResult { - unused() - } - async fn is_catalog_initialized(&self) -> OpResult { - Ok(true) - } - async fn list_table_names(&self) -> OpResult> { - unused() - } - async fn get_data_db_name(&self) -> OpResult> { - unused() - } - async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { - unused() - } - async fn read_catalog_version(&self) -> OpResult> { - Ok(self.version.lock().unwrap().clone()) - } - fn expected_catalog_version(&self) -> String { - "0.0.4".to_owned() - } - fn catalog_database_name(&self) -> String { - "fake".to_owned() - } - fn endpoint_info(&self) -> String { - "fake".to_owned() - } - fn catalog_connection_url(&self) -> String { - "fake://".to_owned() - } - fn generate_backend_config_section(&self) -> String { - String::new() - } - } - - /// A catalog whose policies need no resource-tag rewrite and that already - /// records the completion marker. - struct NoResourceTags; - - impl ResourceTagStep for NoResourceTags { - fn detect( - &self, - ) -> StepFuture< - '_, - anyhow::Result<(extenddb_storage::policy_migration::MigrationReport, bool)>, - > { - Box::pin(async { Ok((Default::default(), true)) }) - } - fn rewrite(&self) -> StepFuture<'_, extenddb_storage::policy_migration::MigrationReport> { - Box::pin(async { Default::default() }) - } - fn record_marker(&self) -> StepFuture<'_, anyhow::Result<()>> { - Box::pin(async { Ok(()) }) - } - } - - fn args(yes: bool) -> MigrateArgs { - MigrateArgs { - config: "extenddb.toml".to_owned(), - pg_user: None, - pg_pass: None, - data_db: None, - cassandra_user: None, - cassandra_pass: None, - yes, - } - } - - #[tokio::test] - async fn a_preview_with_pending_work_exits_3_and_changes_nothing() { - for (version, data) in [ - (Some("0.0.3"), &[][..]), - (Some("0.0.4"), &["rebuild backup_items (2 rows)"][..]), - (None, &["004_x.sql"][..]), - ] { - let fake = Fake::new(version, data); - let code = apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .expect("preview"); - assert_eq!(code, EXIT_PENDING, "{version:?} {data:?}"); - assert_eq!(fake.runs(), (0, 0), "the preview applied something"); - assert_eq!( - fake.read_catalog_version().await.unwrap().as_deref(), - version - ); - } - } - - #[tokio::test] - async fn up_to_date_exits_0_with_or_without_yes() { - for yes in [false, true] { - let fake = Fake::new(Some("0.0.4"), &[]); - assert_eq!( - apply_migrations(&fake, &args(yes), &NoResourceTags) - .await - .unwrap(), - 0 - ); - assert_eq!(fake.runs(), (0, 0)); - } - } - - #[tokio::test] - async fn yes_applies_and_exits_0_and_a_second_preview_is_up_to_date() { - let fake = Fake::new(Some("0.0.3"), &["rebuild backup_items (2 rows)"]); - assert_eq!( - apply_migrations(&fake, &args(true), &NoResourceTags) - .await - .unwrap(), - 0 - ); - assert_eq!(fake.runs(), (1, 1)); - assert_eq!( - apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .unwrap(), - 0 - ); - } - - #[tokio::test] - async fn a_newer_catalog_is_an_error_not_a_preview() { - let fake = Fake::new(Some("0.0.5"), &[]); - let err = apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .expect_err("newer"); - assert!(err.to_string().contains("does not downgrade"), "{err}"); - assert_eq!(fake.runs(), (0, 0)); - } - - #[test] - fn errors_print_their_message_without_the_variant_name() { - let shown = format!( - "{:?}", - op_err(OpError::Internal("Cannot connect to PostgreSQL".to_owned())) - ); - assert_eq!(shown, "Cannot connect to PostgreSQL"); - let shown = format!( - "{:?}", - storage_err(extenddb_storage::error::StorageError::Internal( - "Cannot connect to PostgreSQL".to_owned() - )) - ); - assert_eq!(shown, "Cannot connect to PostgreSQL"); - } -} +#[path = "cmd_migrate_preview_tests.rs"] +mod preview_tests; #[cfg(test)] mod backend_args_tests { diff --git a/crates/app/src/cmd_migrate_preview_tests.rs b/crates/app/src/cmd_migrate_preview_tests.rs new file mode 100644 index 000000000..592ee11ef --- /dev/null +++ b/crates/app/src/cmd_migrate_preview_tests.rs @@ -0,0 +1,237 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The preview's exit status and that it changes nothing (it used to +//! exit 1 with "Error: --yes is required", which stops a +//! scripted upgrade under `set -e`). +use std::sync::Mutex; + +use super::{ResourceTagStep, StepFuture}; + +use async_trait::async_trait; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair}; +use extenddb_storage::management_store::{OpError, OpResult}; + +use super::{EXIT_PENDING, MigrateArgs, apply_migrations, op_err, storage_err}; + +struct Fake { + version: Mutex>, + data_pending: Mutex>, + catalog_runs: Mutex, + data_runs: Mutex, +} + +impl Fake { + fn new(version: Option<&str>, data_pending: &[&str]) -> Self { + Self { + version: Mutex::new(version.map(str::to_owned)), + data_pending: Mutex::new(data_pending.iter().map(|s| (*s).to_owned()).collect()), + catalog_runs: Mutex::new(0), + data_runs: Mutex::new(0), + } + } + fn runs(&self) -> (u32, u32) { + ( + *self.catalog_runs.lock().unwrap(), + *self.data_runs.lock().unwrap(), + ) + } +} + +fn unused() -> OpResult { + Err(OpError::Internal("not used by migrate".to_owned())) +} + +#[async_trait] +impl Bootstrapper for Fake { + async fn ensure_app_user(&self) -> OpResult<()> { + unused() + } + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + unused() + } + async fn create_catalog_db(&self) -> OpResult<()> { + unused() + } + async fn create_data_db(&self) -> OpResult<()> { + unused() + } + async fn run_catalog_migrations(&self) -> OpResult<()> { + *self.catalog_runs.lock().unwrap() += 1; + *self.version.lock().unwrap() = Some(self.expected_catalog_version()); + Ok(()) + } + async fn run_data_migrations(&self) -> OpResult<()> { + *self.data_runs.lock().unwrap() += 1; + self.data_pending.lock().unwrap().clear(); + Ok(()) + } + async fn repair_lost_sort_key_definitions( + &self, + _apply: bool, + ) -> OpResult { + Ok(KeyDefinitionRepair::default()) + } + async fn pending_data_migrations(&self) -> OpResult> { + Ok(self.data_pending.lock().unwrap().clone()) + } + async fn record_data_connection(&self) -> OpResult<()> { + unused() + } + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + unused() + } + async fn bootstrap_default_account(&self) -> OpResult<()> { + unused() + } + async fn bootstrap_admin_user( + &self, + _env_user: Option<&str>, + _env_password: Option<&str>, + ) -> OpResult { + unused() + } + async fn is_catalog_initialized(&self) -> OpResult { + Ok(true) + } + async fn list_table_names(&self) -> OpResult> { + unused() + } + async fn get_data_db_name(&self) -> OpResult> { + unused() + } + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + unused() + } + async fn read_catalog_version(&self) -> OpResult> { + Ok(self.version.lock().unwrap().clone()) + } + fn expected_catalog_version(&self) -> String { + "0.0.4".to_owned() + } + fn catalog_database_name(&self) -> String { + "fake".to_owned() + } + fn endpoint_info(&self) -> String { + "fake".to_owned() + } + fn catalog_connection_url(&self) -> String { + "fake://".to_owned() + } + fn generate_backend_config_section(&self) -> String { + String::new() + } +} + +/// A catalog whose policies need no resource-tag rewrite and that already +/// records the completion marker. +struct NoResourceTags; + +impl ResourceTagStep for NoResourceTags { + fn detect( + &self, + ) -> StepFuture<'_, anyhow::Result<(extenddb_storage::policy_migration::MigrationReport, bool)>> + { + Box::pin(async { Ok((Default::default(), true)) }) + } + fn rewrite(&self) -> StepFuture<'_, extenddb_storage::policy_migration::MigrationReport> { + Box::pin(async { Default::default() }) + } + fn record_marker(&self) -> StepFuture<'_, anyhow::Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +fn args(yes: bool) -> MigrateArgs { + MigrateArgs { + config: None, + pg_user: None, + pg_pass: None, + data_db: None, + cassandra_user: None, + cassandra_pass: None, + yes, + list: false, + reconverge: false, + forget: None, + lock_timeout: None, + } +} + +#[tokio::test] +async fn a_preview_with_pending_work_exits_3_and_changes_nothing() { + for (version, data) in [ + (Some("0.0.3"), &[][..]), + (Some("0.0.4"), &["rebuild backup_items (2 rows)"][..]), + (None, &["004_x.sql"][..]), + ] { + let fake = Fake::new(version, data); + let code = apply_migrations(&fake, &args(false), &NoResourceTags) + .await + .expect("preview"); + assert_eq!(code, EXIT_PENDING, "{version:?} {data:?}"); + assert_eq!(fake.runs(), (0, 0), "the preview applied something"); + assert_eq!( + fake.read_catalog_version().await.unwrap().as_deref(), + version + ); + } +} + +#[tokio::test] +async fn up_to_date_exits_0_with_or_without_yes() { + for yes in [false, true] { + let fake = Fake::new(Some("0.0.4"), &[]); + assert_eq!( + apply_migrations(&fake, &args(yes), &NoResourceTags) + .await + .unwrap(), + 0 + ); + assert_eq!(fake.runs(), (0, 0)); + } +} + +#[tokio::test] +async fn yes_applies_and_exits_0_and_a_second_preview_is_up_to_date() { + let fake = Fake::new(Some("0.0.3"), &["rebuild backup_items (2 rows)"]); + assert_eq!( + apply_migrations(&fake, &args(true), &NoResourceTags) + .await + .unwrap(), + 0 + ); + assert_eq!(fake.runs(), (1, 1)); + assert_eq!( + apply_migrations(&fake, &args(false), &NoResourceTags) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn a_newer_catalog_is_an_error_not_a_preview() { + let fake = Fake::new(Some("0.0.5"), &[]); + let err = apply_migrations(&fake, &args(false), &NoResourceTags) + .await + .expect_err("newer"); + assert!(err.to_string().contains("does not downgrade"), "{err}"); + assert_eq!(fake.runs(), (0, 0)); +} + +#[test] +fn errors_print_their_message_without_the_variant_name() { + let shown = format!( + "{:?}", + op_err(OpError::Internal("Cannot connect to PostgreSQL".to_owned())) + ); + assert_eq!(shown, "Cannot connect to PostgreSQL"); + let shown = format!( + "{:?}", + storage_err(extenddb_storage::error::StorageError::Internal( + "Cannot connect to PostgreSQL".to_owned() + )) + ); + assert_eq!(shown, "Cannot connect to PostgreSQL"); +} diff --git a/crates/app/src/cmd_migrate_runner.rs b/crates/app/src/cmd_migrate_runner.rs new file mode 100644 index 000000000..e2e80b943 --- /dev/null +++ b/crates/app/src/cmd_migrate_runner.rs @@ -0,0 +1,316 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `extenddb migrate` on a backend whose catalog migrates through the shared +//! runner. Without `--yes` it reads the catalog status, taking no lock and +//! writing nothing, so it runs beside a server. `--yes` reads the status +//! again under the migration lock and applies; it refuses a store that is +//! not a catalog, which `extenddb init` creates. `--reconverge` reruns every +//! convergent migration on a catalog a release before 1.0 wrote to, +//! `--forget` clears an unreleased migration, and `--list` prints the +//! migrations without opening the catalog. +//! +//! Exit status: 0 up to date, applied, or ahead by additive migrations only; +//! 1 an error or an uninitialized catalog; 3 migrations pending (the +//! preview); 4 the catalog has a breaking migration this binary does not +//! know; 5 a person must act; 6 the lock was not free within +//! `--lock-timeout`. + +use std::time::Duration; + +use extenddb_config as config; +use extenddb_storage::migrations::advice::{self, Advice, AdviceFor}; +use extenddb_storage::migrations::{ + AdminRole, CatalogMigrations, CatalogStatus, Listed, MigrationError, MigrationId, RunKind, + RunReport, Target, before_lock, exit, forget_locked, run_locked, +}; + +use crate::cmd_migrate::MigrateArgs; + +/// Run `extenddb migrate` on `migrations`; returns the exit status. +pub(crate) async fn run( + args: &MigrateArgs, + migrations: &dyn CatalogMigrations, + release: &str, +) -> anyhow::Result { + if args.list { + print_list(migrations); + return Ok(exit::OK); + } + if args.lock_timeout.is_some() && !args.yes && args.forget.is_none() { + anyhow::bail!( + "--lock-timeout applies to --yes and --forget, which take the migration lock; the \ + preview takes none" + ); + } + let (app_config, config_file) = crate::util::command_config(args.config.as_deref())?; + let location = app_config.storage.connection_config().to_owned(); + let shown = config::redact_password(&location); + let target = Target { + location, + data_store: args.data_db.clone(), + admin: admin_role( + args.pg_user.as_deref(), + args.pg_pass.as_deref(), + std::env::var_os("EXTENDDB_PG_PASSWORD"), + )?, + data_connection: None, + }; + // The backup clause names the data store the catalog records when the + // command names none. + let named = Target { + data_store: migrations.data_store_name(&target).await, + ..target.clone() + }; + let advice = AdviceFor::new( + release, + shown.clone(), + Some(migrations), + &named, + config_file.as_deref(), + ); + let advice = advice.advice(); + + println!("=== extenddb migrate ==="); + println!( + "Config: {}", + config_file + .as_deref() + .unwrap_or("none (built-in defaults and EXTENDDB__ variables)") + ); + println!( + "Catalog: {shown} ({})", + extenddb_storage::backend_name().unwrap_or("unknown") + ); + println!(); + + let lock_timeout = Duration::from_secs( + args.lock_timeout + .unwrap_or(crate::cmd_migrate::DEFAULT_LOCK_TIMEOUT_SECS), + ); + let outcome = if let Some(id) = args.forget { + forget(migrations, &target, id, lock_timeout).await + } else if args.yes { + let kind = if args.reconverge { + RunKind::Reconverge + } else { + RunKind::Migrate + }; + apply(migrations, &target, release, kind, lock_timeout).await + } else { + preview(migrations, &target, &advice).await + }; + Ok(outcome.unwrap_or_else(|error| { + eprintln!("Error: {}", advice::error_message(&error, &advice)); + error.exit_code() + })) +} + +/// The admin role `--pg-user` names, with `--pg-pass` or else the +/// `EXTENDDB_PG_PASSWORD` variable; `None` without `--pg-user`. +fn admin_role( + user: Option<&str>, + password: Option<&str>, + password_env: Option, +) -> anyhow::Result> { + let Some(user) = user else { + return Ok(None); + }; + let password = match (password, password_env) { + (Some(password), _) => Some(password.to_owned()), + (None, Some(env)) => Some( + env.into_string() + .map_err(|_| anyhow::anyhow!("EXTENDDB_PG_PASSWORD must contain valid UTF-8"))?, + ), + (None, None) => None, + }; + Ok(Some(AdminRole { + user: user.to_owned(), + password, + })) +} + +/// Every migration this binary carries, with its kind on this backend. +fn print_list(migrations: &dyn CatalogMigrations) { + let backend = extenddb_storage::backend_name().unwrap_or("unknown"); + println!("Migrations this binary carries, with their kind on the {backend} backend:"); + let width = MigrationId::ALL + .iter() + .map(|id| id.name().len()) + .max() + .unwrap_or_default(); + for id in MigrationId::ALL { + let entry = match migrations.listed(id) { + Listed::Applies { + kind, + convergent, + stores, + } => { + let stores: Vec<&str> = stores.iter().map(|s| s.as_str()).collect(); + format!( + "{:<9} {:<14} {}", + kind.as_str(), + if convergent { + "convergent" + } else { + "not convergent" + }, + stores.join(", ") + ) + } + Listed::NotApplicable(reason) => format!("not applicable: {reason}"), + }; + println!( + " {:>2} {:, +) -> Result { + let status = migrations.status(target).await?; + for note in migrations.status_notes(target).await? { + println!("Note: {}", note.text); + } + let latest = MigrationId::LATEST.number(); + match &status { + CatalogStatus::Current { version } => { + println!( + "Up to date: the catalog is at migration {version}, the latest this binary \ + carries. No migrations needed." + ); + } + CatalogStatus::AheadCompatible { .. } => { + println!("WARNING: {status}. Nothing to apply."); + } + CatalogStatus::Behind { version, pending } => { + if version.is_none() { + println!( + "Stop every server that uses this catalog before applying: a server of a \ + release before 1.0 is not detected while it runs." + ); + } + match version { + Some(v) => { + println!("The catalog is at migration {v}; this binary carries {latest}.") + } + None => println!( + "The catalog has no migration ledger (a release before 1.0 created it); this \ + binary carries migrations 1 to {latest}." + ), + } + println!("Pending:"); + for p in pending { + let description = + MigrationId::from_number(p.id).map_or("", MigrationId::description); + println!( + " {} {} ({}, {}): {description}", + p.id, + p.name, + p.store.as_str(), + p.kind.as_str() + ); + } + match migrations.pending_changes(target, &status).await { + Ok(lines) => { + for line in lines { + println!("{line}"); + } + } + Err(e) => println!("WARNING: {}", advice::error_message(&e, advice)), + } + println!(); + println!( + "Nothing was changed; {}, then run again with --yes to apply (exit status {}: \ + migrations pending).", + advice.backup, + exit::PENDING + ); + } + _ => { + eprintln!("Error: {}", advice::status_message(&status, advice)); + } + } + Ok(status.exit_code()) +} + +/// Take the lock, run, and print what the run did. +async fn apply( + migrations: &dyn CatalogMigrations, + target: &Target, + release: &str, + kind: RunKind, + lock_timeout: Duration, +) -> Result { + for note in before_lock(migrations, target, kind).await? { + println!("Note: {}", note.text); + } + println!( + "--- Taking the migration lock (waiting up to {} s for a server or another migration)...", + lock_timeout.as_secs() + ); + let lock = migrations.lock(target, lock_timeout).await?; + let report = run_locked(migrations, target, &lock, release, kind).await?; + drop(lock); + print_report(&report, kind); + if report.applied.is_empty() && kind == RunKind::Migrate { + // A catalog ahead by additive migrations has nothing to apply and + // serves; the status says which. + if let Ok(status @ CatalogStatus::AheadCompatible { .. }) = migrations.status(target).await + { + println!("WARNING: {status}."); + } + } + Ok(exit::OK) +} + +fn print_report(report: &RunReport, kind: RunKind) { + for note in &report.notes { + println!(" {note}"); + } + let verb = match kind { + RunKind::Create | RunKind::Migrate => "Applied", + RunKind::Reconverge => "Ran again", + }; + crate::util::print_applied(report, " ", verb); + let show = + |v: Option| v.map_or_else(|| "no ledger".to_owned(), |v| format!("migration {v}")); + println!(); + if report.applied.is_empty() { + println!( + "Up to date: the catalog is at {}. No migrations needed.", + show(report.to) + ); + } else { + println!("=== extenddb migrate complete ==="); + println!("Catalog: {} -> {}", show(report.from), show(report.to)); + } +} + +/// Clear an unreleased migration's rows under the lock. A store that is not +/// a catalog is refused before anything opens it to write. +async fn forget( + migrations: &dyn CatalogMigrations, + target: &Target, + id: u32, + lock_timeout: Duration, +) -> Result { + let lock = migrations.lock(target, lock_timeout).await?; + forget_locked(migrations, target, &lock, id).await?; + println!( + "Forgot migration {id}: its ledger rows are deleted, and the next \ + `extenddb migrate --yes` runs it again." + ); + Ok(exit::OK) +} + +#[cfg(test)] +#[path = "cmd_migrate_runner_tests.rs"] +mod tests; diff --git a/crates/app/src/cmd_migrate_runner_tests.rs b/crates/app/src/cmd_migrate_runner_tests.rs new file mode 100644 index 000000000..ba21ed51c --- /dev/null +++ b/crates/app/src/cmd_migrate_runner_tests.rs @@ -0,0 +1,396 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The preview takes no lock and runs nothing; `--yes` and `--forget` +//! take the lock first; each ends with the status's or the error's code. +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::advice::Advice; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, HeldLock, Kind, LedgerRow, Listed, MigrationError, + MigrationId, Pending, RunKind, RunReport, StatusNote, StoreKind, Target, exit, +}; + +use super::{apply, forget, preview}; + +struct Fake { + status: CatalogStatus, + lock: Result<(), MigrationError>, + notes: Vec, + calls: Mutex>, +} + +impl Fake { + fn new(status: CatalogStatus) -> Self { + Self { + status, + lock: Ok(()), + notes: Vec::new(), + calls: Mutex::new(Vec::new()), + } + } + fn calls(&self) -> Vec<&'static str> { + self.calls.lock().unwrap().clone() + } + fn call(&self, name: &'static str) { + self.calls.lock().unwrap().push(name); + } +} + +#[async_trait] +impl CatalogMigrations for Fake { + fn listed(&self, _id: MigrationId) -> Listed { + Listed::NotApplicable("fake") + } + fn backup_advice(&self, target: &Target) -> String { + format!("back up {}", target.location) + } + async fn status(&self, _target: &Target) -> Result { + self.call("status"); + Ok(self.status.clone()) + } + async fn catalog_ledger(&self, _: &Target) -> Result, MigrationError> { + Ok(Vec::new()) + } + async fn lock(&self, target: &Target, _: Duration) -> Result { + self.call("lock"); + self.lock + .clone() + .map(|()| HeldLock::new(target.location.as_str(), ())) + } + async fn run( + &self, + _lock: &HeldLock, + _release: &str, + kind: RunKind, + ) -> Result { + self.call(match kind { + RunKind::Create => "create", + RunKind::Migrate => "run", + RunKind::Reconverge => "reconverge", + }); + Ok(RunReport::default()) + } + async fn forget(&self, _lock: &HeldLock, _id: u32) -> Result<(), MigrationError> { + self.call("forget"); + Ok(()) + } + async fn status_notes(&self, _: &Target) -> Result, MigrationError> { + Ok(self.notes.clone()) + } + async fn pending_changes( + &self, + _: &Target, + status: &CatalogStatus, + ) -> Result, MigrationError> { + self.call("pending_changes"); + assert!( + matches!(status, CatalogStatus::Behind { .. }), + "asked for the changes of {status:?}" + ); + Ok(vec!["Migration 2 rewrites no stored policy.".to_owned()]) + } +} + +const ADVICE: Advice<'static> = Advice { + release: "1.0.0-test", + location: "db", + backup: "back up db", + config: Some("extenddb.toml"), + assignment: None, +}; + +fn behind() -> CatalogStatus { + CatalogStatus::Behind { + version: None, + pending: MigrationId::ALL + .iter() + .map(|id| Pending { + id: id.number(), + name: id.name(), + store: StoreKind::Catalog, + store_name: "db".to_owned(), + kind: Kind::Breaking, + }) + .collect(), + } +} + +#[tokio::test] +async fn the_preview_reads_the_status_and_takes_no_lock() { + let cases = [ + (behind(), exit::PENDING), + ( + CatalogStatus::Current { + version: MigrationId::LATEST.number(), + }, + exit::OK, + ), + ( + CatalogStatus::NeedsAttention { + items: vec!["collision".to_owned()], + }, + exit::ATTENTION, + ), + (CatalogStatus::Uninitialized, exit::ERROR), + ( + CatalogStatus::AheadIncompatible { + unknown: vec![(MigrationId::LATEST.number() + 1, "later".to_owned())], + }, + exit::AHEAD_INCOMPATIBLE, + ), + ]; + for (status, code) in cases { + let fake = Fake::new(status.clone()); + assert_eq!( + preview(&fake, &Target::at("db"), &ADVICE).await, + Ok(code), + "{status:?}" + ); + // A catalog behind the binary also gets the list of what its pending + // migrations change (the policies migration 2 rewrites). + let calls: &[&str] = if code == exit::PENDING { + &["status", "pending_changes"] + } else { + &["status"] + }; + assert_eq!(fake.calls(), calls, "{status:?}"); + } +} + +#[tokio::test] +async fn yes_takes_the_lock_before_it_runs() { + let fake = Fake::new(behind()); + let code = apply( + &fake, + &Target::at("db"), + "1.0.0-test", + RunKind::Migrate, + Duration::ZERO, + ) + .await; + assert_eq!(code, Ok(exit::OK)); + // The status is read under the lock before the run; a run that + // applied nothing reads it again, to warn about a catalog ahead by + // additive migrations. + assert_eq!(fake.calls(), ["lock", "status", "run", "status"]); + + let fake = Fake::new(behind()); + let code = apply( + &fake, + &Target::at("db"), + "1.0.0-test", + RunKind::Reconverge, + Duration::ZERO, + ) + .await; + assert_eq!(code, Ok(exit::OK)); + assert_eq!(fake.calls(), ["lock", "status", "reconverge"]); +} + +/// A store that is not a catalog is refused under the lock, with the +/// status's exit code 1 and the message naming `extenddb init`, and the +/// backend runs nothing. +#[tokio::test] +async fn yes_refuses_an_uninitialized_store() { + for kind in [RunKind::Migrate, RunKind::Reconverge] { + let fake = Fake::new(CatalogStatus::Uninitialized); + let err = apply(&fake, &Target::at("db"), "1.0.0-test", kind, Duration::ZERO) + .await + .expect_err("uninitialized"); + assert_eq!(err.exit_code(), exit::ERROR); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + assert!( + super::advice::error_message(&err, &ADVICE).contains("extenddb init"), + "{err}" + ); + assert_eq!(fake.calls(), ["lock", "status"], "{kind:?}"); + } +} + +/// `--forget` on a store that is not a catalog reads the status under the +/// lock and refuses, naming `extenddb init`, without calling the backend's +/// forget, which would open the store to write. +#[tokio::test] +async fn forget_refuses_an_uninitialized_store() { + let fake = Fake::new(CatalogStatus::Uninitialized); + let err = forget(&fake, &Target::at("db"), 4, Duration::ZERO) + .await + .expect_err("uninitialized"); + assert_eq!(err.exit_code(), exit::ERROR); + assert!( + super::advice::error_message(&err, &ADVICE).contains("extenddb init"), + "{err}" + ); + assert_eq!(fake.calls(), ["lock", "status"]); +} + +/// A note that refuses runs stops `--yes` before the lock on a catalog +/// with something to apply, and every `--reconverge`. On a current +/// catalog `--yes` applies nothing and goes on, as it does past a note +/// that does not refuse; the preview prints the note and keeps the +/// status's code. +#[tokio::test] +async fn a_refusing_note_stops_yes_before_the_lock() { + let current = CatalogStatus::Current { + version: MigrationId::LATEST.number(), + }; + let note = StatusNote { + text: "the role does not own the tables".to_owned(), + refuses_runs: true, + }; + let with = |status: CatalogStatus, note: &StatusNote| { + let mut fake = Fake::new(status); + fake.notes = vec![note.clone()]; + fake + }; + let target = Target::at("db"); + for (status, kind) in [ + (behind(), RunKind::Migrate), + (current.clone(), RunKind::Reconverge), + ] { + let fake = with(status, ¬e); + let err = apply(&fake, &target, "1.0.0-test", kind, Duration::ZERO) + .await + .expect_err("refused"); + assert_eq!(err.exit_code(), exit::ERROR, "{kind:?}"); + assert!(err.to_string().contains("does not own the tables"), "{err}"); + assert!( + !fake.calls().contains(&"lock"), + "{kind:?}: {:?}", + fake.calls() + ); + } + // Each refusing note is a line of its own, as the preview prints it, so + // the statements a note ends with can be copied alone. + let mut fake = with(behind(), ¬e); + fake.notes.push(StatusNote { + text: "a second note, ending with GRANT x TO y;".to_owned(), + refuses_runs: true, + }); + let err = apply( + &fake, + &target, + "1.0.0-test", + RunKind::Migrate, + Duration::ZERO, + ) + .await + .expect_err("refused"); + assert_eq!( + err.to_string().lines().collect::>(), + [ + "migrate --yes refuses to apply migrations while these notes hold:", + "Note: the role does not own the tables", + "Note: a second note, ending with GRANT x TO y;", + "Nothing was changed and no lock was taken.", + ] + ); + let fake = with(current, ¬e); + let code = apply( + &fake, + &target, + "1.0.0-test", + RunKind::Migrate, + Duration::ZERO, + ) + .await; + assert_eq!(code, Ok(exit::OK)); + assert!(fake.calls().contains(&"lock"), "{:?}", fake.calls()); + let informational = StatusNote { + refuses_runs: false, + ..note.clone() + }; + let fake = with(behind(), &informational); + let code = apply( + &fake, + &target, + "1.0.0-test", + RunKind::Migrate, + Duration::ZERO, + ) + .await; + assert_eq!(code, Ok(exit::OK)); + assert!(fake.calls().contains(&"run"), "{:?}", fake.calls()); + let fake = with(behind(), ¬e); + assert_eq!(preview(&fake, &target, &ADVICE).await, Ok(exit::PENDING)); + assert_eq!(fake.calls(), ["status", "pending_changes"]); +} + +#[tokio::test] +async fn a_held_lock_runs_nothing_and_exits_6() { + let mut fake = Fake::new(behind()); + fake.lock = Err(MigrationError::Locked("held".to_owned())); + let err = apply( + &fake, + &Target::at("db"), + "1.0.0-test", + RunKind::Migrate, + Duration::ZERO, + ) + .await + .expect_err("locked"); + assert_eq!(err.exit_code(), exit::LOCKED); + assert_eq!(fake.calls(), ["lock"]); + let err = forget( + &fake, + &Target::at("db"), + MigrationId::LATEST.number(), + Duration::ZERO, + ) + .await + .expect_err("locked"); + assert_eq!(err.exit_code(), exit::LOCKED); + assert_eq!(fake.calls(), ["lock", "lock"]); +} + +#[tokio::test] +async fn forget_takes_the_lock() { + let fake = Fake::new(behind()); + let code = forget( + &fake, + &Target::at("db"), + MigrationId::LATEST.number(), + Duration::ZERO, + ) + .await; + assert_eq!(code, Ok(exit::OK)); + assert_eq!(fake.calls(), ["lock", "status", "forget"]); +} + +fn parse(argv: &[&str]) -> Result { + use clap::{Args as _, FromArgMatches as _}; + let cmd = crate::cmd_migrate::MigrateArgs::augment_args(clap::Command::new("migrate")); + let matches = + cmd.try_get_matches_from(std::iter::once("migrate").chain(argv.iter().copied()))?; + crate::cmd_migrate::MigrateArgs::from_arg_matches(&matches) +} + +/// A lock timeout above a day is refused before anything runs, so the +/// wait never overflows `Instant`. +#[test] +fn a_lock_timeout_above_a_day_is_refused() { + assert!(parse(&["--yes", "--lock-timeout", "18446744073709551615"]).is_err()); + assert!(parse(&["--yes", "--lock-timeout", "86401"]).is_err()); + let args = parse(&["--yes", "--lock-timeout", "86400"]).expect("a day"); + assert_eq!(args.lock_timeout, Some(86_400)); +} + +/// The preview takes no lock, so a lock timeout given to it is refused +/// instead of ignored. +#[tokio::test] +async fn the_preview_refuses_a_lock_timeout() { + let args = parse(&["--lock-timeout", "5"]).expect("parse"); + let fake = Fake::new(behind()); + let err = super::run(&args, &fake, "1.0.0-test") + .await + .expect_err("refused"); + assert!( + err.to_string() + .contains("--lock-timeout applies to --yes and --forget"), + "{err}" + ); + assert!(fake.calls().is_empty()); +} diff --git a/crates/app/src/cmd_serve.rs b/crates/app/src/cmd_serve.rs index 37e5167ac..a754aa4c2 100755 --- a/crates/app/src/cmd_serve.rs +++ b/crates/app/src/cmd_serve.rs @@ -94,9 +94,11 @@ pub fn run(args: &ServeArgs, build: BuildInfo) -> anyhow::Result<()> { ); } - // Validate backend is supported and get catalog version (fail fast before binding port) + // Fail fast, before binding the port, when no backend is installed. let backend = &app_config.storage.backend; - let catalog_version = extenddb_storage::operations::catalog_version()?; + extenddb_storage::backend_name() + .ok_or_else(|| anyhow::anyhow!("no storage backend installed"))?; + let catalog_version = extenddb_storage::migrations::binary_catalog_label(); let port = args.port.unwrap_or(app_config.server.port); @@ -263,18 +265,36 @@ pub fn run(args: &ServeArgs, build: BuildInfo) -> anyhow::Result<()> { } } - tokio::runtime::Builder::new_multi_thread() + let mut params = ServeParams::new(app_config, std_listener, pid_file, build) + .with_log_target(if args.foreground { + LogTarget::Stderr + } else { + LogTarget::Syslog + }) + .with_dev_mode(cfg!(feature = "dev-mode")); + if config_exists { + params = params.with_config_path(&args.config); + } + let result = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()? - .block_on(extenddb_server::serve( - ServeParams::new(app_config, std_listener, pid_file, build) - .with_log_target(if args.foreground { - LogTarget::Stderr - } else { - LogTarget::Syslog - }) - .with_dev_mode(cfg!(feature = "dev-mode")), - )) + .block_on(extenddb_server::serve(params)); + // `serve` has printed the refusal; end with its code, which `migrate` + // and `verify` use for the same status. + if let Err(e) = &result + && let Some(refusal) = e.downcast_ref::() + { + std::process::exit(i32::from(refusal.exit_code)); + } + // `serve` stopped through its shutdown path, which removed the PID file, + // and has logged why. + if let Err(e) = &result + && e.downcast_ref::() + .is_some() + { + std::process::exit(1); + } + result } /// The five wordmark rows (figlet slant), without trailing newlines. diff --git a/crates/app/src/cmd_settings.rs b/crates/app/src/cmd_settings.rs index 76c35070a..42d102289 100755 --- a/crates/app/src/cmd_settings.rs +++ b/crates/app/src/cmd_settings.rs @@ -9,7 +9,10 @@ //! strings carrying a password) are masked the same way the management API //! masks them: `extenddb_config::should_redact` picks the keys and //! `extenddb_config::MASK` is the text printed; `get --show-secret` prints the -//! stored value. +//! stored value. A key the catalog has no row for prints the storage +//! engine's default (`extenddb_storage::operations::setting_default`), as the +//! console does, so a fresh install, which seeds no settings rows, lists the +//! values the engine uses. use clap::{Args, Subcommand}; @@ -88,11 +91,64 @@ fn format_list(rows: &[(String, String)]) -> String { .collect() } +/// The other name of a renamed key (its deprecated alias, or the current +/// name of an alias), whose row the engine reads when the key has none. +fn other_name(key: &str) -> Option<&'static str> { + use extenddb_core::settings_keys::{ + INDEX_PROPAGATION_DELAY_MS, LEGACY_GSI_PROPAGATION_DELAY_MS, + }; + match key { + INDEX_PROPAGATION_DELAY_MS => Some(LEGACY_GSI_PROPAGATION_DELAY_MS), + LEGACY_GSI_PROPAGATION_DELAY_MS => Some(INDEX_PROPAGATION_DELAY_MS), + _ => None, + } +} + +/// `rows` with a row for each writable key that has no row under either +/// of its names and that `default` gives a value for, ordered by key. +fn with_defaults( + mut rows: Vec<(String, String)>, + default: impl Fn(&str) -> Option, +) -> Vec<(String, String)> { + for (key, _) in KNOWN_KEYS { + let names = [Some(*key), other_name(key)]; + if rows.iter().all(|(k, _)| !names.contains(&Some(k.as_str()))) + && let Some(value) = default(key) + { + rows.push(((*key).to_owned(), value)); + } + } + rows.sort(); + rows +} + +/// The value `get` prints for `key`, under either name of a renamed key: +/// the row the engine reads, which is the row of the current name, else +/// the row of the deprecated alias, else `default`'s value. +async fn stored_or_default( + store: &dyn SettingsStore, + key: &str, + default: impl Fn(&str) -> Option, +) -> anyhow::Result> { + let current = extenddb_core::settings_keys::canonical_key(key); + for name in [Some(current), other_name(current)].into_iter().flatten() { + let stored = store + .get_setting(name) + .await + .map_err(|e| anyhow::anyhow!("Failed to get setting: {e:?}"))?; + if stored.is_some() { + return Ok(stored); + } + } + Ok(default(current)) +} + async fn list(store: &dyn SettingsStore) -> anyhow::Result<()> { let rows = store .list_settings() .await .map_err(|e| anyhow::anyhow!("Failed to list settings: {e:?}"))?; + let rows = with_defaults(rows, extenddb_storage::operations::setting_default); if rows.is_empty() { println!("No settings found."); @@ -103,10 +159,8 @@ async fn list(store: &dyn SettingsStore) -> anyhow::Result<()> { } async fn get(store: &dyn SettingsStore, key: &str, show_secret: bool) -> anyhow::Result<()> { - let value = store - .get_setting(key) - .await - .map_err(|e| anyhow::anyhow!("Failed to get setting: {e:?}"))?; + let value = + stored_or_default(store, key, extenddb_storage::operations::setting_default).await?; if let Some(v) = value { println!("{}", display_value(key, &v, show_secret)); @@ -223,6 +277,129 @@ mod tests { assert_eq!(extenddb_config::MASK, "\u{2022}".repeat(8)); } + /// The storage engine's defaults on SQLite and PostgreSQL, which a fresh + /// install no longer seeds as rows. + fn engine_default(key: &str) -> Option { + match key { + "control_plane_delay_seconds" => Some("0.25".to_owned()), + "index_propagation_delay_ms" => Some("10".to_owned()), + _ => None, + } + } + + /// A settings store's answer, as its trait returns it. + type Answer<'a, T> = std::pin::Pin< + Box< + dyn std::future::Future> + + Send + + 'a, + >, + >; + + /// A settings store holding `rows`. + struct Rows(Vec<(String, String)>); + + impl SettingsStore for Rows { + fn get_setting(&self, key: &str) -> Answer<'_, Option> { + let value = self + .0 + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.clone()); + Box::pin(async move { Ok(value) }) + } + fn set_setting(&self, _key: &str, _value: &str) -> Answer<'_, ()> { + Box::pin(async { Ok(()) }) + } + fn list_settings(&self) -> Answer<'_, Vec<(String, String)>> { + let rows = self.0.clone(); + Box::pin(async move { Ok(rows) }) + } + } + + fn rows(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect() + } + + /// A key a fresh install has no row for prints the engine's default, + /// as on a catalog that seeded it, and a stored row wins over it. + #[tokio::test] + async fn get_falls_back_to_the_engine_default() { + let fresh = Rows(rows(&[("catalog_version", "4")])); + let value = stored_or_default(&fresh, "control_plane_delay_seconds", engine_default) + .await + .expect("get"); + assert_eq!(value.as_deref(), Some("0.25")); + let value = stored_or_default(&fresh, "no_such_key", engine_default) + .await + .expect("get"); + assert_eq!(value, None); + let set = Rows(rows(&[("control_plane_delay_seconds", "2")])); + let value = stored_or_default(&set, "control_plane_delay_seconds", engine_default) + .await + .expect("get"); + assert_eq!(value.as_deref(), Some("2")); + // A row under the deprecated name is the one the engine reads. + let legacy = Rows(rows(&[("gsi_propagation_delay_ms", "0")])); + for key in ["index_propagation_delay_ms", "gsi_propagation_delay_ms"] { + let value = stored_or_default(&legacy, key, engine_default) + .await + .expect("get"); + assert_eq!(value.as_deref(), Some("0"), "{key}"); + } + let value = stored_or_default(&fresh, "gsi_propagation_delay_ms", engine_default) + .await + .expect("get"); + assert_eq!(value.as_deref(), Some("10")); + // With rows under both names the engine reads the current name's, + // and either name prints it. + let both = Rows(rows(&[ + ("gsi_propagation_delay_ms", "999"), + ("index_propagation_delay_ms", "7"), + ])); + for key in ["index_propagation_delay_ms", "gsi_propagation_delay_ms"] { + let value = stored_or_default(&both, key, engine_default) + .await + .expect("get"); + assert_eq!(value.as_deref(), Some("7"), "{key}"); + } + } + + /// `list` adds a row for each writable key with an engine default and + /// no row, in key order, and keeps a stored value over the default. + #[test] + fn list_adds_the_engine_defaults_in_key_order() { + let listed = with_defaults( + rows(&[("catalog_version", "4"), ("log_level", "debug")]), + engine_default, + ); + assert_eq!( + format_list(&listed), + "catalog_version = 4\ncontrol_plane_delay_seconds = 0.25\n\ + index_propagation_delay_ms = 10\nlog_level = debug\n" + ); + let listed = with_defaults(rows(&[("index_propagation_delay_ms", "0")]), engine_default); + assert_eq!( + listed, + rows(&[ + ("control_plane_delay_seconds", "0.25"), + ("index_propagation_delay_ms", "0"), + ]) + ); + // No default beside a row under the deprecated name. + let listed = with_defaults(rows(&[("gsi_propagation_delay_ms", "0")]), engine_default); + assert_eq!( + listed, + rows(&[ + ("control_plane_delay_seconds", "0.25"), + ("gsi_propagation_delay_ms", "0"), + ]) + ); + } + #[test] fn no_known_key_matches_a_redaction_pattern() { // `set` prints the value it wrote, and both `set` and the management diff --git a/crates/app/src/cmd_verify.rs b/crates/app/src/cmd_verify.rs index 8e35a397f..4021d73dd 100755 --- a/crates/app/src/cmd_verify.rs +++ b/crates/app/src/cmd_verify.rs @@ -6,6 +6,12 @@ //! Connects to catalog, checks version, enumerates tables and indexes, //! connects to data database, reports healthy/unhealthy. //! +//! On a backend on the migration runner the version check is the catalog +//! status, read without a lock or a write. `verify` exits with the status's +//! code 4 (ahead by a breaking migration) or 5 (a person must act) before +//! the 1 of a failed check, and with that 1 before the 3 of pending +//! migrations, which `migrate --yes` cannot apply while a check fails. +//! //! This command uses raw `sqlx` queries intentionally. It is a diagnostic //! tool that runs outside the server process and needs direct database //! access to verify infrastructure health. Routing through the storage @@ -17,20 +23,23 @@ use extenddb_config as config; #[derive(Args)] pub struct VerifyArgs { - /// Path to configuration file - #[arg(short, long, default_value = "extenddb.toml")] - config: String, + /// Path to configuration file (default: extenddb.toml; a dev-mode build + /// without one reads the EXTENDDB__ environment variables, as serve does) + #[arg(short, long)] + config: Option, + + /// Print every note the migration ledger records. Without it, verify + /// prints the number of notes of each migration and its first 5 + #[arg(long)] + all_notes: bool, } -pub async fn run(args: VerifyArgs) -> anyhow::Result<()> { - if !std::path::Path::new(&args.config).exists() { - anyhow::bail!( - "Config file '{}' not found. Run 'extenddb init' to set up a deployment, \ - or use --config to specify a different location.", - args.config, - ); - } - let app_config = config::load(&args.config)?; +/// How many notes of each migration `verify` prints without `--all-notes`. +const NOTES_SHOWN: usize = 5; + +/// Run `extenddb verify`; returns the process exit status. +pub async fn run(args: VerifyArgs, release: &str) -> anyhow::Result { + let (app_config, config_file) = crate::util::command_config(args.config.as_deref())?; let expected_version = extenddb_storage::operations::catalog_version().unwrap_or_else(|_| "unknown".to_string()); @@ -43,10 +52,37 @@ pub async fn run(args: VerifyArgs) -> anyhow::Result<()> { let mut errors = 0u32; println!("=== extenddb verify ==="); - println!("Config: {}", args.config); + println!( + "Config: {}", + config_file + .as_deref() + .unwrap_or("none (built-in defaults and EXTENDDB__ variables)") + ); println!("Catalog database: {}", parts.database); println!(); + // Read first, before anything opens the catalog: a store that does not + // exist is reported, not created. + let status_code = match extenddb_storage::migrations::installed() { + Some(migrations) => { + let code = check_migrations( + migrations, + config_file.as_deref(), + &app_config, + release, + args.all_notes, + ) + .await; + if code == extenddb_storage::migrations::exit::ERROR { + println!(); + println!("=== UNHEALTHY: see the catalog check above ==="); + return Ok(code); + } + Some(code) + } + None => None, + }; + // Create settings and diagnostics store println!("--- Checking catalog connection..."); let store = match extenddb_storage::settings_store::create_settings_store( @@ -64,24 +100,27 @@ pub async fn run(args: VerifyArgs) -> anyhow::Result<()> { } }; - // Check 2: Catalog version (D-10: strict parsing). - println!("--- Checking catalog version..."); - match store.get_setting("catalog_version").await { - Ok(Some(v)) => { - if v == expected_version { - println!(" OK: Catalog version {v}"); - } else { - println!(" WARN: Catalog version {v} (binary expects {expected_version})"); + // Check 2: Catalog version (D-10: strict parsing), on a backend that + // still records one; the runner's status replaced it above. + if status_code.is_none() { + println!("--- Checking catalog version..."); + match store.get_setting("catalog_version").await { + Ok(Some(v)) => { + if v == expected_version { + println!(" OK: Catalog version {v}"); + } else { + println!(" WARN: Catalog version {v} (binary expects {expected_version})"); + errors += 1; + } + } + Ok(None) => { + println!(" FAIL: No catalog version found. Run 'extenddb init'."); + errors += 1; + } + Err(e) => { + println!(" FAIL: Failed to read catalog version: {e:?}"); errors += 1; } - } - Ok(None) => { - println!(" FAIL: No catalog version found. Run 'extenddb init'."); - errors += 1; - } - Err(e) => { - println!(" FAIL: Failed to read catalog version: {e:?}"); - errors += 1; } } @@ -113,12 +152,189 @@ pub async fn run(args: VerifyArgs) -> anyhow::Result<()> { println!(" Indexes: {index_count}"); println!(); - if errors == 0 { + let code = exit_code(status_code.unwrap_or(0), errors); + if code == 0 { println!("=== HEALTHY: All checks passed ==="); - } else { + } else if errors > 0 { println!("=== UNHEALTHY: {errors} check(s) failed ==="); - std::process::exit(1); + } else { + println!("=== UNHEALTHY: the catalog needs the action above ==="); + } + Ok(code) +} + +/// The exit status of a `verify` whose catalog status exits `status_code` +/// and in which `failed` checks failed: the status's 4 or 5, which ask a +/// person to act whatever the checks found, else 1 when a check failed, +/// else the status's code. A failed check comes before the 3 of pending +/// migrations because `migrate --yes` cannot apply them while the check +/// fails, for example while the data database cannot be reached. +fn exit_code(status_code: u8, failed: u32) -> u8 { + use extenddb_storage::migrations::exit; + match status_code { + exit::AHEAD_INCOMPATIBLE | exit::ATTENTION => status_code, + _ if failed > 0 => exit::ERROR, + _ => status_code, } +} + +/// The lines `verify` prints for the notes the catalog ledger records: per +/// migration with notes, how many it has and the first [`NOTES_SHOWN`], or +/// every note with `all`. +fn ledger_note_lines(rows: &[extenddb_storage::migrations::LedgerRow], all: bool) -> Vec { + let mut lines = Vec::new(); + for row in rows.iter().filter(|r| !r.notes.is_empty()) { + let notes: Vec<&str> = row.notes.lines().collect(); + let count = notes.len(); + let shown = if all { count } else { count.min(NOTES_SHOWN) }; + lines.push(format!( + " Notes of migration {} {} ({count} note{}):", + row.id, + row.name, + if count == 1 { "" } else { "s" } + )); + lines.extend(notes.iter().take(shown).map(|note| format!(" {note}"))); + if shown < count { + lines.push(format!( + " and {} more; verify --all-notes prints every note", + count - shown + )); + } + } + lines +} + +/// Print the catalog status, what its pending migrations change, and the +/// notes the ledger records (each migration's first [`NOTES_SHOWN`], or +/// every note with `all_notes`), and return the status's exit code. +async fn check_migrations( + migrations: &dyn extenddb_storage::migrations::CatalogMigrations, + config_file: Option<&str>, + app_config: &config::AppConfig, + release: &str, + all_notes: bool, +) -> u8 { + use extenddb_storage::migrations::advice::{self, AdviceFor}; + use extenddb_storage::migrations::{CatalogStatus, Target}; - Ok(()) + println!("--- Checking catalog migrations..."); + let location = app_config.storage.connection_config(); + let target = Target::at(location); + // The backup clause names the data store the catalog records. + let named = Target { + data_store: migrations.data_store_name(&target).await, + ..target.clone() + }; + let advice = AdviceFor::new( + release, + config::redact_password(location), + Some(migrations), + &named, + config_file, + ); + let advice = advice.advice(); + let status = match migrations.status(&target).await { + Ok(status) => status, + Err(e) => { + println!(" FAIL: {}", advice::error_message(&e, &advice)); + return e.exit_code(); + } + }; + match &status { + CatalogStatus::Current { .. } => println!(" OK: Current: {status}"), + CatalogStatus::AheadCompatible { .. } => println!(" WARN: {status}"), + _ => { + for (i, line) in advice::status_message(&status, &advice).lines().enumerate() { + println!(" {}{line}", if i == 0 { "FAIL: " } else { "" }); + } + } + } + if matches!(status, CatalogStatus::Behind { .. }) { + match migrations.pending_changes(&target, &status).await { + Ok(lines) => { + for line in lines { + println!(" {line}"); + } + } + Err(e) => println!(" WARN: {}", advice::error_message(&e, &advice)), + } + } + match migrations.status_notes(&target).await { + Ok(notes) => { + for note in notes { + println!(" WARN: {}", note.text); + } + } + Err(e) => println!(" WARN: {}", advice::error_message(&e, &advice)), + } + if let Ok(rows) = migrations.catalog_ledger(&target).await { + for line in ledger_note_lines(&rows, all_notes) { + println!("{line}"); + } + } + status.exit_code() +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{Kind, LedgerRow, Outcome, exit}; + + use super::{exit_code, ledger_note_lines}; + + fn row(id: u32, notes: usize) -> LedgerRow { + LedgerRow { + id, + name: format!("m{id}"), + kind: Kind::Breaking, + checksum: "c".to_owned(), + outcome: Outcome::Applied, + notes: (0..notes) + .map(|i| format!("note {i}")) + .collect::>() + .join("\n"), + applied_at: String::new(), + applied_by: "1.0.0".to_owned(), + } + } + + /// Each migration's note count and first 5 notes; `--all-notes` prints + /// every one. + #[test] + fn the_ledger_notes_are_counted_and_the_first_five_printed() { + let rows = [row(1, 0), row(2, 7), row(4, 1)]; + assert_eq!( + ledger_note_lines(&rows, false), + [ + " Notes of migration 2 m2 (7 notes):", + " note 0", + " note 1", + " note 2", + " note 3", + " note 4", + " and 2 more; verify --all-notes prints every note", + " Notes of migration 4 m4 (1 note):", + " note 0", + ] + ); + let all = ledger_note_lines(&rows, true); + assert_eq!(all.len(), 1 + 7 + 1 + 1, "{all:?}"); + assert_eq!(all[7], " note 6"); + assert!(all.iter().all(|l| !l.contains("more")), "{all:?}"); + let five = ledger_note_lines(&[row(3, 5)], false); + assert_eq!(five.len(), 6, "{five:?}"); + } + + /// A failed check, such as an unreachable data database, exits 1 before + /// the 3 of pending migrations; 4 and 5 still ask a person to act. + #[test] + fn a_failed_check_exits_one_before_a_pending_status() { + assert_eq!(exit_code(exit::PENDING, 1), exit::ERROR); + assert_eq!(exit_code(exit::PENDING, 0), exit::PENDING); + assert_eq!(exit_code(exit::OK, 1), exit::ERROR); + assert_eq!(exit_code(exit::OK, 0), exit::OK); + for code in [exit::AHEAD_INCOMPATIBLE, exit::ATTENTION] { + assert_eq!(exit_code(code, 1), code); + assert_eq!(exit_code(code, 0), code); + } + } } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 3857465bf..153470867 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -28,6 +28,7 @@ mod cmd_healthcheck; mod cmd_init; mod cmd_manage; mod cmd_migrate; +mod cmd_migrate_runner; mod cmd_serve; mod cmd_settings; mod cmd_status; @@ -69,7 +70,9 @@ enum Command { Destroy(cmd_destroy::DestroyArgs), /// Validate a extenddb deployment Verify(cmd_verify::VerifyArgs), - /// Apply catalog schema migrations (exit 0 up to date or applied, 3 work pending without --yes) + /// Apply catalog migrations (exit 0 up to date or applied, 1 error, 3 pending without --yes, + /// 4 the catalog has a breaking migration this binary lacks, 5 a person must act, 6 the + /// catalog stayed locked) Migrate(cmd_migrate::MigrateArgs), /// Check if the extenddb server is running Status(cmd_status::StatusArgs), @@ -106,18 +109,25 @@ pub fn run(build: BuildInfo) -> anyhow::Result<()> { match cli.command.unwrap_or(Command::Version) { Command::Serve(args) => cmd_serve::run(&args, build), Command::Init(args) => { - let code = run_interactive(cmd_init::run(args))?; + let code = run_interactive(cmd_init::run(args, build.version))?; if code != 0 { std::process::exit(i32::from(code)); } Ok(()) } Command::Destroy(args) => run_interactive(cmd_destroy::run(args)), - Command::Verify(args) => run_interactive(cmd_verify::run(args)), + Command::Verify(args) => { + let code = run_interactive(cmd_verify::run(args, build.version))?; + if code != 0 { + std::process::exit(i32::from(code)); + } + Ok(()) + } Command::Migrate(args) => { // 0 up to date or applied; cmd_migrate::EXIT_PENDING for a preview - // that found work; errors exit 1 through main. - let code = run_interactive(cmd_migrate::run(args))?; + // that found work; on the migration ledger, the status's code; + // errors exit 1 through main. + let code = run_interactive(cmd_migrate::run(args, build.version))?; if code != 0 { std::process::exit(i32::from(code)); } @@ -152,14 +162,14 @@ pub fn run(build: BuildInfo) -> anyhow::Result<()> { fn print_version(build: BuildInfo) { println!("extenddb {}", build.version); - // One backend is compiled into this binary; report its catalog version. - match ( - extenddb_storage::backend_name(), - extenddb_storage::operations::catalog_version(), - ) { - (Some(backend), Ok(version)) => println!("catalog {version} ({backend})"), - (Some(backend), Err(_)) => println!("catalog unknown ({backend})"), - (None, _) => println!("catalog unknown (no backend installed)"), + // One backend is compiled into this binary; report the catalog it needs: + // the highest migration it carries, or a backend's catalog version. + match extenddb_storage::backend_name() { + Some(backend) => println!( + "catalog {} ({backend})", + extenddb_storage::migrations::binary_catalog_label() + ), + None => println!("catalog unknown (no backend installed)"), } println!("commit {}", build.git_hash); diff --git a/crates/app/src/util.rs b/crates/app/src/util.rs index b22d330ad..2ee12b99c 100755 --- a/crates/app/src/util.rs +++ b/crates/app/src/util.rs @@ -22,6 +22,54 @@ pub fn is_process_alive(_pid: i32) -> bool { true } +/// The configuration `migrate` and `verify` read: the file `--config` +/// names, else `extenddb.toml` when it exists, else, in a dev-mode build, +/// the built-in defaults with the `EXTENDDB__` environment variables, as +/// `serve` reads them when it starts without a file. Returns the file read, +/// if any. +pub(crate) fn command_config( + given: Option<&str>, +) -> anyhow::Result<(extenddb_config::AppConfig, Option)> { + let path = given.unwrap_or(crate::cmd_migrate::DEFAULT_CONFIG); + if std::path::Path::new(path).exists() { + return Ok((extenddb_config::load(path)?, Some(path.to_owned()))); + } + if given.is_none() && cfg!(feature = "dev-mode") { + return Ok((extenddb_config::load_builtin_defaults()?, None)); + } + anyhow::bail!( + "Config file '{path}' not found. Run 'extenddb init' to set up a deployment, \ + or use --config to specify a different location." + ) +} + +/// Print each step a run applied, as ` `, naming +/// the store only when the run touched more than one, and each step's notes +/// below it. +pub(crate) fn print_applied( + report: &extenddb_storage::migrations::RunReport, + indent: &str, + verb: &str, +) { + let several = report + .applied + .windows(2) + .any(|w| w[0].store_name != w[1].store_name); + for step in &report.applied { + if several { + println!( + "{indent}{verb} {} {} in {}", + step.id, step.name, step.store_name + ); + } else { + println!("{indent}{verb} {} {}", step.id, step.name); + } + for note in &step.notes { + println!("{indent} {note}"); + } + } +} + /// Append a secret to the backend argument vector when the corresponding CLI /// flag was not supplied. The value may come from an environment variable, so /// it is added only to this in-process copy and never to the process command diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index baf3fc7d0..df58be389 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -40,3 +40,9 @@ extenddb-storage-postgres = { workspace = true, optional = true } extenddb-storage-sqlite = { workspace = true, optional = true } anyhow = { workspace = true } extenddb-config = { workspace = true } + +[dev-dependencies] +sqlx = { workspace = true } +tokio = { workspace = true } +uuid = { workspace = true } +extenddb-storage = { workspace = true, features = ["test-util"] } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index 7928b9216..403986e24 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -81,7 +81,9 @@ fn main() -> anyhow::Result<()> { #[cfg(test)] mod tests { - /// Install this binary's backend once for the test process. + /// Install this binary's backend once for the test process. Only the + /// postgres and sqlite builds have tests that need it. + #[cfg(any(feature = "postgres", feature = "sqlite"))] fn install_backend() { #[cfg(feature = "cassandra")] let _ = extenddb_storage::set_backend(extenddb_storage_cassandra::backend()); @@ -131,4 +133,89 @@ mod tests { assert_eq!(cfg.server.bind_addr, "127.0.0.1"); assert!(cfg.server.tls.enabled); } + + /// No PostgreSQL migration is declared additive until CI runs the + /// previous release against a catalog this binary migrated; the PR that + /// adds that job deletes this test. It lives here because the backend's + /// table is public and this crate builds with the postgres feature in + /// `cargo test --workspace`. + #[cfg(feature = "postgres")] + #[test] + fn no_postgres_migration_is_additive_before_the_old_binary_job() { + extenddb_storage::migrations::refuse_additive( + "postgres", + extenddb_storage_postgres::migrations::migration, + ) + .unwrap_or_else(|e| panic!("{e}")); + } + + /// `migrate --yes` on PostgreSQL, as the command runs it (the lock, then + /// the status under it), refuses two empty databases (a DBA created + /// them ahead of `init`) as `Uninitialized`, exit 1, and writes nothing + /// to either. Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (a base URL + /// with no database, for a role that may create databases); skips + /// without it. + #[cfg(feature = "postgres")] + #[tokio::test] + async fn migrate_yes_refuses_empty_postgres_databases() { + use extenddb_storage::migrations::{ + CatalogStatus, MigrationError, RunKind, Target, exit, installed, run_locked, + }; + + let Some(base) = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING") + .ok() + .map(|s| s.trim().trim_end_matches('/').to_owned()) + .filter(|s| !s.is_empty()) + else { + eprintln!("skipped: EXTENDDB_TEST_PG_CONNECTION_STRING not set"); + return; + }; + install_backend(); + let stem = format!( + "eddb_bin_{}", + &uuid::Uuid::new_v4().simple().to_string()[..12] + ); + let databases = [format!("{stem}_catalog"), stem.clone()]; + let admin = sqlx::PgPool::connect(&format!("{base}/postgres")) + .await + .expect("admin connection"); + for name in &databases { + sqlx::query(&format!("CREATE DATABASE \"{name}\"")) + .execute(&admin) + .await + .expect("create database"); + } + let target = Target::at(format!("{base}/{}", databases[0])); + let migrations = installed().expect("the postgres backend migrates on the runner"); + let result = async { + let lock = migrations + .lock(&target, std::time::Duration::from_secs(10)) + .await?; + run_locked(migrations, &target, &lock, "test", RunKind::Migrate).await + } + .await; + let mut written = Vec::new(); + for name in &databases { + let pool = sqlx::PgPool::connect(&format!("{base}/{name}")) + .await + .expect("database connection"); + let tables: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public'") + .fetch_one(&pool) + .await + .expect("count tables"); + pool.close().await; + written.push(tables); + } + for name in &databases { + let _ = sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) + .execute(&admin) + .await; + } + admin.close().await; + let err = result.expect_err("an empty catalog database is not a catalog"); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + assert_eq!(err.exit_code(), exit::ERROR); + assert_eq!(written, [0, 0], "tables written to the empty databases"); + } } diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs index 5380da38d..ceca1cb56 100644 --- a/crates/core/src/settings_keys.rs +++ b/crates/core/src/settings_keys.rs @@ -20,9 +20,8 @@ pub const INDEX_PROPAGATION_DELAY_MS: &str = "index_propagation_delay_ms"; /// The pre-rename name of [`INDEX_PROPAGATION_DELAY_MS`], still honoured. /// /// Two reasons this cannot simply be deleted. A catalog created before the rename -/// holds the operator's value under the old name, and the server refuses to start on -/// a catalog-version mismatch rather than migrating, so there is no upgrade step in -/// which the row could be rewritten. Silently reading past that row would reset a +/// holds the operator's value under the old name, and no catalog migration rewrites +/// that row yet, though one could. Silently reading past that row would reset a /// deliberately configured delay to the default, and a delay of 0 means synchronous, /// so the silent change would be from strict to eventually consistent. /// diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index e9948c111..7b827cd3a 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -181,14 +181,6 @@ pub fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> Dynamo // "unsupported" error class, and the request is invalid against this // deployment rather than a server failure. StorageError::Unsupported(msg) => DynamoDbError::ValidationException(msg), - StorageError::CatalogVersionMismatch { expected, found } => { - tracing::error!("Catalog version mismatch: expected {expected}, found {found}"); - DynamoDbError::InternalServerError("Internal server error".to_owned()) - } - StorageError::CatalogNotInitialized => { - tracing::error!("Catalog not initialized"); - DynamoDbError::InternalServerError("Internal server error".to_owned()) - } // Generic path: discard the old item (callers that need it use // `storage_err_to_dynamo_with_ccf` instead). StorageError::ConditionFailed(_) => DynamoDbError::ConditionalCheckFailedException( diff --git a/crates/server/src/console/pages/mod.rs b/crates/server/src/console/pages/mod.rs index 3de352c49..04945850b 100755 --- a/crates/server/src/console/pages/mod.rs +++ b/crates/server/src/console/pages/mod.rs @@ -146,5 +146,7 @@ fn op_error_message(e: extenddb_storage::management_store::OpError) -> String { | OpError::HasDependents(m) | OpError::NotFound(m) | OpError::Internal(m) => m, + // The driver's text stays out of the page. + OpError::Database { .. } => "Database error".to_owned(), } } diff --git a/crates/server/src/console/pages/settings_pages.rs b/crates/server/src/console/pages/settings_pages.rs index 62a63165f..0190486c4 100755 --- a/crates/server/src/console/pages/settings_pages.rs +++ b/crates/server/src/console/pages/settings_pages.rs @@ -22,20 +22,29 @@ use extenddb_config::{MASK, should_redact}; use super::{identity_label, is_admin, require_session}; -/// Known runtime settings with their default values. -/// These are the settings that can be changed via `extenddb settings set`. -const RUNTIME_DEFAULTS: &[(&str, &str)] = &[ - ("allow_credential_import", "false"), - ("catalog_version", "—"), - ("control_plane_delay_seconds", "0.25"), - ("data_database_connection_string", "—"), - ("data_database_name", "—"), - ("index_propagation_delay_ms", "500"), - ("log_level", "info"), - ("sqlx_log_level", "warn"), - ("throttling_enabled", "false"), +/// Runtime settings the page always lists, with the server's default for a +/// key the server reads. A key the storage engine reads has no entry here: +/// its default is the engine's ([`extenddb_storage::operations::setting_default`]), +/// which differs by backend. +const RUNTIME_KEYS: &[(&str, Option<&str>)] = &[ + ("allow_credential_import", Some("false")), + ("catalog_version", None), + ("control_plane_delay_seconds", None), + ("data_database_connection_string", None), + ("data_database_name", None), + ("index_propagation_delay_ms", None), + ("log_level", Some("info")), + ("sqlx_log_level", Some("warn")), + ("throttling_enabled", Some("false")), ]; +/// What the page prints for `key` when the catalog has no row for it. +fn default_value(key: &str, server_default: Option<&str>) -> String { + extenddb_storage::operations::setting_default(key) + .or_else(|| server_default.map(str::to_owned)) + .unwrap_or_else(|| "—".to_owned()) +} + /// GET /console/settings — read-only settings display (admin only). // Linear page-rendering function: auth check → build toml table → build runtime // table → assemble HTML. Splitting would scatter the rendering logic. @@ -107,12 +116,12 @@ pub async fn settings_page(State(state): State>, headers: Head .map(|(k, v)| (k.as_str(), v.as_str())) .collect(); - for &(key, default) in RUNTIME_DEFAULTS { + for &(key, server_default) in RUNTIME_KEYS { let ek = html::escape(key); let (value, source) = if let Some(&db_val) = db_map.get(key) { (db_val.to_owned(), "database") } else { - (default.to_owned(), "default") + (default_value(key, server_default), "default") }; let ev = if should_redact(key) { MASK.to_owned() @@ -131,9 +140,9 @@ pub async fn settings_page(State(state): State>, headers: Head ); } - // Show any DB settings not in RUNTIME_DEFAULTS (unexpected keys). + // Show any DB settings not in RUNTIME_KEYS (unexpected keys). for (key, value) in &db_rows { - if RUNTIME_DEFAULTS.iter().any(|&(k, _)| k == key) { + if RUNTIME_KEYS.iter().any(|&(k, _)| k == key) { continue; } let ek = html::escape(key); diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 3b1b30309..f608c74d7 100755 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -20,6 +20,7 @@ mod leading_keys; pub mod management; mod metrics_endpoint; pub mod rate_limit; +mod registration; mod request_helpers; mod request_tags; mod response; @@ -27,7 +28,9 @@ mod serve; mod throttle_helpers; mod workers; -pub use serve::{BuildInfo, LogTarget, ServeParams, log_to_syslog_raw, serve}; +pub use serve::{ + BuildInfo, CatalogRefusal, LogTarget, RegistrationLost, ServeParams, log_to_syslog_raw, serve, +}; use std::path::PathBuf; use std::sync::Arc; @@ -94,6 +97,10 @@ pub struct AppState { pub config_entries: Vec<(String, String)>, /// Runtime documentation store. `None` if `docs_dir` is not configured. pub docs_store: Option, + /// Whether new requests are admitted, and whether the server must stop: + /// set while the backend's serving registration is lost + /// ([`registration`]). + pub(crate) admission: registration::Admission, } /// TLS configuration passed from the binary crate. @@ -113,6 +120,10 @@ pub struct ServerTlsConfig { /// /// When `tls` is `Some`, the server serves HTTPS using `axum-server` with rustls. /// When `tls` is `None`, the server serves plaintext HTTP. +/// +/// It stops on SIGINT or SIGTERM, or when `state`'s admission asks for a +/// stop after the serving registration was lost; both drain the listener +/// the same way and remove the PID file. pub async fn start_server( listener: tokio::net::TcpListener, state: AppState, @@ -138,6 +149,7 @@ pub async fn start_server( format!("{scheme}://{addr}") }; let config_entries = state.config_entries.clone(); + let admission = state.admission.clone(); let shared = Arc::new(state); // Security headers applied to all responses. @@ -204,6 +216,13 @@ pub async fn start_server( ); } + // While the serving registration is lost, every route answers + // ServiceUnavailable before its handler runs. + let app = app.layer(axum::middleware::from_fn_with_state( + admission.clone(), + registration::refuse_while_unregistered, + )); + // Apply security headers to all routes. let app = app.layer(security_layers); @@ -239,8 +258,9 @@ pub async fn start_server( let handle = axum_server::Handle::new(); let shutdown_handle = handle.clone(); let shutdown_pid = pid_file.clone(); + let stop = admission.clone(); tokio::spawn(async move { - shutdown_signal().await; + signal_or_stop(&stop).await; shutdown_handle.graceful_shutdown(Some(Duration::from_secs(5))); // P50: Log timeout but don't call std::process::exit — let the // runtime shut down normally so destructors (including ZeroizeOnDrop) run. @@ -268,7 +288,7 @@ pub async fn start_server( nodelay_listener(listener), app.into_make_service_with_connect_info::(), ) - .with_graceful_shutdown(graceful_shutdown(pid_file.clone())) + .with_graceful_shutdown(graceful_shutdown(pid_file.clone(), admission)) .await?; } @@ -283,8 +303,8 @@ async fn health_check() -> impl IntoResponse { (StatusCode::OK, axum::Json(json!({"status": "healthy"}))) } -async fn graceful_shutdown(pid_file: Option) { - shutdown_signal().await; +async fn graceful_shutdown(pid_file: Option, admission: registration::Admission) { + signal_or_stop(&admission).await; // P50: Spawn a timeout task that cleans up the PID file if connections // don't drain. Does NOT call std::process::exit — the runtime shuts down // normally so destructors (including ZeroizeOnDrop) run. @@ -410,6 +430,20 @@ fn send_https_redirect(stream: &tokio::net::TcpStream, addr: std::net::SocketAdd let _ = stream.try_write(response.as_bytes()); } +/// Resolves on a shutdown signal, or once `admission` asks for a stop +/// because the serving registration could not be taken again. +async fn signal_or_stop(admission: ®istration::Admission) { + tokio::select! { + () = shutdown_signal() => admission.mark_signalled(), + () = admission.stop_requested() => { + tracing::info!( + "Stopping because the serving registration could not be taken again, draining \ + connections..." + ); + } + } +} + #[cfg(unix)] async fn shutdown_signal() { let ctrl_c = tokio::signal::ctrl_c(); @@ -498,4 +532,34 @@ mod tests { assert_eq!(peer, client.local_addr().expect("client addr")); assert!(accepted.nodelay().expect("read TCP_NODELAY")); } + + /// A stop that the registration supervisor asks for ends the plaintext + /// server through the graceful shutdown a signal starts. + #[tokio::test] + async fn a_stop_after_a_lost_registration_ends_the_server_as_a_signal_does() { + use std::future::IntoFuture; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let admission = registration::Admission::default(); + let app = Router::new().route("/", get(|| async { "served" })); + let server = tokio::spawn( + axum::serve(listener, app) + .with_graceful_shutdown(graceful_shutdown(None, admission.clone())) + .into_future(), + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!server.is_finished(), "the server stopped by itself"); + admission.stop("the registration could not be taken again".to_owned()); + tokio::time::timeout(Duration::from_secs(10), server) + .await + .expect("the server stops within 10 s of the stop") + .expect("the server task") + .expect("the server"); + assert_eq!( + admission.stop_reason(), + Some("the registration could not be taken again") + ); + } } diff --git a/crates/server/src/management/ops.rs b/crates/server/src/management/ops.rs index d02e143e5..ab53c9283 100755 --- a/crates/server/src/management/ops.rs +++ b/crates/server/src/management/ops.rs @@ -50,7 +50,9 @@ impl OpError { StorageOpError::AlreadyExists(msg) => Self::AlreadyExists(msg), StorageOpError::NotFound(msg) => Self::NotFound(msg), StorageOpError::HasDependents(msg) => Self::HasDependents(msg), - StorageOpError::Internal(msg) => Self::Internal(msg), + StorageOpError::Internal(msg) | StorageOpError::Database { message: msg, .. } => { + Self::Internal(msg) + } } } } diff --git a/crates/server/src/registration.rs b/crates/server/src/registration.rs new file mode 100644 index 000000000..bcdc5d075 --- /dev/null +++ b/crates/server/src/registration.rs @@ -0,0 +1,493 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What the server does when its backend reports the serving registration +//! lost ([`ServingRegistrar`]). +//! +//! A migration can start on the catalog once the registration is gone, so +//! from that moment the server admits no new request: each one gets +//! `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and `/health` +//! answers the same. Requests already running go on. The server asks the +//! backend to register again at once and then every second, and serves +//! again when the backend reports a new registration with the catalog's +//! status unchanged. When the backend refuses (a migration holds the +//! migration lock or waits for the servers to stop, or the status changed), +//! or 30 s pass without a registration, the server asks for a stop: the +//! workers' shutdown token fires at once, and `serve` then stops the way a +//! shutdown signal stops it (the listener drains, letting requests already +//! running finish, the PID file is removed) and exits 1. A signal that came +//! first decides the exit status, also when a stop is asked for during its +//! drain. +//! +//! The server learns of a loss at its backend's next renewal, and serves as +//! before until then; a migration that takes the catalog in that time runs +//! beside it, as it would beside a server whose host stopped answering. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use axum::extract::{Request, State}; +use axum::middleware::Next; +use axum::response::Response; +use extenddb_core::error::DynamoDbError; +use extenddb_storage::CancellationToken; +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + +/// How long the server tries to register again before it stops: the bound +/// the Cassandra and MongoDB backends give a registration they cannot renew. +pub(crate) const WINDOW: Duration = Duration::from_secs(30); + +/// The pause between two attempts to register again. +const RETRY: Duration = Duration::from_secs(1); + +/// The message of the `ServiceUnavailable` a refused request gets. +const REFUSED: &str = "The server is registering again with its catalog. Retry the request."; + +/// Whether the server admits new requests, and whether it must stop. Shared +/// by the request path, the supervisor and `serve`. +#[derive(Clone, Default)] +pub(crate) struct Admission { + inner: Arc, +} + +#[derive(Default)] +struct Inner { + refusing: AtomicBool, + stop: CancellationToken, + reason: OnceLock, + /// A shutdown signal ended the server's wait before any stop was asked + /// for. + signalled: AtomicBool, +} + +impl Admission { + /// Whether new requests are refused now. + pub(crate) fn refusing(&self) -> bool { + self.inner.refusing.load(Ordering::Acquire) + } + + fn refuse(&self) { + self.inner.refusing.store(true, Ordering::Release); + } + + fn admit(&self) { + self.inner.refusing.store(false, Ordering::Release); + } + + /// Ask `serve` to stop through its shutdown path, giving why. The first + /// reason is kept. + pub(crate) fn stop(&self, reason: String) { + let _ = self.inner.reason.set(reason); + self.inner.stop.cancel(); + } + + /// Resolves once a stop has been asked for. + pub(crate) async fn stop_requested(&self) { + self.inner.stop.cancelled().await; + } + + /// Why a stop was asked for, or `None` when none was. + pub(crate) fn stop_reason(&self) -> Option<&str> { + self.inner.reason.get().map(String::as_str) + } + + /// Record that a shutdown signal, not a stop, ended the server's wait. + pub(crate) fn mark_signalled(&self) { + self.inner.signalled.store(true, Ordering::Release); + } + + /// Why the server stopped, when a stop it asked for ended its wait: a + /// stop asked for while a signal's drain ran is not why it stopped. + pub(crate) fn stopped_for(&self) -> Option<&str> { + if self.inner.signalled.load(Ordering::Acquire) { + return None; + } + self.stop_reason() + } +} + +/// The layer in front of every route: while the server is refusing, the +/// request is answered with `ServiceUnavailable` and does not reach its +/// handler. +pub(crate) async fn refuse_while_unregistered( + State(admission): State, + request: Request, + next: Next, +) -> Response { + if admission.refusing() { + let request_id = uuid::Uuid::new_v4().to_string(); + return crate::response::error_response( + &DynamoDbError::ServiceUnavailable(REFUSED.to_owned()), + &request_id, + ); + } + next.run(request).await +} + +/// Watch the registration until `shutdown` fires: on each loss refuse new +/// requests and register again within [`WINDOW`], or ask for a stop. +pub(crate) async fn supervise( + registrar: Arc, + admission: Admission, + shutdown: CancellationToken, +) { + supervise_within(registrar, admission, shutdown, WINDOW, RETRY).await; +} + +/// How one loss ended. +enum Outcome { + Registered { attempts: u32, after: Duration }, + Stop(String), + Shutdown, +} + +/// [`supervise`] with the window and the pause between attempts given, for +/// tests. +async fn supervise_within( + registrar: Arc, + admission: Admission, + shutdown: CancellationToken, + window: Duration, + retry: Duration, +) { + loop { + let reason = tokio::select! { + () = shutdown.cancelled() => return, + reason = registrar.lost() => reason, + }; + admission.refuse(); + tracing::warn!( + "{reason}; new requests get ServiceUnavailable while the server registers again, for \ + up to {} s", + window.as_secs() + ); + match register_again(registrar.as_ref(), &shutdown, window, retry).await { + Outcome::Registered { attempts, after } => { + admission.admit(); + tracing::info!( + "registered again with the catalog after {} ms and {attempts} attempt(s), and \ + its status is the one the server started with; serving requests again", + after.as_millis() + ); + } + Outcome::Stop(why) => { + let message = + format!("the server stops: {reason}, and it could not register again: {why}"); + tracing::error!("{message}"); + // A migration may hold the catalog now: the background + // workers stop at once, before the listener's drain, which + // lets requests already running finish. + shutdown.cancel(); + admission.stop(message); + return; + } + Outcome::Shutdown => return, + } + } +} + +/// Attempts until one registers, the backend refuses, `window` passes or +/// `shutdown` fires. Each attempt is bounded by the time left. +async fn register_again( + registrar: &dyn ServingRegistrar, + shutdown: &CancellationToken, + window: Duration, + retry: Duration, +) -> Outcome { + let started = Instant::now(); + let mut attempts = 0; + let mut last = String::new(); + loop { + let left = window.saturating_sub(started.elapsed()); + if left.is_zero() { + return Outcome::Stop(format!( + "no attempt succeeded within {} s ({attempts} attempt(s); the last one: {last})", + window.as_secs() + )); + } + attempts += 1; + let answer = tokio::select! { + () = shutdown.cancelled() => return Outcome::Shutdown, + answer = tokio::time::timeout(left, registrar.register_again()) => answer, + }; + match answer { + Ok(Reregistration::Registered) => { + return Outcome::Registered { + attempts, + after: started.elapsed(), + }; + } + Ok(Reregistration::Refused(why)) => return Outcome::Stop(why), + Ok(Reregistration::Unavailable(why)) => { + tracing::warn!("attempt {attempts} to register again failed: {why}"); + last = why; + } + Err(_) => { + last = format!("it did not finish within {} ms", left.as_millis()); + } + } + let pause = retry.min(window.saturating_sub(started.elapsed())); + tokio::select! { + () = shutdown.cancelled() => return Outcome::Shutdown, + () = tokio::time::sleep(pause) => {} + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::{Arc, Mutex, OnceLock}; + use std::time::{Duration, Instant}; + + use async_trait::async_trait; + use axum::Router; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use axum::routing::get; + use extenddb_storage::CancellationToken; + use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + use tokio::sync::Notify; + use tower::ServiceExt; + + use super::{Admission, refuse_while_unregistered, supervise_within}; + + /// A registrar whose losses a test fires, and whose attempts answer from + /// a script; an empty script answers `Unavailable`, and `None` in it is + /// an attempt that never finishes. Each attempt records whether the + /// server was refusing requests when it was made. + #[derive(Default)] + struct Scripted { + lose: Notify, + answers: Mutex>>, + attempts: AtomicU32, + admission: OnceLock, + refusing_at_attempts: Mutex>, + } + + impl Scripted { + fn answering(answers: Vec>) -> Arc { + Arc::new(Self { + answers: Mutex::new(answers.into()), + ..Self::default() + }) + } + + fn attempts(&self) -> u32 { + self.attempts.load(Ordering::SeqCst) + } + } + + #[async_trait] + impl ServingRegistrar for Scripted { + async fn lost(&self) -> String { + self.lose.notified().await; + "the session ended".to_owned() + } + + async fn register_again(&self) -> Reregistration { + if let Some(admission) = self.admission.get() { + self.refusing_at_attempts + .lock() + .unwrap() + .push(admission.refusing()); + } + self.attempts.fetch_add(1, Ordering::SeqCst); + let answer = self.answers.lock().unwrap().pop_front(); + match answer { + Some(Some(answer)) => answer, + Some(None) => std::future::pending().await, + None => Reregistration::Unavailable("no answer".to_owned()), + } + } + } + + async fn until(what: &str, mut done: impl FnMut() -> bool) { + let started = Instant::now(); + while !done() { + assert!(started.elapsed() < Duration::from_secs(10), "{what}"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + fn start( + registrar: &Arc, + window: Duration, + ) -> (Admission, CancellationToken, tokio::task::JoinHandle<()>) { + let admission = Admission::default(); + let _ = registrar.admission.set(admission.clone()); + let shutdown = CancellationToken::new(); + let task = tokio::spawn(supervise_within( + Arc::clone(registrar) as Arc, + admission.clone(), + shutdown.clone(), + window, + Duration::from_millis(20), + )); + (admission, shutdown, task) + } + + #[tokio::test] + async fn a_lost_registration_refuses_requests_until_it_registers_again() { + let registrar = Scripted::answering(vec![ + Some(Reregistration::Unavailable("connection refused".to_owned())), + Some(Reregistration::Unavailable("connection refused".to_owned())), + Some(Reregistration::Registered), + ]); + let (admission, shutdown, task) = start(®istrar, Duration::from_secs(5)); + assert!(!admission.refusing()); + registrar.lose.notify_one(); + until("three attempts", || registrar.attempts() == 3).await; + until("admitting once registered", || !admission.refusing()).await; + assert_eq!( + *registrar.refusing_at_attempts.lock().unwrap(), + [true, true, true], + "requests were admitted while the server held no registration" + ); + assert_eq!(admission.stop_reason(), None); + // A second loss is handled the same way. + registrar + .answers + .lock() + .unwrap() + .push_back(Some(Reregistration::Registered)); + registrar.lose.notify_one(); + until("a fourth attempt", || registrar.attempts() == 4).await; + until("admitting again", || !admission.refusing()).await; + assert!( + registrar.refusing_at_attempts.lock().unwrap()[3], + "requests were admitted while the server held no registration" + ); + shutdown.cancel(); + task.await.unwrap(); + assert_eq!(admission.stop_reason(), None); + } + + #[tokio::test] + async fn a_refusal_asks_for_a_stop_naming_it() { + let registrar = Scripted::answering(vec![Some(Reregistration::Refused( + "a migration is running on this catalog".to_owned(), + ))]); + let (admission, shutdown, task) = start(®istrar, Duration::from_secs(5)); + registrar.lose.notify_one(); + tokio::time::timeout(Duration::from_secs(10), admission.stop_requested()) + .await + .expect("a stop is asked for"); + // A migration may hold the catalog by now: the background workers' + // token fires with the stop, before the listener drains. + assert!( + shutdown.is_cancelled(), + "the workers run on beside the migration" + ); + task.await.unwrap(); + let reason = admission.stop_reason().unwrap(); + assert!( + reason.contains("the session ended") + && reason.contains("a migration is running on this catalog"), + "{reason}" + ); + assert!(admission.refusing(), "admitting after a refusal"); + assert_eq!(registrar.attempts(), 1); + assert_eq!(admission.stopped_for(), Some(reason)); + } + + /// A signal that ended the server's wait first decides how it ends: a + /// stop asked for during the signal's drain leaves the exit status 0. + #[test] + fn a_stop_during_a_signals_drain_is_not_why_the_server_stopped() { + let signalled = Admission::default(); + signalled.mark_signalled(); + signalled.stop("the window passed".to_owned()); + assert_eq!(signalled.stopped_for(), None); + let stopped = Admission::default(); + stopped.stop("the window passed".to_owned()); + assert_eq!(stopped.stopped_for(), Some("the window passed")); + } + + #[tokio::test] + async fn the_server_stops_when_the_window_passes_without_a_registration() { + let registrar = Scripted::answering(Vec::new()); + let window = Duration::from_millis(300); + let (admission, _shutdown, task) = start(®istrar, window); + registrar.lose.notify_one(); + let lost_at = Instant::now(); + tokio::time::timeout(Duration::from_secs(10), admission.stop_requested()) + .await + .expect("a stop is asked for"); + let elapsed = lost_at.elapsed(); + task.await.unwrap(); + assert!(elapsed >= window, "stopped {elapsed:?} after the loss"); + let reason = admission.stop_reason().unwrap(); + assert!( + reason.contains("no attempt succeeded") && reason.contains("no answer"), + "{reason}" + ); + assert!(registrar.attempts() > 1); + } + + #[tokio::test] + async fn an_attempt_that_never_answers_is_bounded_by_the_window() { + let registrar = Scripted::answering(vec![None]); + let window = Duration::from_millis(300); + let (admission, _shutdown, task) = start(®istrar, window); + registrar.lose.notify_one(); + tokio::time::timeout(Duration::from_secs(10), admission.stop_requested()) + .await + .expect("a stop is asked for"); + task.await.unwrap(); + let reason = admission.stop_reason().unwrap(); + assert!(reason.contains("did not finish"), "{reason}"); + } + + #[tokio::test] + async fn a_shutdown_while_registering_again_asks_for_no_stop() { + let registrar = Scripted::answering(vec![None]); + let (admission, shutdown, task) = start(®istrar, Duration::from_secs(30)); + registrar.lose.notify_one(); + until("an attempt started", || registrar.attempts() == 1).await; + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the supervisor returns at shutdown") + .unwrap(); + assert_eq!(admission.stop_reason(), None); + } + + #[tokio::test] + async fn a_refusing_server_answers_service_unavailable_on_every_route() { + let admission = Admission::default(); + let app = Router::new() + .route("/", get(|| async { "served" })) + .route("/health", get(|| async { "healthy" })) + .layer(axum::middleware::from_fn_with_state( + admission.clone(), + refuse_while_unregistered, + )); + let get = |path: &str| Request::get(path).body(Body::empty()).unwrap(); + let served = app.clone().oneshot(get("/")).await.unwrap(); + assert_eq!(served.status(), StatusCode::OK); + admission.refuse(); + for path in ["/", "/health"] { + let refused = app.clone().oneshot(get(path)).await.unwrap(); + assert_eq!(refused.status(), StatusCode::SERVICE_UNAVAILABLE, "{path}"); + assert_eq!( + refused.headers()["content-type"], + "application/x-amz-json-1.0" + ); + assert!(refused.headers().contains_key("x-amzn-RequestId")); + let body = axum::body::to_bytes(refused.into_body(), 4096) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + body["__type"], + "com.amazonaws.dynamodb.v20120810#ServiceUnavailable" + ); + } + admission.admit(); + let served = app.oneshot(get("/health")).await.unwrap(); + assert_eq!(served.status(), StatusCode::OK); + } +} diff --git a/crates/server/src/request_helpers.rs b/crates/server/src/request_helpers.rs index 9ca4d7dce..ab1791692 100755 --- a/crates/server/src/request_helpers.rs +++ b/crates/server/src/request_helpers.rs @@ -478,14 +478,7 @@ mod tests { ); } - let internal = [ - StorageError::Internal("backend fault".to_owned()), - StorageError::CatalogVersionMismatch { - expected: "3".to_owned(), - found: "2".to_owned(), - }, - StorageError::CatalogNotInitialized, - ]; + let internal = [StorageError::Internal("backend fault".to_owned())]; for err in internal { let out = key_info_for_authz(Err(err.clone())); assert!( diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 9ebbaccd4..394f6df12 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -108,6 +108,9 @@ pub struct ServeParams { /// Only a `dev-mode` build of the app crate ever sets this, and it enforces /// the loopback-only bind before handing over the listener. pub dev_mode: bool, + /// The config file `serve` was started with, named in the command a + /// catalog refusal prints. `None` when it started from built-in defaults. + pub config_path: Option, } impl ServeParams { @@ -126,6 +129,7 @@ impl ServeParams { log_target: LogTarget::Syslog, build, dev_mode: false, + config_path: None, } } @@ -143,12 +147,56 @@ impl ServeParams { self.dev_mode = dev_mode; self } + + /// Name the config file in the command a catalog refusal prints. + #[must_use] + pub fn with_config_path(mut self, config_path: impl Into) -> Self { + self.config_path = Some(config_path.into()); + self + } +} + +/// `serve` stopped on the catalog's status before it opened the database: +/// the message names the status and the command that resolves it, and the +/// process should end with `exit_code` (the exit codes `migrate` and +/// `verify` use for the same status). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogRefusal { + pub exit_code: u8, + pub message: String, +} + +impl std::fmt::Display for CatalogRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for CatalogRefusal {} + +/// `serve` stopped because the backend's serving registration was lost and +/// could not be taken again. It stopped through the shutdown path a signal +/// takes: the listener drained, the workers stopped and the PID file was +/// removed. The process should end with exit status 1; the message, which +/// `serve` has logged, says why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RegistrationLost { + pub message: String, +} + +impl std::fmt::Display for RegistrationLost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } } +impl std::error::Error for RegistrationLost {} + /// Run the ExtendDB server on the pre-bound listener in `params` until shutdown. /// -/// On any error before the HTTP server starts, the PID file is removed and a -/// fatal message is written to the configured [`LogTarget`]. +/// On any error before the HTTP server starts, and on [`RegistrationLost`] +/// after it stopped, the PID file is removed and a fatal message is written +/// to the configured [`LogTarget`]. /// /// # Errors /// @@ -197,9 +245,9 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { log_target, build, dev_mode, + config_path, } = params; - let catalog_version = - extenddb_storage::operations::catalog_version().unwrap_or_else(|_| "unknown".to_string()); + let catalog_version = extenddb_storage::migrations::binary_catalog_label(); // Write the PID file, when the caller asked for one, so `extenddb status`, // `extenddb stop`, and `start_server`'s graceful shutdown cleanup work. When @@ -289,6 +337,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { extenddb_storage::server_components::ServerComponentsOptions::default(); component_options.bootstrap_if_uninitialized = dev_mode; + component_options.release = build.version; // Wire the node's own bind address as the instance ID so each ExtendDB node // has a distinct node ID (to support logical clocks for sequences, etc.). @@ -296,29 +345,67 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { .storage .set_instance_id(&format!("{}:{}", app_config.server.bind_addr, port)); - let components = extenddb_storage::create_server_components( + let components = match extenddb_storage::create_server_components( app_config.storage.as_trait(), &app_config.server.region, component_options, ) - .await?; + .await + { + Ok(components) => components, + Err(extenddb_storage::server_components::BackendError::Catalog(error)) => { + use extenddb_storage::migrations::{Target, advice, installed}; + // A server started without a config file (a dev server configured + // by `EXTENDDB__` variables) advises `migrate` without one, which + // reads the same variables. The backup clause names the data + // store the catalog records. + let target = Target::at(app_config.storage.connection_config()); + let named = match installed() { + Some(migrations) => Target { + data_store: migrations.data_store_name(&target).await, + ..target + }, + None => target, + }; + let advice = advice::AdviceFor::new( + build.version, + config::redact_password(app_config.storage.connection_config()), + installed(), + &named, + config_path.as_deref(), + ); + return Err(CatalogRefusal { + exit_code: error.exit_code(), + message: advice::error_message(&error, &advice.advice()), + } + .into()); + } + Err(other) => return Err(other.into()), + }; let storage = components.engine; let catalog_store = components.catalog_store; let cred_store = components.credential_store; let runtime_hooks = components.runtime_hooks; + let registrar = runtime_hooks + .as_ref() + .and_then(|hooks| hooks.serving_registrar()); + let admission = crate::registration::Admission::default(); // Resource-tag migration guard. ExtendDB evaluates resource tags only as // aws:ResourceTag/* and leaves dynamodb:ResourceTag/* absent, so a catalog // whose stored policies still name the dynamodb: form must be rewritten by // `extenddb migrate --yes` before this binary serves it. `migrate`, `init` // and every fresh-catalog bootstrap record a settings marker; refuse to - // start when it is missing. One settings-row read, no policy scan. - if catalog_store - .get_setting(extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION) - .await - .map_err(|e| anyhow::anyhow!("reading the resource-tag migration marker: {e:?}"))? - .is_none() + // start when it is missing. One settings-row read, no policy scan. A + // backend on the migration runner read the catalog's status in its + // factory instead, and migration 1 deletes the marker. + if extenddb_storage::migrations::installed().is_none() + && catalog_store + .get_setting(extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION) + .await + .map_err(|e| anyhow::anyhow!("reading the resource-tag migration marker: {e:?}"))? + .is_none() { anyhow::bail!( "this catalog has not had the resource-tag condition-key migration applied. \ @@ -596,6 +683,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { table_key_info_cache, config_entries, docs_store, + admission: admission.clone(), }; // Workers run until `shutdown` is cancelled, which happens after the HTTP @@ -648,6 +736,16 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { }; worker_handles.extend(hooks.spawn_workers(&worker_ctx).await); } + // A backend whose registration the database can end: the supervisor + // refuses requests while it is lost, and asks for a stop when it cannot + // be taken again. It is drained with the other workers. + if let Some(registrar) = registrar { + worker_handles.push(tokio::spawn(crate::registration::supervise( + registrar, + admission.clone(), + shutdown.clone(), + ))); + } let tls_config = if tls_enabled { let cert_path = config::expand_tilde(&app_config.server.tls.cert_path); @@ -668,6 +766,15 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { server_result?; + // The listener stopped because the registration could not be taken + // again, not on a signal: end with exit status 1. + if let Some(reason) = admission.stopped_for() { + return Err(RegistrationLost { + message: reason.to_owned(), + } + .into()); + } + Ok(()) } diff --git a/crates/storage-cassandra/src/bootstrapper.rs b/crates/storage-cassandra/src/bootstrapper.rs index f614971a7..8d4353ae5 100644 --- a/crates/storage-cassandra/src/bootstrapper.rs +++ b/crates/storage-cassandra/src/bootstrapper.rs @@ -241,7 +241,8 @@ impl CassandraBootstrapper { | OpError::AlreadyExists(m) | OpError::NotFound(m) | OpError::HasDependents(m) - | OpError::Internal(m) => StorageError::Internal(m), + | OpError::Internal(m) + | OpError::Database { message: m, .. } => StorageError::Internal(m), }) } } diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 5628926b4..9bf36d3ad 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -194,7 +194,7 @@ impl CassandraEngine { let value: String = row.get_r_by_name("value").ok()?; value.parse::().ok() }) - .unwrap_or(0.25); + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS); // Restore targets have no generic transition deadline; only the restore // path may publish them after payload verification. diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index 7dbfe98c4..e21bf181e 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -9,6 +9,11 @@ pub const CATALOG_VERSION: &str = "0.0.4"; +/// Default control-plane delay (seconds) when the `control_plane_delay_seconds` +/// setting is absent or unparseable: CreateTable, the control-plane worker +/// and the console's default all read this one value. +pub(crate) const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; + mod admin_store; mod authorization_store; mod backup_engine; @@ -253,6 +258,8 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, + // Migrates through its bootstrapper until it moves onto the runner. + catalog_migrations: None, } } diff --git a/crates/storage-cassandra/src/migrations.rs b/crates/storage-cassandra/src/migrations.rs index ddd6255ee..8eabe63b4 100644 --- a/crates/storage-cassandra/src/migrations.rs +++ b/crates/storage-cassandra/src/migrations.rs @@ -415,7 +415,8 @@ pub(crate) async fn pending_catalog_migrations( | OpError::AlreadyExists(m) | OpError::NotFound(m) | OpError::HasDependents(m) - | OpError::Internal(m) => StorageError::Internal(m), + | OpError::Internal(m) + | OpError::Database { message: m, .. } => StorageError::Internal(m), })?; if !recorded.contains(&version) { pending.push(*filename); diff --git a/crates/storage-cassandra/src/operations.rs b/crates/storage-cassandra/src/operations.rs index 1aad91d3f..e33cd0838 100644 --- a/crates/storage-cassandra/src/operations.rs +++ b/crates/storage-cassandra/src/operations.rs @@ -93,6 +93,19 @@ impl OperationsEngine for CassandraOperationsEngine { crate::CATALOG_VERSION.to_string() } + fn setting_default(&self, key: &str) -> Option { + use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; + match key { + // CreateTable's fallback and the control-plane worker's. + "control_plane_delay_seconds" => { + Some(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS.to_string()) + } + // The engine's initial GSI delay, kept while the row is absent. + INDEX_PROPAGATION_DELAY_MS => Some("10".to_owned()), + _ => None, + } + } + fn is_sensitive_key(&self, key: &str) -> bool { key.contains("password") || key.contains("secret") } diff --git a/crates/storage-cassandra/src/workers.rs b/crates/storage-cassandra/src/workers.rs index 520175ed6..2bd5996e7 100644 --- a/crates/storage-cassandra/src/workers.rs +++ b/crates/storage-cassandra/src/workers.rs @@ -181,7 +181,7 @@ async fn read_control_plane_delay(store: &S) -> f64 { .flatten() .and_then(|v| v.parse::().ok()) .filter(|&v| v >= 0.0) - .unwrap_or(0.25) + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS) } /// Handle that stops GSI workers when dropped. diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 37d9ed246..c4b95af91 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -266,6 +266,8 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, + // Migrates through its bootstrapper until it moves onto the runner. + catalog_migrations: None, } } diff --git a/crates/storage-mongodb/src/operations.rs b/crates/storage-mongodb/src/operations.rs index 2389a8f11..4257fdbdf 100644 --- a/crates/storage-mongodb/src/operations.rs +++ b/crates/storage-mongodb/src/operations.rs @@ -114,6 +114,16 @@ impl OperationsEngine for MongoOperationsEngine { "0.0.2".to_owned() } + fn setting_default(&self, key: &str) -> Option { + // The engine reads no index propagation delay. + match key { + "control_plane_delay_seconds" => { + Some(crate::worker_store::DEFAULT_CONTROL_PLANE_DELAY_SECONDS.to_string()) + } + _ => None, + } + } + fn is_sensitive_key(&self, key: &str) -> bool { let lower = key.to_lowercase(); [ diff --git a/crates/storage-mongodb/src/worker_store.rs b/crates/storage-mongodb/src/worker_store.rs index 6308f4fdb..4b51bf6bf 100644 --- a/crates/storage-mongodb/src/worker_store.rs +++ b/crates/storage-mongodb/src/worker_store.rs @@ -29,8 +29,8 @@ use extenddb_storage::error::StorageError; use crate::MongoEngine; /// Default control-plane delay (seconds) when the setting is absent or -/// unparseable. Matches the Postgres backend default. -const DEFAULT_CONTROL_PLANE_DELAY_SECS: f64 = 0.25; +/// unparseable: the workers and the console's default read this one value. +pub(crate) const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; impl MongoEngine { /// Read `control_plane_delay_seconds` from the settings collection, @@ -45,7 +45,7 @@ impl MongoEngine { .and_then(|d| d.get_str("value").ok().map(str::to_owned)) .and_then(|v| v.parse::().ok()) .filter(|v| *v >= 0.0) - .unwrap_or(DEFAULT_CONTROL_PLANE_DELAY_SECS) + .unwrap_or(DEFAULT_CONTROL_PLANE_DELAY_SECONDS) } } diff --git a/crates/storage-postgres/Cargo.toml b/crates/storage-postgres/Cargo.toml index 887647b57..3938e578f 100755 --- a/crates/storage-postgres/Cargo.toml +++ b/crates/storage-postgres/Cargo.toml @@ -11,14 +11,21 @@ license.workspace = true # Compiles the snapshot page-size setter the consistency test uses and the # shared conformance check. Production builds do not compile them. test-hooks = ["extenddb-storage/test-hooks"] +# Test hooks: the runner without a lock and its catalog connection, the lock +# and the registration with another keepalive interval, the serving +# session's settings, and the post-check's definitions. +test-util = ["extenddb-storage/test-util"] [dev-dependencies] # The storage-level vector control-plane tests drive the engine directly against # a scratch database, so they need a runtime and the test macro. tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -# The crate's own tests need `test-hooks`; a dev-dependency on itself turns -# the feature on for test builds only. -extenddb-storage-postgres = { path = ".", features = ["test-hooks"] } +# The checksum test checks this backend's migration table with the framework's +# test helper. +extenddb-storage = { workspace = true, features = ["test-util"] } +# The crate's own tests need `test-hooks` and `test-util`; a dev-dependency on +# itself turns both features on for test builds only. +extenddb-storage-postgres = { path = ".", features = ["test-hooks", "test-util"] } [dependencies] anyhow = { workspace = true } diff --git a/crates/storage-postgres/data_migrations/0001_baseline.expected.json b/crates/storage-postgres/data_migrations/0001_baseline.expected.json new file mode 100644 index 000000000..f1764b577 --- /dev/null +++ b/crates/storage-postgres/data_migrations/0001_baseline.expected.json @@ -0,0 +1,242 @@ +{ + "columns": { + "gsi_pending.id": { + "type": "bigint", + "not_null": true, + "default": "nextval('gsi_pending_id_seq'::regclass)", + "collation": null + }, + "gsi_pending.index_context": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "gsi_pending.new_item": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "gsi_pending.old_item": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "gsi_pending.ready_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "gsi_pending.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "gsi_pending.worker_partition": { + "type": "integer", + "not_null": true, + "default": null, + "collation": null + }, + "idempotency_tokens.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "idempotency_tokens.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "idempotency_tokens.fingerprint": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "idempotency_tokens.token": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "schema_history.applied_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "schema_history.filename": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "stream_records.event_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.record_data": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.sequence_number": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.shard_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "stream_shards.ending_sequence_number": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "stream_shards.parent_shard_id": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "stream_shards.shard_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.starting_sequence_number": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_index_holds.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "vector_index_holds.index_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_index_holds.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + } + }, + "indexes": { + "gsi_pending_pkey": { + "table": "gsi_pending", + "definition": "CREATE UNIQUE INDEX gsi_pending_pkey ON public.gsi_pending USING btree (id)", + "valid": true + }, + "idempotency_tokens_pkey": { + "table": "idempotency_tokens", + "definition": "CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (account_id, token)", + "valid": true + }, + "idx_gsi_pending_claim": { + "table": "gsi_pending", + "definition": "CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id)", + "valid": true + }, + "idx_idempotency_tokens_created": { + "table": "idempotency_tokens", + "definition": "CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at)", + "valid": true + }, + "idx_stream_records_created": { + "table": "stream_records", + "definition": "CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at)", + "valid": true + }, + "idx_stream_shards_table": { + "table": "stream_shards", + "definition": "CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id)", + "valid": true + }, + "idx_vector_index_holds_table": { + "table": "vector_index_holds", + "definition": "CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id)", + "valid": true + }, + "schema_history_pkey": { + "table": "schema_history", + "definition": "CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename)", + "valid": true + }, + "stream_records_pkey": { + "table": "stream_records", + "definition": "CREATE UNIQUE INDEX stream_records_pkey ON public.stream_records USING btree (shard_id, sequence_number)", + "valid": true + }, + "stream_shards_pkey": { + "table": "stream_shards", + "definition": "CREATE UNIQUE INDEX stream_shards_pkey ON public.stream_shards USING btree (shard_id)", + "valid": true + }, + "vector_index_holds_pkey": { + "table": "vector_index_holds", + "definition": "CREATE UNIQUE INDEX vector_index_holds_pkey ON public.vector_index_holds USING btree (table_id, index_id)", + "valid": true + } + }, + "constraints": { + "gsi_pending.gsi_pending_pkey": "PRIMARY KEY (id)", + "idempotency_tokens.idempotency_tokens_pkey": "PRIMARY KEY (account_id, token)", + "schema_history.schema_history_pkey": "PRIMARY KEY (filename)", + "stream_records.stream_records_pkey": "PRIMARY KEY (shard_id, sequence_number)", + "stream_records.stream_records_shard_id_fkey": "FOREIGN KEY (shard_id) REFERENCES stream_shards(shard_id) ON DELETE CASCADE", + "stream_shards.stream_shards_pkey": "PRIMARY KEY (shard_id)", + "vector_index_holds.vector_index_holds_pkey": "PRIMARY KEY (table_id, index_id)" + }, + "sequences": [ + "gsi_pending_id_seq", + "stream_seq" + ] +} diff --git a/crates/storage-postgres/data_migrations/0001_baseline.sql b/crates/storage-postgres/data_migrations/0001_baseline.sql new file mode 100644 index 000000000..e828fe4dd --- /dev/null +++ b/crates/storage-postgres/data_migrations/0001_baseline.sql @@ -0,0 +1,130 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 1, baseline, PostgreSQL data database. +-- +-- Creates the v1.0 data schema on an empty database and brings a data +-- database created by any released version to the same schema: the legacy +-- data files 001 to 004 in order, every statement safe to run again. Per-table +-- objects (`_ddb_`) are not created here; CreateTable creates them, and +-- later migrations change them. +-- +-- These tables live in the data database, beside item data, so stream +-- records and idempotency tokens commit in one transaction with the items. +-- No foreign key reaches the catalog, which is another database. + +-- Stream shards: fixed shards per table, assigned by partition key hash. +CREATE TABLE IF NOT EXISTS stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_stream_shards_table + ON stream_shards (table_id); + +-- Stream records: change data capture records. +CREATE TABLE IF NOT EXISTS stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (shard_id, sequence_number) +); + +CREATE INDEX IF NOT EXISTS idx_stream_records_created + ON stream_records (created_at); + +-- Monotonic sequence for stream record ordering. It starts at the current +-- epoch microseconds, and a run on a database that has one never moves it +-- back: a lower value would hand out sequence numbers already used. +CREATE SEQUENCE IF NOT EXISTS stream_seq START 1; +SELECT setval('stream_seq', GREATEST( + (EXTRACT(EPOCH FROM now()) * 1000000)::BIGINT, + (SELECT last_value FROM stream_seq), + 1 +)); + +-- Persistent queue for asynchronous GSI propagation (legacy 002). Rows are +-- inserted in the base write's transaction and consumed by workers in one +-- claim, apply and delete transaction. `index_context` carries everything a +-- worker needs; `worker_partition` hashes the base key, so one worker applies +-- every update of an item in `id` order. +CREATE TABLE IF NOT EXISTS gsi_pending ( + id BIGSERIAL PRIMARY KEY, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item JSONB, + new_item JSONB, + index_context JSONB NOT NULL, + ready_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); + +-- Idempotency tokens for TransactWriteItems, keyed per account (legacy 003). +-- A data database created by v0.1.0 or v0.1.1 that no later release migrated +-- has the table keyed on the token alone. The tokens are a cache with a ten +-- minute lifetime, so that table is recreated rather than copied, as legacy +-- 003 did; a table that has `account_id` is left as it is, with its tokens. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'idempotency_tokens' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'idempotency_tokens' + AND column_name = 'account_id' + ) THEN + DROP TABLE idempotency_tokens; + END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (account_id, token) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created + ON idempotency_tokens (created_at); + +-- Vector index build holds the propagation queue reads (legacy 004): one row +-- per table whose vector backfill is running. The row is inserted before the +-- catalog's CREATING row commits and deleted after the index is ACTIVE, so +-- the queue never applies a write a running backfill would overwrite. +CREATE TABLE IF NOT EXISTS vector_index_holds ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (table_id, index_id) +); + +CREATE INDEX IF NOT EXISTS idx_vector_index_holds_table + ON vector_index_holds (table_id); + +-- The ledger of releases before 1.0, with every legacy data file and the +-- legacy base key index step, so their `migrate` runs none of them: a replay +-- of legacy 003 would delete every idempotency token. +CREATE TABLE IF NOT EXISTS schema_history ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +INSERT INTO schema_history (filename) VALUES + ('001_data_schema.sql'), + ('002_gsi_pending.sql'), + ('003_idempotency_account_scope.sql'), + ('004_vector_index_state.sql'), + ('003_gsi_base_key_index') +ON CONFLICT (filename) DO NOTHING; diff --git a/crates/storage-postgres/data_migrations/001_data_schema.sql b/crates/storage-postgres/data_migrations/001_data_schema.sql deleted file mode 100755 index 72dce8014..000000000 --- a/crates/storage-postgres/data_migrations/001_data_schema.sql +++ /dev/null @@ -1,59 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Data database schema for extenddb. --- These tables live in the data database (separate from the catalog) so that --- stream records and idempotency tokens can be written atomically with item --- data within a single PostgreSQL transaction (P54 Bug 1). - -BEGIN; - --- Stream shards — fixed shards per table, assigned by partition key hash. --- No FK to catalog tables (cross-database FKs are not possible). --- Application-level integrity ensures table_id validity. -CREATE TABLE IF NOT EXISTS stream_shards ( - shard_id TEXT PRIMARY KEY, - table_id TEXT NOT NULL, - parent_shard_id TEXT, - starting_sequence_number TEXT NOT NULL, - ending_sequence_number TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_stream_shards_table - ON stream_shards (table_id); - --- Stream records — change data capture records. -CREATE TABLE IF NOT EXISTS stream_records ( - shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, - sequence_number TEXT NOT NULL, - table_id TEXT NOT NULL, - event_name TEXT NOT NULL, - record_data JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (shard_id, sequence_number) -); - -CREATE INDEX IF NOT EXISTS idx_stream_records_created - ON stream_records (created_at); - --- Monotonic sequence for stream record ordering (CB-21). --- Note: the idempotency check in run_data_migrations uses stream_shards --- existence to decide whether to skip this entire migration. If stream_shards --- is created manually without the sequence, stream_seq will not exist. -CREATE SEQUENCE IF NOT EXISTS stream_seq START 1; -SELECT setval('stream_seq', GREATEST( - (EXTRACT(EPOCH FROM now()) * 1000000)::BIGINT, - 1 -)); - --- Idempotency token storage for TransactWriteItems. -CREATE TABLE IF NOT EXISTS idempotency_tokens ( - token TEXT PRIMARY KEY, - fingerprint TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created - ON idempotency_tokens (created_at); - -COMMIT; diff --git a/crates/storage-postgres/data_migrations/002_gsi_pending.sql b/crates/storage-postgres/data_migrations/002_gsi_pending.sql deleted file mode 100644 index c798da3c1..000000000 --- a/crates/storage-postgres/data_migrations/002_gsi_pending.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Persistent queue for async GSI propagation. --- --- Rows are inserted atomically within the base write transaction and consumed --- by background workers in a single claim+apply+delete transaction, so an --- uncommitted claim rolls back and the row is reprocessed after a crash. --- --- Each row is self-describing: `index_context` carries the base key schema, --- attribute definitions, and the target index definitions captured at enqueue --- time, so workers apply updates with zero catalog reads. A GSI's key schema --- and projection are immutable after creation, so the snapshot can never be --- stale relative to the live index definition. --- --- `worker_partition` is a stable hash of the base table key. All updates to a --- given base item share a partition and are consumed by a single worker in `id` --- order, preserving per-key FIFO ordering (so successive updates to the same --- item never race in the index and converge to the latest state). - -CREATE TABLE IF NOT EXISTS gsi_pending ( - id BIGSERIAL PRIMARY KEY, - table_id TEXT NOT NULL, - worker_partition INTEGER NOT NULL, - old_item JSONB, - new_item JSONB, - index_context JSONB NOT NULL, - ready_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Serves the worker claim: WHERE worker_partition = $1 AND ready_at <= NOW() --- ORDER BY id. The leading partition column also keeps each worker's scan --- confined to its own rows. -CREATE INDEX IF NOT EXISTS idx_gsi_pending_claim - ON gsi_pending (worker_partition, ready_at, id); diff --git a/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql b/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql deleted file mode 100644 index d20ed56f6..000000000 --- a/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql +++ /dev/null @@ -1,33 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Scope idempotency tokens per account. --- --- A ClientRequestToken is unique per account in Amazon DynamoDB, not globally. --- The original table keyed on `token` alone, so tokens from different accounts --- shared one keyspace: an identical token could be treated as a replay of, or a --- mismatch against, an unrelated account's transaction. Re-key on --- (account_id, token) to match every other account-scoped table. --- --- Tokens are an ephemeral dedup cache with a ~10 minute TTL, so the table is --- recreated rather than back-filled: any in-flight token simply re-registers --- under its account on the next request. A client retry that straddles this --- migration loses dedup and may re-apply a non-idempotent transaction, which is --- acceptable for a short-lived cache. Apply data migrations during the --- stop / migrate / restart upgrade sequence to bound that window. - -BEGIN; - -DROP TABLE IF EXISTS idempotency_tokens; - -CREATE TABLE idempotency_tokens ( - account_id TEXT NOT NULL, - token TEXT NOT NULL, - fingerprint TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (account_id, token) -); - -CREATE INDEX idx_idempotency_tokens_created - ON idempotency_tokens (created_at); - -COMMIT; diff --git a/crates/storage-postgres/data_migrations/004_vector_index_state.sql b/crates/storage-postgres/data_migrations/004_vector_index_state.sql deleted file mode 100644 index 52aa07061..000000000 --- a/crates/storage-postgres/data_migrations/004_vector_index_state.sql +++ /dev/null @@ -1,40 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Vector index build state that the propagation queue has to read. --- --- The queue must not apply a write to an index whose backfill is still running: --- the backfill holds an older snapshot of the same item, so applying the newer --- write first lets the backfill overwrite it, and the backfill's deliberately --- plain INSERT would collide with the row the write left behind. --- --- The SQLite backend answers this by joining its claim query against the catalog's --- vector_indexes table. That is impossible here, because the catalog is a separate --- database from the data one and a claim transaction cannot span them. So the --- claim-time fact lives in the data database as its own row. --- --- Ordering rules, which are what make it safe rather than merely present: --- * the hold row is inserted BEFORE the catalog's CREATING row commits, so no --- writer can enqueue against an index the queue does not yet know to hold; --- * the hold row is deleted AFTER the catalog flips the index to ACTIVE, so the --- queue never resumes against an index that is not yet published. --- Held slightly too long is harmless: the rows wait. Released early is not. --- --- Per TABLE, not per index, matching the shared lifecycle contract: a secondary --- index row and a vector row for the same item must keep their relative order, so --- the hold stops the table's queue rather than one index's. - -BEGIN; - -CREATE TABLE IF NOT EXISTS vector_index_holds ( - table_id TEXT NOT NULL, - index_id TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (table_id, index_id) -); - --- The claim query filters on table_id, so that is what needs to be fast. A --- crashed build leaves an orphan row, which the reconciler sweeps by age. -CREATE INDEX IF NOT EXISTS idx_vector_index_holds_table - ON vector_index_holds (table_id); - -COMMIT; diff --git a/crates/storage-postgres/migrations/0001_baseline.expected.json b/crates/storage-postgres/migrations/0001_baseline.expected.json new file mode 100644 index 000000000..c509ca207 --- /dev/null +++ b/crates/storage-postgres/migrations/0001_baseline.expected.json @@ -0,0 +1,1284 @@ +{ + "columns": { + "access_keys.access_key_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "access_keys.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "access_keys.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "access_keys.is_active": { + "type": "boolean", + "not_null": true, + "default": "true", + "collation": null + }, + "access_keys.secret_key_encrypted": { + "type": "bytea", + "not_null": true, + "default": null, + "collation": null + }, + "access_keys.user_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "accounts.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "accounts.account_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "accounts.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "admin_users.admin_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "admin_users.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "admin_users.password_hash": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backup_definitions.backup_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backup_definitions.definition": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "backup_items.backup_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backup_items.item_data": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "backup_items.pk": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backup_items.sk": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "backups.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backups.attribute_definitions": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "backups.backup_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backups.backup_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backups.backup_size_bytes": { + "type": "bigint", + "not_null": true, + "default": "0", + "collation": null + }, + "backups.backup_status": { + "type": "text", + "not_null": true, + "default": "'AVAILABLE'::text", + "collation": null + }, + "backups.backup_type": { + "type": "text", + "not_null": true, + "default": "'USER'::text", + "collation": null + }, + "backups.billing_mode": { + "type": "text", + "not_null": true, + "default": "'PAY_PER_REQUEST'::text", + "collation": null + }, + "backups.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "backups.item_count": { + "type": "bigint", + "not_null": true, + "default": "0", + "collation": null + }, + "backups.key_schema": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "backups.provisioned_throughput": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "backups.stream_specification": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "backups.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backups.table_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "backups.vector_indexes": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "continuous_backups.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "continuous_backups.earliest_restorable": { + "type": "timestamp with time zone", + "not_null": false, + "default": null, + "collation": null + }, + "continuous_backups.latest_restorable": { + "type": "timestamp with time zone", + "not_null": false, + "default": null, + "collation": null + }, + "continuous_backups.pitr_enabled": { + "type": "boolean", + "not_null": true, + "default": "false", + "collation": null + }, + "continuous_backups.table_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_group_members.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_group_members.group_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_group_members.user_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_groups.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_groups.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "iam_groups.group_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_groups.group_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_permissions_boundaries.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_permissions_boundaries.policy_document": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "iam_permissions_boundaries.principal_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_permissions_boundaries.principal_type": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_policies.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_policies.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "iam_policies.policy_document": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "iam_policies.policy_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_policies.principal_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_policies.principal_type": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_role_tags.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_role_tags.role_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_role_tags.tag_key": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_role_tags.tag_value": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_roles.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_roles.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "iam_roles.permissions_boundary_arn": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "iam_roles.role_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_roles.role_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_roles.trust_policy": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.access_key_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "iam_sessions.expires_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.role_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.secret_key_encrypted": { + "type": "bytea", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.session_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_sessions.session_policy": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "iam_sessions.session_tags": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "iam_sessions.session_token": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_user_tags.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_user_tags.tag_key": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_user_tags.tag_value": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_user_tags.user_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_users.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_users.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "iam_users.password_hash": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "iam_users.user_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "iam_users.user_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "idempotency_tokens.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "idempotency_tokens.fingerprint": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "idempotency_tokens.token": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "indexes.index_id": { + "type": "text", + "not_null": true, + "default": "(gen_random_uuid())::text", + "collation": null + }, + "indexes.index_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "indexes.index_status": { + "type": "text", + "not_null": true, + "default": "'ACTIVE'::text", + "collation": null + }, + "indexes.index_type": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "indexes.key_schema": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "indexes.projection": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "indexes.propagation_delay_ms": { + "type": "integer", + "not_null": false, + "default": null, + "collation": null + }, + "indexes.provisioned_throughput": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "indexes.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "login_attempts.attempted_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "login_attempts.principal": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "login_attempts.source_ip": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "login_attempts.success": { + "type": "boolean", + "not_null": true, + "default": null, + "collation": null + }, + "metrics.bucket": { + "type": "timestamp with time zone", + "not_null": true, + "default": null, + "collation": null + }, + "metrics.count": { + "type": "bigint", + "not_null": true, + "default": "0", + "collation": null + }, + "metrics.index_name": { + "type": "text", + "not_null": true, + "default": "''::text", + "collation": null + }, + "metrics.max": { + "type": "double precision", + "not_null": true, + "default": "'-Infinity'::double precision", + "collation": null + }, + "metrics.metric": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "metrics.min": { + "type": "double precision", + "not_null": true, + "default": "'Infinity'::double precision", + "collation": null + }, + "metrics.operation": { + "type": "text", + "not_null": true, + "default": "''::text", + "collation": null + }, + "metrics.sum": { + "type": "double precision", + "not_null": true, + "default": "0", + "collation": null + }, + "metrics.table_name": { + "type": "text", + "not_null": true, + "default": "''::text", + "collation": null + }, + "schema_history.applied_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "schema_history.filename": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "settings.key": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "settings.value": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "stream_records.event_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.record_data": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.sequence_number": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.shard_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_records.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.created_at": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "stream_shards.ending_sequence_number": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "stream_shards.parent_shard_id": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "stream_shards.shard_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.starting_sequence_number": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "stream_shards.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "table_restores.restore_date_time": { + "type": "timestamp with time zone", + "not_null": true, + "default": null, + "collation": null + }, + "table_restores.source_backup_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "table_restores.source_table_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "table_restores.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tables.account_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tables.attribute_definitions": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "tables.billing_mode": { + "type": "text", + "not_null": true, + "default": "'PAY_PER_REQUEST'::text", + "collation": null + }, + "tables.creation_date_time": { + "type": "timestamp with time zone", + "not_null": true, + "default": "now()", + "collation": null + }, + "tables.deletion_protection_enabled": { + "type": "boolean", + "not_null": true, + "default": "false", + "collation": null + }, + "tables.item_count": { + "type": "bigint", + "not_null": true, + "default": "0", + "collation": null + }, + "tables.key_schema": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "tables.on_demand_throughput": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "tables.provisioned_throughput": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "tables.sse_specification": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "tables.status_transition_at": { + "type": "timestamp with time zone", + "not_null": false, + "default": null, + "collation": null + }, + "tables.stream_label": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "tables.stream_specification": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "tables.table_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tables.table_class": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "tables.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tables.table_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tables.table_size_bytes": { + "type": "bigint", + "not_null": true, + "default": "0", + "collation": null + }, + "tables.table_status": { + "type": "text", + "not_null": true, + "default": "'CREATING'::text", + "collation": null + }, + "tables.ttl_attribute": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "tables.ttl_index_ready": { + "type": "boolean", + "not_null": true, + "default": "false", + "collation": null + }, + "tags.resource_arn": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tags.tag_key": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "tags.tag_value": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.backfilling": { + "type": "boolean", + "not_null": false, + "default": null, + "collation": null + }, + "vector_indexes.build_heartbeat_at": { + "type": "timestamp with time zone", + "not_null": false, + "default": null, + "collation": null + }, + "vector_indexes.build_owner": { + "type": "text", + "not_null": false, + "default": null, + "collation": null + }, + "vector_indexes.dimensions": { + "type": "integer", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.distance_function": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.index_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.index_name": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.index_status": { + "type": "text", + "not_null": true, + "default": "'CREATING'::text", + "collation": null + }, + "vector_indexes.projection": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.search_schema": { + "type": "jsonb", + "not_null": false, + "default": null, + "collation": null + }, + "vector_indexes.skipped_item_count": { + "type": "bigint", + "not_null": false, + "default": null, + "collation": null + }, + "vector_indexes.table_id": { + "type": "text", + "not_null": true, + "default": null, + "collation": null + }, + "vector_indexes.vector_attribute": { + "type": "jsonb", + "not_null": true, + "default": null, + "collation": null + } + }, + "indexes": { + "access_keys_pkey": { + "table": "access_keys", + "definition": "CREATE UNIQUE INDEX access_keys_pkey ON public.access_keys USING btree (access_key_id)", + "valid": true + }, + "accounts_account_name_key": { + "table": "accounts", + "definition": "CREATE UNIQUE INDEX accounts_account_name_key ON public.accounts USING btree (account_name)", + "valid": true + }, + "accounts_pkey": { + "table": "accounts", + "definition": "CREATE UNIQUE INDEX accounts_pkey ON public.accounts USING btree (account_id)", + "valid": true + }, + "admin_users_pkey": { + "table": "admin_users", + "definition": "CREATE UNIQUE INDEX admin_users_pkey ON public.admin_users USING btree (admin_name)", + "valid": true + }, + "backup_definitions_pkey": { + "table": "backup_definitions", + "definition": "CREATE UNIQUE INDEX backup_definitions_pkey ON public.backup_definitions USING btree (backup_arn)", + "valid": true + }, + "backups_pkey": { + "table": "backups", + "definition": "CREATE UNIQUE INDEX backups_pkey ON public.backups USING btree (backup_arn)", + "valid": true + }, + "continuous_backups_pkey": { + "table": "continuous_backups", + "definition": "CREATE UNIQUE INDEX continuous_backups_pkey ON public.continuous_backups USING btree (account_id, table_name)", + "valid": true + }, + "iam_group_members_pkey": { + "table": "iam_group_members", + "definition": "CREATE UNIQUE INDEX iam_group_members_pkey ON public.iam_group_members USING btree (account_id, group_name, user_name)", + "valid": true + }, + "iam_groups_group_arn_key": { + "table": "iam_groups", + "definition": "CREATE UNIQUE INDEX iam_groups_group_arn_key ON public.iam_groups USING btree (group_arn)", + "valid": true + }, + "iam_groups_pkey": { + "table": "iam_groups", + "definition": "CREATE UNIQUE INDEX iam_groups_pkey ON public.iam_groups USING btree (account_id, group_name)", + "valid": true + }, + "iam_permissions_boundaries_pkey": { + "table": "iam_permissions_boundaries", + "definition": "CREATE UNIQUE INDEX iam_permissions_boundaries_pkey ON public.iam_permissions_boundaries USING btree (account_id, principal_type, principal_name)", + "valid": true + }, + "iam_policies_pkey": { + "table": "iam_policies", + "definition": "CREATE UNIQUE INDEX iam_policies_pkey ON public.iam_policies USING btree (account_id, principal_type, principal_name, policy_name)", + "valid": true + }, + "iam_role_tags_pkey": { + "table": "iam_role_tags", + "definition": "CREATE UNIQUE INDEX iam_role_tags_pkey ON public.iam_role_tags USING btree (account_id, role_name, tag_key)", + "valid": true + }, + "iam_roles_pkey": { + "table": "iam_roles", + "definition": "CREATE UNIQUE INDEX iam_roles_pkey ON public.iam_roles USING btree (account_id, role_name)", + "valid": true + }, + "iam_roles_role_arn_key": { + "table": "iam_roles", + "definition": "CREATE UNIQUE INDEX iam_roles_role_arn_key ON public.iam_roles USING btree (role_arn)", + "valid": true + }, + "iam_sessions_access_key_id_key": { + "table": "iam_sessions", + "definition": "CREATE UNIQUE INDEX iam_sessions_access_key_id_key ON public.iam_sessions USING btree (access_key_id)", + "valid": true + }, + "iam_sessions_pkey": { + "table": "iam_sessions", + "definition": "CREATE UNIQUE INDEX iam_sessions_pkey ON public.iam_sessions USING btree (session_token)", + "valid": true + }, + "iam_user_tags_pkey": { + "table": "iam_user_tags", + "definition": "CREATE UNIQUE INDEX iam_user_tags_pkey ON public.iam_user_tags USING btree (account_id, user_name, tag_key)", + "valid": true + }, + "iam_users_pkey": { + "table": "iam_users", + "definition": "CREATE UNIQUE INDEX iam_users_pkey ON public.iam_users USING btree (account_id, user_name)", + "valid": true + }, + "iam_users_user_arn_key": { + "table": "iam_users", + "definition": "CREATE UNIQUE INDEX iam_users_user_arn_key ON public.iam_users USING btree (user_arn)", + "valid": true + }, + "idempotency_tokens_pkey": { + "table": "idempotency_tokens", + "definition": "CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (token)", + "valid": true + }, + "idx_backup_items_arn": { + "table": "backup_items", + "definition": "CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn)", + "valid": true + }, + "idx_backups_table": { + "table": "backups", + "definition": "CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name)", + "valid": true + }, + "idx_idempotency_tokens_created": { + "table": "idempotency_tokens", + "definition": "CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at)", + "valid": true + }, + "idx_login_attempts_principal_time": { + "table": "login_attempts", + "definition": "CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC)", + "valid": true + }, + "idx_login_attempts_source_ip_time": { + "table": "login_attempts", + "definition": "CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL)", + "valid": true + }, + "idx_metrics_bucket": { + "table": "metrics", + "definition": "CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket)", + "valid": true + }, + "idx_stream_records_created": { + "table": "stream_records", + "definition": "CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at)", + "valid": true + }, + "idx_stream_shards_table": { + "table": "stream_shards", + "definition": "CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id)", + "valid": true + }, + "idx_table_restores_backup": { + "table": "table_restores", + "definition": "CREATE INDEX idx_table_restores_backup ON public.table_restores USING btree (source_backup_arn)", + "valid": true + }, + "idx_tables_pending_transition": { + "table": "tables", + "definition": "CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL)", + "valid": true + }, + "idx_vector_indexes_index_id": { + "table": "vector_indexes", + "definition": "CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id)", + "valid": true + }, + "indexes_pkey": { + "table": "indexes", + "definition": "CREATE UNIQUE INDEX indexes_pkey ON public.indexes USING btree (table_id, index_name)", + "valid": true + }, + "metrics_pkey": { + "table": "metrics", + "definition": "CREATE UNIQUE INDEX metrics_pkey ON public.metrics USING btree (bucket, metric, table_name, index_name, operation)", + "valid": true + }, + "schema_history_pkey": { + "table": "schema_history", + "definition": "CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename)", + "valid": true + }, + "settings_pkey": { + "table": "settings", + "definition": "CREATE UNIQUE INDEX settings_pkey ON public.settings USING btree (key)", + "valid": true + }, + "stream_records_pkey": { + "table": "stream_records", + "definition": "CREATE UNIQUE INDEX stream_records_pkey ON public.stream_records USING btree (shard_id, sequence_number)", + "valid": true + }, + "stream_shards_pkey": { + "table": "stream_shards", + "definition": "CREATE UNIQUE INDEX stream_shards_pkey ON public.stream_shards USING btree (shard_id)", + "valid": true + }, + "table_restores_pkey": { + "table": "table_restores", + "definition": "CREATE UNIQUE INDEX table_restores_pkey ON public.table_restores USING btree (table_id)", + "valid": true + }, + "tables_pkey": { + "table": "tables", + "definition": "CREATE UNIQUE INDEX tables_pkey ON public.tables USING btree (account_id, table_name)", + "valid": true + }, + "tables_table_id_unique": { + "table": "tables", + "definition": "CREATE UNIQUE INDEX tables_table_id_unique ON public.tables USING btree (table_id)", + "valid": true + }, + "tags_pkey": { + "table": "tags", + "definition": "CREATE UNIQUE INDEX tags_pkey ON public.tags USING btree (resource_arn, tag_key)", + "valid": true + }, + "vector_indexes_pkey": { + "table": "vector_indexes", + "definition": "CREATE UNIQUE INDEX vector_indexes_pkey ON public.vector_indexes USING btree (table_id, index_name)", + "valid": true + } + }, + "constraints": { + "access_keys.access_keys_account_id_user_name_fkey": "FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE", + "access_keys.access_keys_pkey": "PRIMARY KEY (access_key_id)", + "accounts.accounts_account_name_key": "UNIQUE (account_name)", + "accounts.accounts_pkey": "PRIMARY KEY (account_id)", + "admin_users.admin_users_pkey": "PRIMARY KEY (admin_name)", + "backup_definitions.backup_definitions_backup_arn_fkey": "FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE", + "backup_definitions.backup_definitions_pkey": "PRIMARY KEY (backup_arn)", + "backup_items.backup_items_backup_arn_fkey": "FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE", + "backups.backups_pkey": "PRIMARY KEY (backup_arn)", + "continuous_backups.continuous_backups_pkey": "PRIMARY KEY (account_id, table_name)", + "iam_group_members.iam_group_members_account_id_group_name_fkey": "FOREIGN KEY (account_id, group_name) REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE", + "iam_group_members.iam_group_members_account_id_user_name_fkey": "FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE", + "iam_group_members.iam_group_members_pkey": "PRIMARY KEY (account_id, group_name, user_name)", + "iam_groups.iam_groups_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "iam_groups.iam_groups_group_arn_key": "UNIQUE (group_arn)", + "iam_groups.iam_groups_pkey": "PRIMARY KEY (account_id, group_name)", + "iam_permissions_boundaries.iam_permissions_boundaries_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "iam_permissions_boundaries.iam_permissions_boundaries_pkey": "PRIMARY KEY (account_id, principal_type, principal_name)", + "iam_permissions_boundaries.iam_permissions_boundaries_principal_type_check": "CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text])))", + "iam_policies.iam_policies_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "iam_policies.iam_policies_pkey": "PRIMARY KEY (account_id, principal_type, principal_name, policy_name)", + "iam_policies.iam_policies_principal_type_check": "CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text])))", + "iam_role_tags.iam_role_tags_account_id_role_name_fkey": "FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE", + "iam_role_tags.iam_role_tags_pkey": "PRIMARY KEY (account_id, role_name, tag_key)", + "iam_roles.iam_roles_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "iam_roles.iam_roles_pkey": "PRIMARY KEY (account_id, role_name)", + "iam_roles.iam_roles_role_arn_key": "UNIQUE (role_arn)", + "iam_sessions.iam_sessions_access_key_id_key": "UNIQUE (access_key_id)", + "iam_sessions.iam_sessions_account_id_role_name_fkey": "FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE", + "iam_sessions.iam_sessions_pkey": "PRIMARY KEY (session_token)", + "iam_user_tags.iam_user_tags_account_id_user_name_fkey": "FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE", + "iam_user_tags.iam_user_tags_pkey": "PRIMARY KEY (account_id, user_name, tag_key)", + "iam_users.iam_users_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "iam_users.iam_users_pkey": "PRIMARY KEY (account_id, user_name)", + "iam_users.iam_users_user_arn_key": "UNIQUE (user_arn)", + "idempotency_tokens.idempotency_tokens_pkey": "PRIMARY KEY (token)", + "indexes.chk_propagation_delay_ms_non_negative": "CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0)))", + "indexes.indexes_pkey": "PRIMARY KEY (table_id, index_name)", + "indexes.indexes_table_id_fkey": "FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE", + "metrics.metrics_pkey": "PRIMARY KEY (bucket, metric, table_name, index_name, operation)", + "schema_history.schema_history_pkey": "PRIMARY KEY (filename)", + "settings.settings_pkey": "PRIMARY KEY (key)", + "stream_records.stream_records_pkey": "PRIMARY KEY (shard_id, sequence_number)", + "stream_records.stream_records_shard_id_fkey": "FOREIGN KEY (shard_id) REFERENCES stream_shards(shard_id) ON DELETE CASCADE", + "stream_shards.stream_shards_pkey": "PRIMARY KEY (shard_id)", + "stream_shards.stream_shards_table_id_fkey": "FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE", + "table_restores.table_restores_pkey": "PRIMARY KEY (table_id)", + "table_restores.table_restores_table_id_fkey": "FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE", + "tables.tables_account_id_fkey": "FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE", + "tables.tables_pkey": "PRIMARY KEY (account_id, table_name)", + "tables.tables_table_id_unique": "UNIQUE (table_id)", + "tags.tags_pkey": "PRIMARY KEY (resource_arn, tag_key)", + "vector_indexes.chk_vector_active_has_no_backfilling": "CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL)))", + "vector_indexes.chk_vector_dimensions_positive": "CHECK ((dimensions > 0))", + "vector_indexes.vector_indexes_pkey": "PRIMARY KEY (table_id, index_name)", + "vector_indexes.vector_indexes_table_id_fkey": "FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE" + }, + "sequences": [ + "stream_seq" + ] +} diff --git a/crates/storage-postgres/migrations/001_schema.sql b/crates/storage-postgres/migrations/0001_baseline.sql similarity index 60% rename from crates/storage-postgres/migrations/001_schema.sql rename to crates/storage-postgres/migrations/0001_baseline.sql index 3203651f4..512329014 100644 --- a/crates/storage-postgres/migrations/001_schema.sql +++ b/crates/storage-postgres/migrations/0001_baseline.sql @@ -1,11 +1,26 @@ -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 --- Consolidated catalog schema for extenddb (catalog version 0.0.2). --- This is the complete schema applied on fresh installs. - -BEGIN; - --- Accounts — multi-account support (REQ-AUTH-005). +-- +-- Migration 1, baseline, PostgreSQL catalog database. +-- +-- Creates the v1.0 catalog schema on an empty database and brings a catalog +-- created by any released version to the same schema. It is the legacy +-- catalog files 001, 002 and 003 in order, with every statement safe to run +-- on a catalog that already has some or all of their objects: +-- +-- * every CREATE uses IF NOT EXISTS, and the index names are 001's own, so a +-- release that replays 001 against this catalog stops at its first index +-- before it changes anything; +-- * the three `tables` columns added to 001 after it shipped are added with +-- ADD COLUMN IF NOT EXISTS, for catalogs created by v0.1.0 and v0.1.1; +-- * the shape steps earlier builds ran from `migrate` and `serve` are here; +-- * the stream sequence is set forward only, never below its current value. +-- +-- No settings row is seeded: every one has an equal default in code. The +-- runner writes the version and runs this file and its ledger row in one +-- transaction, so the file has no transaction statements of its own. + +-- Accounts. CREATE TABLE IF NOT EXISTS accounts ( account_id TEXT PRIMARY KEY, account_name TEXT NOT NULL UNIQUE, @@ -39,7 +54,13 @@ CREATE TABLE IF NOT EXISTS tables ( CONSTRAINT tables_table_id_unique UNIQUE (table_id) ); -CREATE INDEX idx_tables_pending_transition +-- Catalogs created by v0.1.0 and v0.1.1 lack these three columns, which +-- CreateTable and DescribeTable name. +ALTER TABLE tables ADD COLUMN IF NOT EXISTS table_class TEXT; +ALTER TABLE tables ADD COLUMN IF NOT EXISTS sse_specification JSONB; +ALTER TABLE tables ADD COLUMN IF NOT EXISTS on_demand_throughput JSONB; + +CREATE INDEX IF NOT EXISTS idx_tables_pending_transition ON tables (status_transition_at) WHERE status_transition_at IS NOT NULL; @@ -69,19 +90,21 @@ CREATE TABLE IF NOT EXISTS tags ( PRIMARY KEY (resource_arn, tag_key) ); --- Migration tracking. +-- The ledger of releases before 1.0, kept so their `migrate` finds every +-- file it knows recorded and runs none of them. CREATE TABLE IF NOT EXISTS schema_history ( filename TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); --- Settings (catalog version, data database connection, runtime config). +-- Settings (version, data database connection, runtime config). CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); --- Stream shards. +-- Stream shards. Unused in the catalog database: streams live in the data +-- database. Kept because every released catalog has them. CREATE TABLE IF NOT EXISTS stream_shards ( shard_id TEXT PRIMARY KEY, table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, @@ -91,9 +114,9 @@ CREATE TABLE IF NOT EXISTS stream_shards ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX IF NOT EXISTS idx_stream_shards_table ON stream_shards (table_id); --- Stream records. +-- Stream records. Unused in the catalog database, as stream_shards. CREATE TABLE IF NOT EXISTS stream_records ( shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, sequence_number TEXT NOT NULL, @@ -104,7 +127,7 @@ CREATE TABLE IF NOT EXISTS stream_records ( PRIMARY KEY (shard_id, sequence_number) ); -CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX IF NOT EXISTS idx_stream_records_created ON stream_records (created_at); -- Admin users. CREATE TABLE IF NOT EXISTS admin_users ( @@ -219,17 +242,16 @@ CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( PRIMARY KEY (account_id, principal_type, principal_name) ); --- Idempotency tokens for TransactWriteItems. --- Vestigial in the catalog database: at runtime tokens are read and written via --- the data database (see data_migrations 001 + 003, keyed on --- (account_id, token)). This catalog copy is unused; do not wire it up. +-- Idempotency tokens. Unused in the catalog database: the data database +-- holds them, keyed on (account_id, token). Kept because every released +-- catalog has it. CREATE TABLE IF NOT EXISTS idempotency_tokens ( token TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created ON idempotency_tokens (created_at); -- Metrics (1-minute aggregation). CREATE TABLE IF NOT EXISTS metrics ( @@ -245,7 +267,7 @@ CREATE TABLE IF NOT EXISTS metrics ( PRIMARY KEY (bucket, metric, table_name, index_name, operation) ); -CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX IF NOT EXISTS idx_metrics_bucket ON metrics (bucket); -- Login attempt tracking. CREATE TABLE IF NOT EXISTS login_attempts ( @@ -255,10 +277,10 @@ CREATE TABLE IF NOT EXISTS login_attempts ( source_ip TEXT ); -CREATE INDEX idx_login_attempts_principal_time +CREATE INDEX IF NOT EXISTS idx_login_attempts_principal_time ON login_attempts (principal, attempted_at DESC); -CREATE INDEX idx_login_attempts_source_ip_time +CREATE INDEX IF NOT EXISTS idx_login_attempts_source_ip_time ON login_attempts (source_ip, attempted_at DESC) WHERE source_ip IS NOT NULL; @@ -281,7 +303,7 @@ CREATE TABLE IF NOT EXISTS backups ( created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX IF NOT EXISTS idx_backups_table ON backups (account_id, table_name); -- Backup items. CREATE TABLE IF NOT EXISTS backup_items ( @@ -291,7 +313,7 @@ CREATE TABLE IF NOT EXISTS backup_items ( item_data JSONB NOT NULL ); -CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX IF NOT EXISTS idx_backup_items_arn ON backup_items (backup_arn); -- Continuous backups / PITR status. CREATE TABLE IF NOT EXISTS continuous_backups ( @@ -303,21 +325,84 @@ CREATE TABLE IF NOT EXISTS continuous_backups ( PRIMARY KEY (account_id, table_name) ); --- Stream sequence (monotonic, starts at current epoch microseconds). +-- Stream sequence: starts at the current epoch microseconds, and a run on a +-- catalog that has one never moves it back. CREATE SEQUENCE IF NOT EXISTS stream_seq START 1; SELECT setval('stream_seq', GREATEST( (EXTRACT(EPOCH FROM now()) * 1000000)::BIGINT, + (SELECT last_value FROM stream_seq), 1 )); --- Seed settings. -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.2') -ON CONFLICT (key) DO NOTHING; -INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25') -ON CONFLICT (key) DO NOTHING; -INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10') -ON CONFLICT (key) DO NOTHING; -INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied') -ON CONFLICT (key) DO NOTHING; - -COMMIT; +-- Vector index metadata (legacy 002). Kept out of `indexes`: a vector index +-- has no key schema. `search_schema` is nullable because the HASH element is +-- optional. `backfilling` is false while CREATING before the scan starts, +-- true while it runs, and NULL once ACTIVE. `skipped_item_count` counts items +-- a backfill could not index. `build_owner` and `build_heartbeat_at` record +-- which process builds the index. +CREATE TABLE IF NOT EXISTS vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute JSONB NOT NULL, + search_schema JSONB, + projection JSONB NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling BOOLEAN, + skipped_item_count BIGINT, + build_owner TEXT, + build_heartbeat_at TIMESTAMPTZ, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id ON vector_indexes (index_id); + +-- The source table's vector index configuration at backup time; NULL for +-- backups taken before vector indexes existed. +ALTER TABLE backups ADD COLUMN IF NOT EXISTS vector_indexes JSONB; + +-- A backup's table definition (legacy 003): one row per backup, in the +-- wire's shape behind a version marker. A backup without a row restores +-- from its key schema and attribute definitions alone. +CREATE TABLE IF NOT EXISTS backup_definitions ( + backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, + definition JSONB NOT NULL +); + +-- One row per table RestoreTableFromBackup created: the backup and source +-- table it came from, and the restore's RestoreDateTime. Not a foreign key +-- to backups: the summary keeps naming a backup deleted after the restore. +-- restore_date_time has no default: the engine always passes the value. +CREATE TABLE IF NOT EXISTS table_restores ( + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, + source_backup_arn TEXT NOT NULL, + source_table_arn TEXT NOT NULL, + restore_date_time TIMESTAMPTZ NOT NULL +); + +-- A catalog that ran a pre-release revision of legacy 003 has table_restores +-- without source_table_arn (added here as nullable: its rows have no value) +-- and with a default on restore_date_time. +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS source_table_arn TEXT; +ALTER TABLE table_restores ALTER COLUMN restore_date_time DROP DEFAULT; + +CREATE INDEX IF NOT EXISTS idx_table_restores_backup ON table_restores (source_backup_arn); + +-- Releases before 1.0 refuse a catalog without this row and insert it again +-- from their `migrate`; its presence beside a ledger tells v1.0 that one of +-- them wrote to the catalog. +DELETE FROM settings WHERE key = 'resource_tag_migration'; + +-- Every legacy catalog file, recorded in the legacy ledger. +INSERT INTO schema_history (filename) VALUES + ('001_schema.sql'), + ('002_vector_indexes.sql'), + ('003_backup_definitions.sql') +ON CONFLICT (filename) DO NOTHING; diff --git a/crates/storage-postgres/migrations/002_vector_indexes.sql b/crates/storage-postgres/migrations/002_vector_indexes.sql deleted file mode 100644 index c61a32fdd..000000000 --- a/crates/storage-postgres/migrations/002_vector_indexes.sql +++ /dev/null @@ -1,79 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Vector index catalog metadata (catalog version 0.0.3). --- --- Ports the SQLite catalog shape with PostgreSQL types: JSONB where SQLite --- stores JSON text, BOOLEAN where it stores 0/1 (so no check constraint is --- needed for booleanness), BIGINT for the skip counter. - -BEGIN; - --- Vector index metadata. Kept out of `indexes` deliberately: a vector index is --- not described by a key schema, so reusing that table's `key_schema` column --- would mean storing something meaningless in a NOT NULL column. The engine --- supplies index_id, as it does for GSIs. --- --- Every statement here is written to tolerate a replay. The runner applies a --- migration and records it in `schema_history` as two separate commits, so a --- crash in between leaves this file applied but unrecorded, and the next --- migrate would run it again. Without the guards that second run fails on --- "relation already exists" and blocks every later migration, on a deployment --- that is otherwise serving correctly because the version gate is satisfied. --- Recovering from that needs a hand-written ledger row. 001 guards every one of --- its tables the same way. --- --- `search_schema` is nullable because the HASH element is optional (measured --- against the live service): with one the search is partition-scoped and --- SearchConditionExpression is required, without one it spans the table. --- --- `backfilling` mirrors the measured lifecycle: false while CREATING before the --- scan starts, true while it runs, and the member is absent (NULL) once ACTIVE. -CREATE TABLE IF NOT EXISTS vector_indexes ( - table_id TEXT NOT NULL, - index_id TEXT NOT NULL, - index_name TEXT NOT NULL, - dimensions INTEGER NOT NULL, - distance_function TEXT NOT NULL, - vector_attribute JSONB NOT NULL, - search_schema JSONB, - projection JSONB NOT NULL, - index_status TEXT NOT NULL DEFAULT 'CREATING', - backfilling BOOLEAN, - -- Items a backfill skipped because their stored bytes cannot enter the - -- index (unparseable row, malformed or wrong-dimension vector). NULL until - -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so - -- an operator can see that an ACTIVE index deliberately omits rows, rather - -- than the build looping forever on them or dying part-way. - skipped_item_count BIGINT, - -- Build ownership for multi-process deployments. Several front-ends can - -- share one PostgreSQL, so an in-process registry cannot answer "is some - -- process still building this index". The builder records its identity and - -- renews the heartbeat per batch; a stuck-build sweep in any process reads - -- both. Unused until the UpdateTable-create lifecycle lands; NULL until a - -- build claims the row. - build_owner TEXT, - build_heartbeat_at TIMESTAMPTZ, - PRIMARY KEY (table_id, index_name), - CONSTRAINT vector_indexes_table_id_fkey - FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, - CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), - -- An ACTIVE index must not carry the member at all, which is what the - -- service does. Enforced here as well as in core, so a bug in the backend - -- cannot persist a state the wire contract forbids. - CONSTRAINT chk_vector_active_has_no_backfilling - CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id ON vector_indexes (index_id); - --- Snapshot of the source table's vector index configuration at backup time, --- the same way `key_schema` and `attribute_definitions` are snapshotted. NULL --- for backups taken before this migration, which cannot have carried vector --- indexes because the backend could not create them. Restore refuses a backup --- whose snapshot is non-empty rather than silently dropping a declared index; --- the snapshot also carries what a future vector-preserving restore needs. -ALTER TABLE backups ADD COLUMN IF NOT EXISTS vector_indexes JSONB; - -UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'; - -COMMIT; diff --git a/crates/storage-postgres/migrations/003_backup_definitions.sql b/crates/storage-postgres/migrations/003_backup_definitions.sql deleted file mode 100644 index 0bafff09e..000000000 --- a/crates/storage-postgres/migrations/003_backup_definitions.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Migration 003: record a backup's table definition, and which backup a --- restored table came from (catalog version 0.0.4). --- --- A backup kept the source table's key schema, attribute definitions, and --- billing mode, and nothing else, so a restored table came back without its --- global and local secondary indexes, with 5/5 provisioned throughput, and --- without its table class or encryption settings. One row per backup holds --- those, in the wire's own shape behind a version marker (see --- `extenddb_storage::backup_definition`). A backup taken before this --- migration has no row and restores as before. --- --- Written to tolerate a replay, like 002: the runner applies a migration and --- records it in `schema_history` as two separate commits, so a crash in --- between leaves this file applied but unrecorded, and the next migrate runs --- it again. CREATE TABLE IF NOT EXISTS and the version UPDATE are idempotent. - -BEGIN; - -CREATE TABLE IF NOT EXISTS backup_definitions ( - backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, - definition JSONB NOT NULL -); - --- One row per table created by RestoreTableFromBackup: the backup it came --- from, the table that backup was taken of, and the restore's --- RestoreDateTime, reported by DescribeTable as RestoreSummary, and used to --- refuse DeleteBackup while the restore is still running. Not a foreign key --- to backups: the backup may be deleted after the restore, and the summary --- keeps naming it, as on the service. restore_date_time has no default: the --- engine passes the value to record (the source backup's creation time, which --- is what the service reports), so a missing value is an error, not "now". -CREATE TABLE IF NOT EXISTS table_restores ( - table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, - source_backup_arn TEXT NOT NULL, - source_table_arn TEXT NOT NULL, - restore_date_time TIMESTAMPTZ NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_table_restores_backup ON table_restores (source_backup_arn); - -UPDATE settings SET value = '0.0.4' WHERE key = 'catalog_version'; - -COMMIT; diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index 5eea7b26d..23461be07 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -7,42 +7,19 @@ //! teardown using PostgreSQL-specific DDL. Connection pools are created //! lazily as needed during the bootstrap sequence. -use std::time::Duration; - use async_trait::async_trait; -use extenddb_core::types::{AttributeDefinition, KeySchemaElement}; use extenddb_storage::bootstrapper::{ - AdminBootstrapResult, BootstrapConfig, Bootstrapper, KeyDefinitionRepair, + AdminBootstrapResult, BootstrapConfig, Bootstrapper, helpers::{ check_conflict, check_conflict_redacted, extract_arg, generate_account_id, generate_encryption_key, generate_random_password, hash_password_async, }, }; use extenddb_storage::management_store::{OpError, OpResult}; -use extenddb_storage::util::recover_sort_key_definitions; +use extenddb_storage::migrations::{AdminRole, MigrationId, Target}; use sqlx::PgPool; use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use tokio::sync::{Mutex, OnceCell}; - -use crate::CATALOG_VERSION; -use crate::migrations; - -/// ExtendDB's advisory-lock namespace. PostgreSQL's two-argument form, -/// `pg_advisory_xact_lock(classid, objid)`, lets ExtendDB reserve one stable -/// `classid` by convention and assign a distinct `objid` to each internal lock. -/// Other applications in the same database must avoid choosing the same keys. -/// -/// The value itself is arbitrary. It only has to stay stable across releases and -/// differ from any other namespace we add later. -const ADVISORY_LOCK_NAMESPACE: i32 = 0x0045_4442; // 'E', 'D', 'B' -/// `objid` for the schema-migration lock, which serializes concurrent `migrate` -/// runs (for example several replicas starting at once). -const MIGRATION_LOCK_OBJID: i32 = 1; - -/// Maximum time to wait for another migrator. Normal contention should clear -/// well within this window; expiry indicates a peer that is likely wedged and -/// needs operator attention rather than an indefinitely stuck init Job. -const MIGRATION_LOCK_TIMEOUT: Duration = Duration::from_secs(5 * 60); +use tokio::sync::OnceCell; /// Utilities for bootstrapping a `PostgreSQL` backend store. /// @@ -53,9 +30,6 @@ const MIGRATION_LOCK_TIMEOUT: Duration = Duration::from_secs(5 * 60); pub struct PostgresBootstrapper { config: BootstrapConfig, admin_pool: OnceCell, - /// Dedicated connection whose open transaction holds the migration advisory - /// lock. `None` when no migration lock is held. - lock_conn: Mutex>, } impl PostgresBootstrapper { @@ -66,7 +40,6 @@ impl PostgresBootstrapper { Self { config, admin_pool: OnceCell::new(), - lock_conn: Mutex::new(None), } } @@ -145,68 +118,16 @@ impl PostgresBootstrapper { pub fn catalog_connection_url(&self) -> String { self.app_connection_url(&self.config.catalog_db) } +} - /// Try to install pgvector on the data database, tolerating refusal. - /// - /// The attempt runs as the admin role first: pgvector's control file does - /// not mark the extension trusted, so a stock server refuses `CREATE - /// EXTENSION` from any non-superuser, even on a database its role owns - /// ("Must be superuser", measured on a stock install). The application role - /// can therefore never succeed there, and the admin credentials this command - /// already holds are exactly the ones that can. The application role is - /// still tried on admin failure, which is the shape managed platforms - /// allowlisting pgvector for the database owner need. Serve-time code never - /// attempts this, because a request path must not carry data-definition - /// privileges it only needs once. - /// - /// Failure is a notice, not an error. A deployment that does not want vector - /// indexes, or a managed PostgreSQL that does not offer pgvector, must still - /// initialise and upgrade normally; the server then refuses vector - /// operations, which is the fail-closed half of the same decision. - async fn try_create_vector_extension(&self, pool: &PgPool) { - use sqlx::Connection as _; - - println!("--- Checking pgvector extension on the data database..."); - let admin_opts = PgConnectOptions::new() - .host(&self.config.host) - .port(self.config.port) - .username(&self.config.admin_user) - .database(&self.config.data_db); - let admin_opts = if let Some(ref pass) = self.config.admin_password { - admin_opts.password(pass) - } else { - admin_opts - }; - let admin_error = match sqlx::PgConnection::connect_with(&admin_opts).await { - Ok(mut conn) => { - let created = sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") - .execute(&mut conn) - .await; - let _ = conn.close().await; - match created { - Ok(_) => { - println!(" pgvector available; vector indexes are supported."); - return; - } - Err(e) => e.to_string(), - } - } - Err(e) => e.to_string(), - }; - match sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") - .execute(pool) - .await - { - Ok(_) => println!(" pgvector available; vector indexes are supported."), - Err(e) => { - let hint = crate::vector::create_extension_hint(&e); - println!( - " NOTICE: could not create the pgvector extension (as the admin role: \ - {admin_error}; as the application role: {e}). {hint}." - ); - } - } - } +/// The error of a bootstrapper method the commands do not call on this +/// backend: its catalog migrates through +/// [`crate::migrations::PostgresCatalogMigrations`] and the shared runner. +fn through_the_runner(method: &str) -> OpError { + OpError::Internal(format!( + "postgres migrates through its CatalogMigrations; the bootstrapper method {method} is \ + not used" + )) } #[async_trait] @@ -287,163 +208,15 @@ impl Bootstrapper for PostgresBootstrapper { } async fn run_catalog_migrations(&self) -> OpResult<()> { - let pool = self.app_pool(&self.config.catalog_db).await?; - migrations::run_catalog_migrations(&pool).await + Err(through_the_runner("run_catalog_migrations")) } async fn run_data_migrations(&self) -> OpResult<()> { - // Catalog work the version string does not show (see - // `pending_data_migrations`) is applied first, by the same walk - // `run_catalog_migrations` runs, so `extenddb migrate --yes` applies - // it even when the stored catalog version already matches. - let catalog_pool = self.app_pool(&self.config.catalog_db).await?; - if !pending_catalog_work(&catalog_pool).await?.is_empty() { - migrations::run_catalog_migrations(&catalog_pool).await?; - } - let pool = self.app_pool(&self.config.data_db).await?; - self.try_create_vector_extension(&pool).await; - migrations::run_data_migrations(&pool).await?; - // Programmatic migrations need the catalog pool (to enumerate index - // tables) plus the data pool (where the `_ddb_*` tables live). - migrations::run_data_code_migrations(&catalog_pool, &pool).await - } - - async fn acquire_migration_lock(&self) -> OpResult<()> { - use std::io::Write as _; - - use sqlx::Connection; - - // This is not re-entrant. A second call would open a second transaction - // and block on the lock the first one holds, deadlocking against itself. - let mut held = self.lock_conn.lock().await; - if held.is_some() { - return Err(OpError::Internal( - "Migration lock is already held by this process".to_owned(), - )); - } - - // Keep an explicit transaction open on a dedicated catalog connection - // for the whole migration. Transaction-level advisory locks are released - // automatically when that transaction ends or the connection dies. - // - // The explicit transaction is important for transaction-pooling proxies: - // they must retain one PostgreSQL backend until COMMIT/ROLLBACK, so the - // lock cannot silently move between backends as separate autocommit - // statements can. RDS Proxy also supports this path without session - // pinning. The migration statements can use separate connections because - // advisory locks coordinate globally within the catalog database. - // - // Advisory locks are scoped to a database, so migrators only serialize - // if they share this catalog database. ExtendDB replicas normally do, - // because they use the same catalog connection string. - let mut conn = - sqlx::PgConnection::connect_with(&self.app_connect_opts(&self.config.catalog_db)) - .await - .map_err(|e| { - OpError::Internal(format!("Cannot connect to take migration lock: {e}")) - })?; - sqlx::query("BEGIN") - .execute(&mut conn) - .await - .map_err(|e| OpError::Internal(format!("Cannot begin migration lock: {e}")))?; - - // Equivalent to SET LOCAL lock_timeout, but parameterized so the Rust - // duration remains the single source of truth. This applies only to the - // dedicated transaction and is discarded by rollback on release. - let lock_timeout = format!("{}ms", MIGRATION_LOCK_TIMEOUT.as_millis()); - let _: String = sqlx::query_scalar("SELECT set_config('lock_timeout', $1, true)") - .bind(&lock_timeout) - .fetch_one(&mut conn) - .await - .map_err(|e| { - OpError::Internal(format!("Cannot configure migration lock timeout: {e}")) - })?; - - // Try first so that a migrator which has to wait can say so, rather - // than sitting silent for as long as the other migration takes. - let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1, $2)") - .bind(ADVISORY_LOCK_NAMESPACE) - .bind(MIGRATION_LOCK_OBJID) - .fetch_one(&mut conn) - .await - .map_err(|e| OpError::Internal(format!("Cannot acquire migration lock: {e}")))?; - if !acquired { - println!("--- Another migrator holds the migration lock; waiting for it to finish..."); - std::io::stdout() - .flush() - .map_err(|e| OpError::Internal(format!("Cannot report migration wait: {e}")))?; - sqlx::query("SELECT pg_advisory_xact_lock($1, $2)") - .bind(ADVISORY_LOCK_NAMESPACE) - .bind(MIGRATION_LOCK_OBJID) - .execute(&mut conn) - .await - .map_err(|e| { - if let sqlx::Error::Database(db_err) = &e - && db_err.code().as_deref() == Some("55P03") - { - return OpError::Internal(format!( - "Timed out after {}s waiting for the migration advisory lock; \ - another migrator may be wedged. Check the holder's logs or \ - terminate it before retrying: {e}", - MIGRATION_LOCK_TIMEOUT.as_secs(), - )); - } - OpError::Internal(format!("Cannot acquire migration lock: {e}")) - })?; - println!("--- Migration lock acquired."); - std::io::stdout() - .flush() - .map_err(|e| OpError::Internal(format!("Cannot report migration lock: {e}")))?; - } - - // Guard against a key or lock-mode mismatch in the acquisition queries. - // `objsubid = 2` distinguishes the two-i32 advisory-lock keyspace from - // the one-i64 form, and `granted` excludes a merely waiting request. - let held_by_this_transaction: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM pg_locks \ - WHERE locktype = 'advisory' AND classid = $1 AND objid = $2 \ - AND objsubid = 2 AND mode = 'ExclusiveLock' AND granted \ - AND pid = pg_backend_pid())", - ) - .bind(ADVISORY_LOCK_NAMESPACE) - .bind(MIGRATION_LOCK_OBJID) - .fetch_one(&mut conn) - .await - .map_err(|e| OpError::Internal(format!("Cannot verify migration lock: {e}")))?; - if !held_by_this_transaction { - return Err(OpError::Internal( - "PostgreSQL did not report the migration transaction's advisory lock after \ - acquisition; refusing to run migrations without verified serialization" - .to_owned(), - )); - } - - *held = Some(conn); - Ok(()) - } - - async fn release_migration_lock(&self) -> OpResult<()> { - use sqlx::Connection; - - let Some(mut conn) = self.lock_conn.lock().await.take() else { - return Ok(()); - }; - // This transaction contains only the advisory lock, so rollback is the - // safest release: it cannot accidentally commit future work added to the - // dedicated connection. Closing is a backstop if rollback fails. - let release = sqlx::query("ROLLBACK").execute(&mut conn).await; - let _ = conn.close().await; - release - .map(|_| ()) - .map_err(|e| OpError::Internal(format!("Cannot release migration lock: {e}"))) + Err(through_the_runner("run_data_migrations")) } async fn pending_data_migrations(&self) -> OpResult> { - let catalog_pool = self.app_pool(&self.config.catalog_db).await?; - let mut pending = pending_catalog_work(&catalog_pool).await?; - let pool = self.app_pool(&self.config.data_db).await?; - pending.extend(migrations::pending_data_migrations(&pool).await?); - Ok(pending) + Err(through_the_runner("pending_data_migrations")) } async fn record_data_connection(&self) -> OpResult<()> { @@ -597,149 +370,7 @@ impl Bootstrapper for PostgresBootstrapper { } async fn is_catalog_initialized(&self) -> OpResult { - let pool = self.app_pool(&self.config.catalog_db).await?; - migrations::table_exists(&pool, "settings").await - } - - /// Repair table metadata damaged by the pre-fix `UpdateTable` (#259). - /// - /// Reads the catalog's key schemas and attribute definitions, and for any base - /// sort key with no definition recovers the type from the data table's - /// physical sort key column (`sk_s`/`sk_n`/`sk_b`, or the numbered variants for - /// multiple sort keys). Runs under the migration lock the caller holds. - async fn repair_lost_sort_key_definitions(&self, apply: bool) -> OpResult { - let mut report = KeyDefinitionRepair::default(); - let Ok(catalog) = self.app_pool(&self.config.catalog_db).await else { - return Ok(report); - }; - let Some(data_db) = self.get_data_db_name().await? else { - return Ok(report); - }; - let data = self.app_pool(&data_db).await?; - - let rows: Vec<(String, String, String, serde_json::Value, serde_json::Value)> = - sqlx::query_as( - "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ - FROM tables ORDER BY table_name", - ) - .fetch_all(&catalog) - .await - .map_err(|e| OpError::Internal(format!("Cannot read tables: {e}")))?; - - for (account_id, table_name, table_id, ks_json, ad_json) in rows { - let key_schema: Vec = match serde_json::from_value(ks_json) { - Ok(v) => v, - Err(e) => { - report - .needs_attention - .push(format!("{table_name}: unreadable key_schema ({e})")); - continue; - } - }; - let attr_defs: Vec = - serde_json::from_value(ad_json).unwrap_or_default(); - - // Nothing to do unless a sort key has lost its definition. - // Reported for every table on every run, not only when a sort key is - // repaired: the partition key definition is dropped by the same write and - // is not recoverable from the schema, because the pk column is always TEXT. - // It is not needed for correctness either, since partition key values are - // encoded from the key schema alone, so it is reported rather than guessed - // and keeps being reported until a human restores it. - let pk_missing: Vec<&str> = key_schema - .iter() - .filter(|ks| ks.key_type == extenddb_core::types::KeyType::Hash) - .filter(|ks| { - !attr_defs - .iter() - .any(|ad| ad.attribute_name == ks.attribute_name) - }) - .map(|ks| ks.attribute_name.as_str()) - .collect(); - if !pk_missing.is_empty() { - report.needs_attention.push(format!( - "{table_name}: partition key definition(s) [{}] are absent; reads and \ - writes are unaffected, but index key type validation cannot check them", - pk_missing.join(", ") - )); - } - - let missing: Vec<&KeySchemaElement> = key_schema - .iter() - .filter(|ks| ks.key_type == extenddb_core::types::KeyType::Range) - .filter(|ks| { - !attr_defs - .iter() - .any(|ad| ad.attribute_name == ks.attribute_name) - }) - .collect(); - if missing.is_empty() { - continue; - } - - // PRIMARY KEY columns only, in key order. Every data table carries all - // three typed columns for each sort key position (sk_s, sk_n, sk_b), so - // column existence says nothing about the declared type: only PRIMARY KEY - // membership does. `to_regclass` resolves the name through the current - // search_path, so this cannot match a same-named table in another schema, - // and returns NULL (no rows) rather than raising when the data table is - // absent. - let columns: Vec = sqlx::query_as::<_, (String,)>( - "SELECT a.attname \ - FROM pg_index i \ - JOIN unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON TRUE \ - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum \ - WHERE i.indrelid = to_regclass($1) AND i.indisprimary \ - ORDER BY k.ord", - ) - .bind(format!("\"_ddb_{table_id}\"")) - .fetch_all(&data) - .await - .map_err(|e| OpError::Internal(format!("Cannot read columns of {table_name}: {e}")))? - .into_iter() - .map(|(c,)| c) - .collect(); - - let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); - if recovered.is_empty() { - report.needs_attention.push(format!( - "{table_name}: sort key(s) [{}] have no attribute definition and no \ - matching PRIMARY KEY column was found to recover the type from", - missing - .iter() - .map(|ks| ks.attribute_name.as_str()) - .collect::>() - .join(", ") - )); - continue; - } - - let mut merged = attr_defs; - merged.extend(recovered.iter().cloned()); - let merged_json = serde_json::to_value(&merged) - .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; - if apply { - sqlx::query( - "UPDATE tables SET attribute_definitions = $1 \ - WHERE account_id = $2 AND table_name = $3", - ) - .bind(&merged_json) - .bind(&account_id) - .bind(&table_name) - .execute(&catalog) - .await - .map_err(|e| OpError::Internal(format!("Cannot repair {table_name}: {e}")))?; - } - - for def in &recovered { - report.repaired.push(format!( - "{table_name}: {} ({:?})", - def.attribute_name, def.attribute_type - )); - } - } - - Ok(report) + Err(through_the_runner("is_catalog_initialized")) } async fn list_table_names(&self) -> OpResult> { @@ -790,24 +421,11 @@ impl Bootstrapper for PostgresBootstrapper { } async fn read_catalog_version(&self) -> OpResult> { - let pool = self.app_pool(&self.config.catalog_db).await?; - - if !migrations::table_exists(&pool, "settings").await? { - return Ok(None); - } - - let row = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'catalog_version'", - ) - .fetch_optional(&pool) - .await - .map_err(|e| OpError::Internal(format!("Read catalog version: {e}")))?; - - Ok(row.map(|(v,)| v)) + Err(through_the_runner("read_catalog_version")) } fn expected_catalog_version(&self) -> String { - CATALOG_VERSION.to_string() + MigrationId::LATEST.number().to_string() } fn catalog_database_name(&self) -> String { @@ -822,6 +440,21 @@ impl Bootstrapper for PostgresBootstrapper { self.app_connection_url(&self.config.catalog_db) } + /// The catalog's and the data database's connection strings, the data + /// database's name and the admin role, as this bootstrapper resolved + /// them; migration 1 records the data database's rows. + fn init_target(&self) -> Target { + Target { + location: self.catalog_connection_url(), + data_store: Some(self.config.data_db.clone()), + admin: Some(AdminRole { + user: self.config.admin_user.clone(), + password: self.config.admin_password.clone(), + }), + data_connection: Some(self.app_connection_url(&self.config.data_db)), + } + } + fn generate_backend_config_section(&self) -> String { format!( r#"[storage.postgres] @@ -835,29 +468,6 @@ connection_string = "{}" // ── Helpers ──────────────────────────────────────────────────────────── -/// Catalog work `extenddb migrate` must report and apply even when the -/// stored `catalog_version` already equals this binary's: catalog migration -/// files not recorded in `schema_history` (`catalog `), and catalog -/// shape steps (`catalog: `, see `migrations::CATALOG_SHAPE_STEPS`). -/// -/// The `Bootstrapper` trait has no catalog counterpart of -/// `pending_data_migrations`, so this list is reported through that hook and -/// applied by `run_data_migrations`, which `migrate --yes` always calls. -pub(crate) async fn pending_catalog_work(catalog_pool: &PgPool) -> OpResult> { - let mut pending: Vec = migrations::pending_catalog_migrations(catalog_pool) - .await? - .into_iter() - .map(|f| format!("catalog {f}")) - .collect(); - pending.extend( - migrations::pending_catalog_shapes(catalog_pool) - .await? - .into_iter() - .map(|s| format!("catalog: {s}")), - ); - Ok(pending) -} - /// Create a database, aborting if it already exists. async fn create_database(pool: &PgPool, name: &str, owner: &str) -> OpResult<()> { println!("--- Creating database '{name}'..."); @@ -989,7 +599,8 @@ impl PostgresBootstrapper { | OpError::AlreadyExists(m) | OpError::NotFound(m) | OpError::HasDependents(m) - | OpError::Internal(m) => m, + | OpError::Internal(m) + | OpError::Database { message: m, .. } => m, }; let mut store = Self::connect(config) .await diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index fc41c8c0b..daabe324e 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -191,7 +191,7 @@ impl PostgresEngine { let inserted: Result<(f64, String), sqlx::Error> = sqlx::query_as( r"WITH delay AS ( SELECT COALESCE( - (SELECT value::FLOAT8 FROM settings WHERE key = 'control_plane_delay_seconds'), 0.25 + (SELECT value::FLOAT8 FROM settings WHERE key = 'control_plane_delay_seconds'), $15 ) AS secs ) INSERT INTO tables @@ -226,6 +226,7 @@ impl PostgresEngine { .bind(&sse_spec_json) .bind(&on_demand_json) .bind(defer_active) + .bind(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS) .fetch_one(&mut *tx) .await; let (creation_epoch, actual_status): (f64, String) = match inserted { diff --git a/crates/storage-postgres/src/data/ddl.rs b/crates/storage-postgres/src/data/ddl.rs index 16f1a3172..789447fd4 100755 --- a/crates/storage-postgres/src/data/ddl.rs +++ b/crates/storage-postgres/src/data/ddl.rs @@ -742,7 +742,9 @@ async fn set_lock_timeout_until( } /// SQLSTATE `lock_not_available` (55P03), which `lock_timeout` raises. -const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; +/// The migration runner classifies a failed body by it too, from the code a +/// `MigrationError` keeps (`migrations/failure.rs`). +pub(crate) const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// Whether `e` is a lock wait that ran out its `lock_timeout` (55P03): a /// session held a lock the statement needed, which a later attempt can get diff --git a/crates/storage-postgres/src/data/mod.rs b/crates/storage-postgres/src/data/mod.rs index 7f37ea17d..d8056ea14 100755 --- a/crates/storage-postgres/src/data/mod.rs +++ b/crates/storage-postgres/src/data/mod.rs @@ -146,7 +146,7 @@ mod update_item; pub mod vector_index; pub(crate) use ddl::{ - LockHolders, delete_pending_queue_rows, drop_table_relations, lock_not_available, - lock_wait_failed, storage_in_use, table_storage_relations, + LockHolders, SQLSTATE_LOCK_NOT_AVAILABLE, delete_pending_queue_rows, drop_table_relations, + lock_not_available, lock_wait_failed, storage_in_use, table_storage_relations, }; pub(crate) use index::{insert_index_rows_multi, item_has_index_keys, project_item_for_index}; diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index f03d2c7f0..81b75856b 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -129,8 +129,9 @@ impl PostgresEngine { // before the next request, causing stale DELETING rows. Synchronous // delete avoids this. Mirrors create_table's H-5 fast path. let delay_row: (f64,) = sqlx::query_as( - "SELECT COALESCE((SELECT value::FLOAT8 FROM settings WHERE key = 'control_plane_delay_seconds'), 0.25)", + "SELECT COALESCE((SELECT value::FLOAT8 FROM settings WHERE key = 'control_plane_delay_seconds'), $1)", ) + .bind(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS) .fetch_one(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 9603d68ff..55ad68cb4 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -21,7 +21,7 @@ mod delete_table; pub(crate) mod gsi_queue; mod management_store; mod metadata_engine; -mod migrations; +pub mod migrations; mod operations; mod pg_util; mod snapshot_engine; @@ -168,22 +168,16 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, + catalog_migrations: Some(&migrations::PostgresCatalogMigrations), } } use std::sync::Arc; -use extenddb_core::version::CatalogVersion; use extenddb_storage::error::StorageError; use sqlx::PgPool; use sqlx::postgres::PgPoolOptions; -/// Expected catalog version — compiled into the binary (REQ-CAT-006, D-9). -/// -/// The tuple is the single source of truth. Use `CATALOG_VERSION.to_string()` -/// wherever a string representation is needed. -pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 0, 4); - /// Minimum number of connections allowed per pool. /// /// Each `DynamoDB` request triggers an auth/authz query fanout against the @@ -215,12 +209,18 @@ pub struct PostgresConfig { /// on the write path and the background refresh worker. pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; +/// Control-plane delay (seconds) when the `control_plane_delay_seconds` +/// setting is absent: how long CreateTable and DeleteTable keep a table +/// CREATING or DELETING. CreateTable, DeleteTable, the control-plane worker +/// and the console's default all read it. +pub const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; + /// Read the propagation-delay setting, preferring the canonical key and falling back /// to the pre-rename one. /// /// A catalog created before the rename holds the operator's value under the old name, -/// and the server refuses to start on a catalog-version mismatch rather than migrating, -/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// and no catalog migration rewrites that row yet, though one could. Reading past it +/// would silently reset a /// configured delay to the default; since 0 means synchronous, the silent change would /// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference /// deterministic when both rows exist. @@ -270,6 +270,10 @@ pub struct PostgresEngine { /// Restores each account may run at once in this process; see /// `backup_engine::RestoreAccountSlots`. pub(crate) restore_account_slots: backup_engine::RestoreAccountSlots, + /// The serving registration a server holds for its life, so `migrate + /// --yes` refuses while it serves, and takes again after the database + /// ended its session. `None` outside a server. + pub(crate) serving: Option>, /// Bounds how many table snapshots may be open at once (one permit from /// the open until the stream's transaction ends): half the pool /// ([`snapshot_slot_count`]) on either layout. @@ -424,6 +428,7 @@ impl PostgresEngine { restore_server_slots: Arc::new(tokio::sync::Semaphore::new(server_slots)), restore_server_slot_count: server_slots, restore_account_slots: backup_engine::RestoreAccountSlots::default(), + serving: None, snapshot_open_slots: Arc::new(tokio::sync::Semaphore::new(snapshot_slot_count( pool_size, ))), @@ -548,110 +553,6 @@ impl PostgresEngine { Ok(()) } - /// Validate catalog version matches the compiled-in expectation (REQ-CAT-007, D-10). - /// - /// Reads the version string from the `settings` table and parses it - /// strictly into a `CatalogVersion`. Rejects malformed strings. - /// - /// # Errors - /// - /// Returns `StorageError::CatalogNotInitialized` if the catalog tables don't exist. - /// Returns `StorageError::CatalogVersionMismatch` if the version doesn't match. - /// Returns `StorageError::Internal` if the stored version string is malformed, carrying - /// `check_stored_catalog_version`'s refusal text. - pub async fn check_catalog_version(&self) -> Result<(), StorageError> { - // Check table existence via information_schema (robust, not string-matching). - let exists: (bool,) = sqlx::query_as( - "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'settings' AND table_schema = 'public')", - ) - .fetch_one(&self.pool) - .await - .map_err(|e| StorageError::Connection(e.to_string()))?; - - if !exists.0 { - return Err(StorageError::CatalogNotInitialized); - } - - let row: Option<(String,)> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Connection(e.to_string()))?; - - let found_str = row.ok_or(StorageError::CatalogNotInitialized)?.0; - - // An unreadable version gets the shared guard's refusal, the text - // `extenddb migrate` prints for the same catalog. - let found = found_str.parse::().map_err(|e| { - StorageError::Internal( - extenddb_storage::util::check_stored_catalog_version( - Some(&found_str), - CATALOG_VERSION, - ) - .err() - .unwrap_or_else(|| e.to_string()), - ) - })?; - - if found != CATALOG_VERSION { - return Err(StorageError::CatalogVersionMismatch { - expected: CATALOG_VERSION.to_string(), - found: found_str, - }); - } - - Ok(()) - } - - /// Everything `serve` checks and changes in the catalog before serving, - /// in order; returns the shape steps applied. - /// - /// 1. The version gate ([`Self::check_catalog_version`]). - /// 2. The ledger: a catalog migration file this binary carries that the - /// catalog's `schema_history` does not record refuses startup, naming - /// the files. The version alone does not show it: another migration of - /// the same number (another branch's `003_*`) can have stamped the - /// current version. Every catalog has a ledger (`001_schema.sql` - /// creates it and the runner records each file it applies), so a - /// catalog that ran every file lists none here. - /// 3. The startup shape steps (`migrations::converge_catalog_shapes_at_serve`): - /// a catalog stamped with the current version by an earlier revision of - /// a migration file is otherwise served in a shape on which every - /// DescribeTable fails. Each step waits at most - /// `migrations::SERVE_SHAPE_LOCK_TIMEOUT_SECS` for its lock. On a - /// current catalog this only reads `information_schema`. - /// - /// # Errors - /// - /// `BackendError::CatalogVersionMismatch` from the gate; - /// `BackendError::InitializationFailed` for every other failure, with - /// the advice to run `extenddb migrate --yes` for steps 2 and 3. - pub async fn prepare_catalog_for_serve(&self) -> Result, BackendError> { - self.check_catalog_version().await.map_err(|e| match e { - StorageError::CatalogVersionMismatch { expected, found } => { - BackendError::CatalogVersionMismatch { expected, found } - } - // The unreadable-version refusal: printed as is, not as - // "Internal error: ...". - StorageError::Internal(refusal) => BackendError::InitializationFailed(refusal), - _ => BackendError::InitializationFailed(e.to_string()), - })?; - let migrate = - "; stop the servers and run `extenddb migrate --yes` with the admin credentials"; - let unrecorded = migrations::pending_catalog_migrations(&self.pool) - .await - .map_err(|e| BackendError::InitializationFailed(format!("{}{migrate}", op_msg(e))))?; - if !unrecorded.is_empty() { - return Err(BackendError::InitializationFailed(format!( - "the catalog does not record these catalog migration files as applied: {}{migrate}", - unrecorded.join(", ") - ))); - } - migrations::converge_catalog_shapes_at_serve(&self.pool) - .await - .map_err(|e| BackendError::InitializationFailed(format!("{}{migrate}", op_msg(e)))) - } - /// Query the data database name from the catalog for the startup banner (REQ-LOG-001). /// /// Returns `"(not configured)"` if no data database has been registered. @@ -758,21 +659,9 @@ impl PostgresEngine { // ============================================================================ use extenddb_auth::CredentialStore; -use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::hooks::{ServerRuntimeHooks, ServingRegistrar, WorkerContext}; use extenddb_storage::server_components::{BackendError, ServerComponents}; -/// The message of a management-store error, without its variant name. -fn op_msg(e: extenddb_storage::management_store::OpError) -> String { - use extenddb_storage::management_store::OpError; - match e { - OpError::Validation(m) - | OpError::AlreadyExists(m) - | OpError::NotFound(m) - | OpError::HasDependents(m) - | OpError::Internal(m) => m, - } -} - /// Backend-specific runtime hooks for `PostgreSQL`. struct PostgresRuntimeHooks { engine: Arc, @@ -883,6 +772,16 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { fn backend_info(&self) -> Option { Some(format!("data_db={}", self.data_db_name)) } + + /// The database can end the registration's session (a restart, a + /// failover, `pg_terminate_backend`, a pooler that drops it), which + /// frees the serving key; the server then registers again or stops. + fn serving_registrar(&self) -> Option> { + self.engine + .serving + .clone() + .map(|registration| registration as Arc) + } } /// Build server components for the Postgres backend (registered in [`register`]). @@ -901,6 +800,10 @@ fn server_components_factory( let max_catalog_connections = config.max_catalog_connections(); let region = region.to_string(); Box::pin(async move { + let registration = migrations::ServerRegistration::register(&connection_string) + .await + .map_err(BackendError::Catalog)?; + // Build PostgresConfig from extracted values let pg_config = PostgresConfig { connection_string: connection_string.clone(), @@ -909,17 +812,13 @@ fn server_components_factory( }; // Create PostgresEngine - let engine = PostgresEngine::new(&pg_config, ®ion) + let mut engine = PostgresEngine::new(&pg_config, ®ion) .await .map_err(|e| BackendError::ConnectionFailed { backend: "postgres".to_string(), details: e.to_string(), })?; - - // Version gate, unrecorded migration files, startup shape steps. - for step in engine.prepare_catalog_for_serve().await? { - tracing::warn!("converged the catalog shape at startup: {step}"); - } + engine.serving = Some(Arc::new(registration)); // Rebuild any vector index a crash left CREATING, before serving: an index // in that state is not searchable, and nothing else will repair it. diff --git a/crates/storage-postgres/src/management_store/access_keys.rs b/crates/storage-postgres/src/management_store/access_keys.rs index 86002f3e9..c25f21f51 100755 --- a/crates/storage-postgres/src/management_store/access_keys.rs +++ b/crates/storage-postgres/src/management_store/access_keys.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{AccessKeyCreated, OpError, OpResult}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::{is_fk_violation, is_unique_violation}; @@ -210,10 +211,7 @@ impl PostgresCatalogStore { ) .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_session_policies failed: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| database_error("list_session_policies", &e))?; Ok(rows .into_iter() .map( @@ -241,10 +239,7 @@ impl PostgresCatalogStore { .bind(session_token) .execute(self.pool()) .await - .map_err(|e| { - tracing::error!("update_session_policy failed: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| database_error("update_session_policy", &e))?; Ok(()) } diff --git a/crates/storage-postgres/src/management_store/accounts.rs b/crates/storage-postgres/src/management_store/accounts.rs index a4e5df6ae..8065eb99e 100755 --- a/crates/storage-postgres/src/management_store/accounts.rs +++ b/crates/storage-postgres/src/management_store/accounts.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{AccountDetail, OpError, OpResult}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::is_unique_violation; @@ -93,10 +94,7 @@ impl PostgresCatalogStore { sqlx::query_as("SELECT account_id, account_name FROM accounts ORDER BY account_id") .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_all_accounts: {e}"); - OpError::Internal("Database error".to_owned()) - }) + .map_err(|e| database_error("list_all_accounts", &e)) } pub(crate) async fn default_account_id_impl(&self) -> OpResult> { diff --git a/crates/storage-postgres/src/management_store/groups.rs b/crates/storage-postgres/src/management_store/groups.rs index 9ec442808..570047642 100755 --- a/crates/storage-postgres/src/management_store/groups.rs +++ b/crates/storage-postgres/src/management_store/groups.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{GroupDetail, OpError, OpResult}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::{is_fk_violation, is_unique_violation}; @@ -70,10 +71,7 @@ impl PostgresCatalogStore { .bind(account_id) .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_groups: {e}"); - OpError::Internal("Database error".to_owned()) - }) + .map_err(|e| database_error("list_groups", &e)) } pub(crate) async fn get_group_detail_impl( diff --git a/crates/storage-postgres/src/management_store/mod.rs b/crates/storage-postgres/src/management_store/mod.rs index 28b6948f3..ceb087928 100755 --- a/crates/storage-postgres/src/management_store/mod.rs +++ b/crates/storage-postgres/src/management_store/mod.rs @@ -7,7 +7,7 @@ //! file under the 500-line limit. use extenddb_storage::management_store::{ - AccessKeyCreated, AccountDetail, GroupDetail, OpResult, RoleDetail, UserDetail, + AccessKeyCreated, AccountDetail, GroupDetail, OpError, OpResult, RoleDetail, UserDetail, }; use futures::future::BoxFuture; @@ -20,6 +20,22 @@ mod policies; mod roles; mod users; +/// The error of a failed statement, logged with `what`. It keeps the +/// statement's SQLSTATE, so the migration runner tells a lock timeout or a +/// missing privilege from other failures by its code; the driver's text +/// goes to the log and to [`OpError::Database`], which API responses never +/// show. +pub(crate) fn database_error(what: &str, e: &sqlx::Error) -> OpError { + tracing::error!("{what}: {e}"); + match e.as_database_error().and_then(|d| d.code()) { + Some(code) => OpError::Database { + message: format!("{what}: {e}"), + sqlstate: code.into_owned(), + }, + None => OpError::Internal("Database error".to_owned()), + } +} + impl extenddb_storage::management_store::ManagementStore for PostgresCatalogStore { // ── Accounts ─────────────────────────────────────────────────── diff --git a/crates/storage-postgres/src/management_store/policies.rs b/crates/storage-postgres/src/management_store/policies.rs index 284b94515..771e83350 100755 --- a/crates/storage-postgres/src/management_store/policies.rs +++ b/crates/storage-postgres/src/management_store/policies.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{OpError, OpResult}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::is_fk_violation; @@ -35,10 +36,7 @@ impl PostgresCatalogStore { match result { Ok(_) => Ok(()), Err(e) if is_fk_violation(&e) => Err(OpError::NotFound("Account not found".to_owned())), - Err(e) => { - tracing::error!("put_policy failed: {e}"); - Err(OpError::Internal("Database error".to_owned())) - } + Err(e) => Err(database_error("put_policy", &e)), } } @@ -87,10 +85,7 @@ impl PostgresCatalogStore { .bind(principal_name) .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_policies: {e}"); - OpError::Internal("Database error".to_owned()) - }) + .map_err(|e| database_error("list_policies", &e)) } // ── Permissions boundaries ───────────────────────────────────── @@ -117,10 +112,7 @@ impl PostgresCatalogStore { match result { Ok(_) => Ok(()), Err(e) if is_fk_violation(&e) => Err(OpError::NotFound("Account not found".to_owned())), - Err(e) => { - tracing::error!("set_boundary failed: {e}"); - Err(OpError::Internal("Database error".to_owned())) - } + Err(e) => Err(database_error("set_boundary", &e)), } } @@ -139,10 +131,7 @@ impl PostgresCatalogStore { .bind(principal_name) .fetch_optional(self.pool()) .await - .map_err(|e| { - tracing::error!("get_boundary: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| database_error("get_boundary", &e))?; Ok(row.map(|(doc,)| doc)) } diff --git a/crates/storage-postgres/src/management_store/roles.rs b/crates/storage-postgres/src/management_store/roles.rs index 479c1d480..8c231fef1 100755 --- a/crates/storage-postgres/src/management_store/roles.rs +++ b/crates/storage-postgres/src/management_store/roles.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{OpError, OpResult, RoleDetail}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::{is_fk_violation, is_unique_violation}; @@ -76,10 +77,7 @@ impl PostgresCatalogStore { .bind(account_id) .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_roles: {e}"); - OpError::Internal("Database error".to_owned()) - }) + .map_err(|e| database_error("list_roles", &e)) } pub(crate) async fn get_role_detail_impl( diff --git a/crates/storage-postgres/src/management_store/users.rs b/crates/storage-postgres/src/management_store/users.rs index 245ad77ae..372a6b043 100755 --- a/crates/storage-postgres/src/management_store/users.rs +++ b/crates/storage-postgres/src/management_store/users.rs @@ -5,6 +5,7 @@ use extenddb_storage::management_store::{OpError, OpResult, UserDetail}; +use super::database_error; use crate::catalog_store::PostgresCatalogStore; use crate::pg_util::{is_fk_violation, is_unique_violation}; @@ -114,10 +115,7 @@ impl PostgresCatalogStore { .bind(account_id) .fetch_all(self.pool()) .await - .map_err(|e| { - tracing::error!("list_users: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| database_error("list_users", &e))?; Ok(rows .into_iter() .map(|(aid, un, arn, pw, ca)| (aid, un, arn, pw.is_some(), ca)) diff --git a/crates/storage-postgres/src/migrations.rs b/crates/storage-postgres/src/migrations.rs deleted file mode 100755 index 952f7920a..000000000 --- a/crates/storage-postgres/src/migrations.rs +++ /dev/null @@ -1,686 +0,0 @@ -// Copyright 2026 ExtendDB contributors -// SPDX-License-Identifier: Apache-2.0 - -//! `PostgreSQL` schema migration helpers for catalog and data databases. - -use extenddb_storage::management_store::{OpError, OpResult}; -use sqlx::PgPool; - -/// Embedded catalog migration files, applied in order. -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ - ( - "001_schema.sql", - include_str!("../../storage-postgres/migrations/001_schema.sql"), - ), - ( - "002_vector_indexes.sql", - include_str!("../../storage-postgres/migrations/002_vector_indexes.sql"), - ), - ( - "003_backup_definitions.sql", - include_str!("../../storage-postgres/migrations/003_backup_definitions.sql"), - ), -]; - -/// The runner's convergence write, executed after the walk over -/// [`CATALOG_MIGRATIONS`]. Same statement shape as the files' own in-file -/// writes, parameterized on the compiled version. -pub(crate) const SET_CATALOG_VERSION_SQL: &str = - "UPDATE settings SET value = $1 WHERE key = 'catalog_version'"; - -/// Run catalog migrations, skipping already-applied ones. -pub(crate) async fn run_catalog_migrations(pool: &PgPool) -> OpResult<()> { - println!("--- Running catalog migrations..."); - for (filename, sql) in CATALOG_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - sqlx::raw_sql(sql) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Migration {filename} failed: {e}")))?; - // TODO(#221): applying this SQL and recording it are separate commits. - // A crash here can leave a migration applied but unrecorded. Catalog 001 - // is normally shielded from replay by its version write, data 001 has an - // adoption guard, and data 002 is repeatable, but those are narrow - // recovery properties: catalog 001 is not idempotent and replaying data - // 003 drops the token table. The sqlx adoption must remove the files' - // own BEGIN/COMMIT and commit each ledger row with its migration before - // another migration lands. - record_migration(pool, filename).await?; - } - // The runner owns the final version write. Each file still writes the - // version it introduces, but a replay can re-apply an EARLIER file while a - // later one stays recorded and skipped: the re-applied file's in-file write - // then leaves the stored version behind the schema actually present, the - // startup gate refuses the catalog, and a second migrate finds nothing - // unrecorded and writes nothing, stranding the deployment. Converging on - // the compiled version after every completed walk closes that gap. The - // in-file writes stay until #221 moves version ownership into the runner - // entirely. - converge_catalog_version(pool).await?; - for step in converge_catalog_shapes(pool).await? { - println!(" Converged catalog shape: {step}."); - } - println!(" Migrations applied."); - Ok(()) -} - -/// Filenames of [`CATALOG_MIGRATIONS`] not recorded in the catalog's -/// `schema_history`. The ledger keys on the filename, so this is the set -/// [`run_catalog_migrations`] would apply, whatever `catalog_version` says: -/// a catalog stamped with the current version by another migration of the -/// same number (a different branch's `003_*`) still lists this binary's file -/// here. -pub(crate) async fn pending_catalog_migrations(pool: &PgPool) -> OpResult> { - let mut pending = Vec::new(); - for (filename, _sql) in CATALOG_MIGRATIONS { - if !is_migration_applied(pool, filename).await? { - pending.push((*filename).to_owned()); - } - } - Ok(pending) -} - -/// One catalog shape step: a description for reports, whether the catalog -/// needs it (a query returning one boolean), and the idempotent SQL that -/// applies it. -struct ShapeStep { - what: &'static str, - needed: &'static str, - apply: &'static str, - /// Whether `serve` applies this step at startup - /// ([`converge_catalog_shapes_at_serve`]); every step is applied by - /// `extenddb migrate --yes`. - at_serve: bool, -} - -/// Shape steps for a catalog whose recorded migrations ran an earlier -/// revision of a file (the ledger keys on the filename, so the walk skips -/// the edited file). Each `needed` query is false on a catalog the current -/// files created, and on one where `table_restores` does not exist yet (the -/// walk creates it in the current shape). -/// -/// `003_backup_definitions.sql` was edited in place before release: its -/// earlier revision created `table_restores` without `source_table_arn` and -/// with `DEFAULT NOW()` on `restore_date_time`. DescribeTable selects -/// `source_table_arn` for every table, so on that shape every DescribeTable -/// fails. The added column is nullable (rows written by the earlier build -/// have no value, and are reported without a SourceTableArn); every row this -/// build writes sets it. -/// -/// Dropping the default is applied by `migrate --yes` only, never by -/// `serve`: the earlier build inserts `table_restores` rows without -/// `restore_date_time` and relies on the default, so dropping it while an -/// earlier-build server is still running makes that server's restores fail. -/// `migrate` is run with every server stopped. Until then the default is -/// harmless to this build, which always writes the value. -const CATALOG_SHAPE_STEPS: &[ShapeStep] = &[ - ShapeStep { - what: "add table_restores.source_table_arn", - needed: "SELECT EXISTS(SELECT 1 FROM information_schema.tables \ - WHERE table_schema = 'public' AND table_name = 'table_restores') \ - AND NOT EXISTS(SELECT 1 FROM information_schema.columns \ - WHERE table_schema = 'public' AND table_name = 'table_restores' \ - AND column_name = 'source_table_arn')", - apply: "ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS source_table_arn TEXT", - at_serve: true, - }, - ShapeStep { - what: "drop the default of table_restores.restore_date_time", - needed: "SELECT EXISTS(SELECT 1 FROM information_schema.columns \ - WHERE table_schema = 'public' AND table_name = 'table_restores' \ - AND column_name = 'restore_date_time' AND column_default IS NOT NULL)", - apply: "ALTER TABLE table_restores ALTER COLUMN restore_date_time DROP DEFAULT", - at_serve: false, - }, -]; - -/// How long a startup shape step waits for its table lock, in seconds. An -/// `ALTER TABLE` waits behind every open transaction that touched the table, -/// and every later statement on the table waits behind the `ALTER`; with -/// other servers still serving, an unbounded wait would stall them all. On -/// timeout the server refuses to start instead. -pub(crate) const SERVE_SHAPE_LOCK_TIMEOUT_SECS: u64 = 5; - -async fn shape_step_needed(pool: &PgPool, step: &ShapeStep) -> OpResult { - sqlx::query_scalar(step.needed) - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("Check catalog shape ({}): {e}", step.what))) -} - -/// The [`CATALOG_SHAPE_STEPS`] this catalog needs, by description. -pub(crate) async fn pending_catalog_shapes(pool: &PgPool) -> OpResult> { - let mut pending = Vec::new(); - for step in CATALOG_SHAPE_STEPS { - if shape_step_needed(pool, step).await? { - pending.push(step.what.to_owned()); - } - } - Ok(pending) -} - -/// Apply the [`CATALOG_SHAPE_STEPS`] this catalog needs; returns their -/// descriptions. Runs at the end of [`run_catalog_migrations`], that is from -/// `extenddb migrate --yes`. A step that is not needed issues no DDL, so on a -/// current catalog this only reads `information_schema`. Every step is -/// idempotent, so two runs at once are safe. -pub(crate) async fn converge_catalog_shapes(pool: &PgPool) -> OpResult> { - let mut applied = Vec::new(); - for step in CATALOG_SHAPE_STEPS { - if !shape_step_needed(pool, step).await? { - continue; - } - sqlx::query(step.apply) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Catalog shape step ({}): {e}", step.what)))?; - applied.push(step.what.to_owned()); - } - Ok(applied) -} - -/// `serve`'s startup converge: apply the needed [`CATALOG_SHAPE_STEPS`] -/// marked `at_serve`, each in its own transaction under `SET LOCAL -/// lock_timeout` of [`SERVE_SHAPE_LOCK_TIMEOUT_SECS`]. Returns the applied -/// steps' descriptions. A step that cannot take its lock in time fails with -/// an error saying so and changes nothing. -pub(crate) async fn converge_catalog_shapes_at_serve(pool: &PgPool) -> OpResult> { - let mut applied = Vec::new(); - for step in CATALOG_SHAPE_STEPS.iter().filter(|s| s.at_serve) { - if !shape_step_needed(pool, step).await? { - continue; - } - let fail = |e: sqlx::Error| { - if crate::data::lock_not_available(&e) { - OpError::Internal(format!( - "Catalog shape step ({}): could not lock the table within \ - {SERVE_SHAPE_LOCK_TIMEOUT_SECS} s, another session holds it", - step.what - )) - } else { - OpError::Internal(format!("Catalog shape step ({}): {e}", step.what)) - } - }; - let mut tx = pool.begin().await.map_err(fail)?; - sqlx::query(&format!( - "SET LOCAL lock_timeout = '{SERVE_SHAPE_LOCK_TIMEOUT_SECS}s'" - )) - .execute(&mut *tx) - .await - .map_err(fail)?; - sqlx::query(step.apply) - .execute(&mut *tx) - .await - .map_err(fail)?; - tx.commit().await.map_err(fail)?; - applied.push(step.what.to_owned()); - } - Ok(applied) -} - -/// Write the compiled catalog version, but never move the stored version -/// backwards and never overwrite one this binary cannot parse: a catalog -/// stamped by a newer binary (or in a scheme this one does not know) stays -/// stamped, so that binary's startup gate keeps working and this one's keeps -/// refusing it. `extenddb migrate` refuses such a catalog before the walk; -/// this is the runner-level guard for every other caller. The rule is -/// `extenddb_storage::util::check_stored_catalog_version`, shared with SQLite -/// and `extenddb migrate`. -async fn converge_catalog_version(pool: &PgPool) -> OpResult<()> { - let stored: Option<(String,)> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_optional(pool) - .await - .map_err(|e| OpError::Internal(format!("Read catalog_version: {e}")))?; - extenddb_storage::util::check_stored_catalog_version( - stored.as_ref().map(|(v,)| v.as_str()), - crate::CATALOG_VERSION, - ) - .map_err(OpError::Internal)?; - sqlx::query(SET_CATALOG_VERSION_SQL) - .bind(crate::CATALOG_VERSION.to_string()) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Write catalog_version: {e}")))?; - Ok(()) -} - -/// Embedded data-database migration files, applied in order. Tracked in the -/// data database's own `schema_history` table (a separate database from the -/// catalog), so `extenddb migrate` applies exactly the pending migrations. -pub(crate) const DATA_MIGRATIONS: &[(&str, &str)] = &[ - ( - "001_data_schema.sql", - include_str!("../../storage-postgres/data_migrations/001_data_schema.sql"), - ), - ( - "002_gsi_pending.sql", - include_str!("../../storage-postgres/data_migrations/002_gsi_pending.sql"), - ), - ( - "003_idempotency_account_scope.sql", - include_str!("../../storage-postgres/data_migrations/003_idempotency_account_scope.sql"), - ), - ( - "004_vector_index_state.sql", - include_str!("../../storage-postgres/data_migrations/004_vector_index_state.sql"), - ), -]; - -/// Run data database migrations, skipping already-applied ones. -/// -/// Mirrors [`run_catalog_migrations`]: each migration is recorded in -/// `schema_history` and skipped on later runs. The data database has its own -/// ledger because it is a separate database from the catalog. -pub(crate) async fn run_data_migrations(pool: &PgPool) -> OpResult<()> { - println!("--- Running data migrations..."); - - // Ensure the data database has a migration ledger before tracking. (The - // catalog ledger lives in a different database and cannot be reused here.) - sqlx::query( - "CREATE TABLE IF NOT EXISTS schema_history (\ - filename TEXT PRIMARY KEY, \ - applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()\ - )", - ) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Create data schema_history: {e}")))?; - - // Adopt a pre-tracking deployment: if 001 was applied by an earlier version - // (its tables exist) but isn't recorded, record it WITHOUT re-running it. - // Re-running 001 would execute `setval('stream_seq', ...)` again and could - // regress the stream sequence on a live database, producing duplicate - // sequence numbers. - if !is_migration_applied(pool, "001_data_schema.sql").await? - && table_exists(pool, "stream_shards").await? - { - println!(" Adopting existing 001_data_schema.sql (pre-tracking deployment)."); - record_migration(pool, "001_data_schema.sql").await?; - } - - for (filename, sql) in DATA_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - sqlx::raw_sql(sql) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Data migration {filename} failed: {e}")))?; - // TODO(#221): applying this SQL and recording it are separate commits. - // A crash here can leave a migration applied but unrecorded. Catalog 001 - // is normally shielded from replay by its version write, data 001 has an - // adoption guard, and data 002 is repeatable, but those are narrow - // recovery properties: catalog 001 is not idempotent and replaying data - // 003 drops the token table. The sqlx adoption must remove the files' - // own BEGIN/COMMIT and commit each ledger row with its migration before - // another migration lands. - record_migration(pool, filename).await?; - } - println!(" Data migrations applied."); - Ok(()) -} - -/// Programmatic ("code") data migrations, tracked in `schema_history` alongside -/// the SQL migrations. Unlike a static `.sql` file, these enumerate the -/// dynamically-named index tables (`_ddb_`) from the catalog and must run -/// outside a transaction (they use `CREATE INDEX CONCURRENTLY`), so they cannot -/// be expressed as SQL in [`DATA_MIGRATIONS`]. Applied by `extenddb migrate` -/// after the SQL migrations, so the operator controls when the change happens. -pub(crate) const DATA_CODE_MIGRATIONS: &[&str] = &["003_gsi_base_key_index"]; - -/// Run programmatic data migrations, skipping already-applied ones. -/// -/// Needs the catalog pool (to enumerate index tables and their base key schema) -/// and the data pool (where the `_ddb_*` tables and the `schema_history` ledger -/// live). Each step is recorded in `schema_history` and skipped on later runs, -/// exactly like the SQL migrations. -pub(crate) async fn run_data_code_migrations( - catalog_pool: &PgPool, - data_pool: &PgPool, -) -> OpResult<()> { - for name in DATA_CODE_MIGRATIONS { - if is_migration_applied(data_pool, name).await? { - println!(" {name} — already applied, skipping."); - continue; - } - println!(" Applying {name}..."); - match *name { - "003_gsi_base_key_index" => { - ensure_gsi_base_key_indexes(catalog_pool, data_pool).await?; - } - other => { - return Err(OpError::Internal(format!( - "Unknown data code migration: {other}" - ))); - } - } - record_migration(data_pool, name).await?; - } - Ok(()) -} - -/// Bring a fresh catalog database and a fresh data database (the same pool -/// twice on the shared layout) to the current schema the way `extenddb -/// migrate --yes` does: every catalog migration, every data migration, then -/// the data code migrations, each recorded in its ledger. For test harnesses -/// in this crate's own `tests/` directory, which otherwise copy the file -/// lists and go stale when a migration is added here. -/// -/// # Errors -/// -/// Returns the first migration's error, formatted. -#[cfg(feature = "test-hooks")] -#[doc(hidden)] -pub async fn apply_all_migrations(catalog: &PgPool, data: &PgPool) -> Result<(), String> { - run_catalog_migrations(catalog) - .await - .map_err(|e| format!("{e:?}"))?; - run_data_migrations(data) - .await - .map_err(|e| format!("{e:?}"))?; - run_data_code_migrations(catalog, data) - .await - .map_err(|e| format!("{e:?}")) -} - -/// Create the base-table-key index on every existing GSI/LSI table. -/// -/// During GSI propagation each index table (`_ddb_`) is looked up back to -/// its base item via `WHERE base_pk = $1 AND base_sk_* = $2`; without a leading -/// `(base_pk, base_sk_*)` index that is a sequential scan. New tables get this -/// index at creation time (see `ddl.rs`); this migration adds it to tables -/// created before the index existed. `CREATE INDEX CONCURRENTLY IF NOT EXISTS` -/// is idempotent and does not block concurrent writes. -async fn ensure_gsi_base_key_indexes(catalog_pool: &PgPool, data_pool: &PgPool) -> OpResult<()> { - use extenddb_core::types::{AttributeDefinition, KeySchemaElement}; - use extenddb_storage::util::{sk_column, sk_column_n}; - - // Enumerate every index and its base table key schema from the catalog. - let rows: Vec<(String, serde_json::Value, serde_json::Value)> = sqlx::query_as( - "SELECT i.index_id, t.key_schema, t.attribute_definitions \ - FROM indexes i \ - JOIN tables t ON i.table_id = t.table_id", - ) - .fetch_all(catalog_pool) - .await - .map_err(|e| OpError::Internal(format!("Enumerate indexes: {e}")))?; - - for (index_id, ks_json, ad_json) in rows { - let base_ks: Vec = - serde_json::from_value(ks_json).map_err(|e| OpError::Internal(e.to_string()))?; - let attr_defs: Vec = - serde_json::from_value(ad_json).map_err(|e| OpError::Internal(e.to_string()))?; - - let base_sks = crate::data::all_sort_key_info(&base_ks, &attr_defs); - let idx_table = crate::data::index_table_name(&index_id); - let idx_name = format!("_ddb_{index_id}_base_key_idx"); - - let mut base_key_cols = vec!["base_pk".to_owned()]; - for (i, &(_, sk_type)) in base_sks.iter().enumerate() { - let col = if i == 0 { - format!("base_{}", sk_column(sk_type)) - } else { - format!("base_{}", sk_column_n(i, sk_type)) - }; - base_key_cols.push(col); - } - - // CONCURRENTLY cannot run inside a transaction, so execute on the pool. - let sql = format!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS \"{}\" ON {} ({})", - idx_name, - idx_table, - base_key_cols.join(", ") - ); - sqlx::query(&sql) - .execute(data_pool) - .await - .map_err(|e| OpError::Internal(format!("Base key index on {idx_table}: {e}")))?; - } - Ok(()) -} - -/// Check if a table exists in the public schema. -pub(crate) async fn table_exists(pool: &PgPool, name: &str) -> OpResult { - let exists: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM information_schema.tables \ - WHERE table_name = $1 AND table_schema = 'public')", - ) - .bind(name) - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("Check table exists: {e}")))?; - Ok(exists) -} - -/// Filenames of [`DATA_MIGRATIONS`] not yet applied to this data database. -/// -/// Mirrors the apply logic in [`run_data_migrations`] without executing -/// anything, so callers (e.g. `extenddb migrate`) can report and gate on -/// pending work. A pre-tracking baseline (`001_data_schema.sql` whose tables -/// already exist but isn't recorded) is treated as already applied: it will be -/// adopted — recorded without re-running — not applied, so it is not reported -/// as pending. -pub(crate) async fn pending_data_migrations(pool: &PgPool) -> OpResult> { - let has_history = table_exists(pool, "schema_history").await?; - // Pre-tracking deployment: 001 ran under an earlier version (its tables - // exist) but was never recorded. It is adopted, not re-run. - let adopts_baseline = !has_history && table_exists(pool, "stream_shards").await?; - - let mut pending = Vec::new(); - for (filename, _sql) in DATA_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - continue; - } - if *filename == "001_data_schema.sql" && adopts_baseline { - continue; - } - pending.push((*filename).to_owned()); - } - // Code migrations are tracked in the same data-database ledger. - for name in DATA_CODE_MIGRATIONS { - if !is_migration_applied(pool, name).await? { - pending.push((*name).to_owned()); - } - } - Ok(pending) -} - -/// Check if a migration has already been applied. -async fn is_migration_applied(pool: &PgPool, filename: &str) -> OpResult { - if table_exists(pool, "schema_history").await? { - let applied: (bool,) = - sqlx::query_as("SELECT EXISTS(SELECT 1 FROM schema_history WHERE filename = $1)") - .bind(filename) - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("Check migration: {e}")))?; - return Ok(applied.0); - } - Ok(false) -} - -/// Record a migration in the `schema_history` table. -async fn record_migration(pool: &PgPool, filename: &str) -> OpResult<()> { - if !table_exists(pool, "schema_history").await? { - return Ok(()); - } - sqlx::query( - "INSERT INTO schema_history (filename) VALUES ($1) ON CONFLICT (filename) DO NOTHING", - ) - .bind(filename) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Record migration: {e}")))?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::CATALOG_MIGRATIONS; - use crate::CATALOG_VERSION; - - /// The catalog version and the migration list must move together. - /// - /// A migration that creates its schema without moving the version leaves a - /// deployment the binary refuses to serve; moving the version without a - /// migration leaves one that cannot reach it. Both are caught today only by a - /// test that needs a live PostgreSQL and a built binary, so this is the - /// tripwire that fires in an ordinary `cargo test`: adding a migration file - /// breaks the count, which forces a decision about the version. - #[test] - fn the_migration_count_and_the_catalog_version_agree() { - assert_eq!( - CATALOG_MIGRATIONS.len(), - 3, - "a catalog migration was added or removed; update CATALOG_VERSION and this count" - ); - assert_eq!(CATALOG_VERSION.to_string(), "0.0.4"); - } - - /// The version the binary expects must be the version the schema writes. - /// - /// These two live in different languages and different files, so nothing but - /// a check like this ties them together. Without it a version bump that - /// forgets the SQL side produces a deployment that migrates "successfully" - /// and then refuses to start. - #[test] - fn the_last_migration_writes_the_expected_catalog_version() { - let (filename, sql) = CATALOG_MIGRATIONS - .last() - .expect("there is at least one catalog migration"); - let expected = format!("'{}'", CATALOG_VERSION); - assert!( - sql.contains("catalog_version") && sql.contains(&expected), - "{filename} must set catalog_version to {expected}" - ); - } - - /// Each migration is registered under the filename it is stored as. - /// - /// The ledger keys on this string, so a mismatch between the registered name - /// and the file would record one name and look for another, and the migration - /// would be applied again on every run. - #[test] - fn every_migration_is_registered_under_a_sql_filename_with_its_own_contents() { - for (filename, sql) in CATALOG_MIGRATIONS { - assert!(filename.ends_with(".sql"), "{filename}"); - assert!(!sql.trim().is_empty(), "{filename} is empty"); - } - // The failure this guards is a copy-pasted `include_str!` that points one - // entry at another file's bytes: the ledger would then record one name - // while the SQL of another ran, and the missed migration would be applied - // again on every upgrade. Two entries sharing contents is what that looks - // like, so distinctness is the assertion that delivers the rationale. - for (i, (left_name, left_sql)) in CATALOG_MIGRATIONS.iter().enumerate() { - for (right_name, right_sql) in &CATALOG_MIGRATIONS[i + 1..] { - assert_ne!( - left_sql, right_sql, - "{left_name} and {right_name} embed identical SQL; check their include_str! paths" - ); - } - } - } - - /// The fail-closed catalog guard (`converge_catalog_version`, via - /// `check_stored_catalog_version`) against a live catalog: a stored - /// version this binary cannot parse, or a newer one, is refused and left - /// exactly as stored; an older one is moved up and an equal one kept. - /// Needs `EXTENDDB_TEST_PG_CONNECTION_STRING`; reports a skip without it. - #[tokio::test] - async fn converge_refuses_an_unreadable_or_newer_catalog_version_and_leaves_it() { - let Some(base) = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING") - .ok() - .filter(|s| !s.trim().is_empty()) - else { - eprintln!( - "SKIP converge_refuses_an_unreadable_or_newer_catalog_version: no PostgreSQL" - ); - return; - }; - let base = base.trim_end_matches('/').to_owned(); - let name = format!("eddb_conv_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); - let admin = sqlx::postgres::PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{base}/postgres")) - .await - .expect("admin"); - sqlx::query(&format!("CREATE DATABASE \"{name}\"")) - .execute(&admin) - .await - .expect("create scratch"); - let pool = sqlx::postgres::PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{base}/{name}")) - .await - .expect("scratch"); - sqlx::query("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)") - .execute(&pool) - .await - .expect("settings"); - - let compiled = CATALOG_VERSION.to_string(); - // (stored, expect Ok, stored value afterwards) - let cases: [(Option<&str>, bool, Option<&str>); 5] = [ - (Some("garbage"), false, Some("garbage")), - (Some(""), false, Some("")), - (Some("9.9.9"), false, Some("9.9.9")), - (Some("0.0.3"), true, Some(compiled.as_str())), - (Some(compiled.as_str()), true, Some(compiled.as_str())), - ]; - let mut outcomes = Vec::new(); - for (stored, _, _) in &cases { - sqlx::query("DELETE FROM settings") - .execute(&pool) - .await - .expect("clear"); - if let Some(v) = stored { - sqlx::query("INSERT INTO settings (key, value) VALUES ('catalog_version', $1)") - .bind(v) - .execute(&pool) - .await - .expect("seed"); - } - let result = super::converge_catalog_version(&pool).await; - let after: Option = - sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_optional(&pool) - .await - .expect("read back"); - outcomes.push((result.map_err(|e| format!("{e:?}")), after)); - } - // Absent: proceeds, and (the write is an UPDATE) stays absent. - sqlx::query("DELETE FROM settings") - .execute(&pool) - .await - .expect("clear"); - let absent = super::converge_catalog_version(&pool).await.is_ok(); - - pool.close().await; - let _ = sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) - .execute(&admin) - .await; - admin.close().await; - - for ((stored, ok, want_after), (result, after)) in cases.iter().zip(&outcomes) { - assert_eq!(result.is_ok(), *ok, "stored {stored:?}: {result:?}"); - assert_eq!( - after.as_deref(), - *want_after, - "stored {stored:?} afterwards" - ); - } - let garbage = outcomes[0].0.as_ref().expect_err("garbage refused"); - assert!(garbage.contains("refusing to overwrite"), "{garbage}"); - let newer = outcomes[2].0.as_ref().expect_err("newer refused"); - assert!(newer.contains("newer than this binary"), "{newer}"); - assert!(absent, "an absent version proceeds"); - } -} diff --git a/crates/storage-postgres/src/migrations/backend.rs b/crates/storage-postgres/src/migrations/backend.rs new file mode 100644 index 000000000..2924fa23f --- /dev/null +++ b/crates/storage-postgres/src/migrations/backend.rs @@ -0,0 +1,588 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The runner's PostgreSQL parts. +//! +//! The runner opens one connection to each database, as a pool of exactly one +//! connection, so code bodies that take a pool (the management store, for +//! migration 2) run inside the runner's transaction. A connection that is +//! lost is not reopened: its transaction is gone, and statements would then +//! commit one by one. +//! +//! Per body, in its store: `BEGIN`, `SET LOCAL lock_timeout = '5s'`, the body, +//! for migration 1's catalog body the data connection rows `init` passes, +//! the post-check the first time migration 1 is applied, the ledger row and +//! (in the catalog) the version row, `COMMIT`. A lock timeout (SQLSTATE +//! 55P03) rolls back and runs the body again, three times, then stops naming +//! the tables other sessions hold locks on. A body that runs outside a +//! transaction (migration 4's concurrent index builds) runs on the pool in +//! autocommit through the index helper, and its row is written after its +//! last statement. Before every body the migration lock's connection is +//! checked. + +use async_trait::async_trait; +use extenddb_storage::management_store::ManagementStore; +use extenddb_storage::migrations::{ + CatalogTraces, Done, HasManagementStore, LedgerColumns, LedgerRow, LedgerWrites, Migration, + MigrationBackend, MigrationError, MigrationId, Released, Step, StoreKind, StoreRef, + in_transaction, record_rows, +}; +use sqlx::PgPool; + +use super::connect::{ + missing_database, one_connection, relation_exists, runner_application, sql_error, +}; +use super::failure::{Failure, current_txid, exec}; +use super::locks::{LockHandle, MigrationLock}; +use super::rows::Rows; +use super::sessions::{ + CONNECTION_CHECK, connection_check_supported, lock_timeout_stop, ownership_stop, served_data, + wait_for_earlier_runners, +}; +use super::{PostgresTarget, RunsIn, runs_in}; +use crate::catalog_store::PostgresCatalogStore; + +/// What PostgreSQL code bodies receive: the runner's connection to each +/// database, a management store over the catalog connection, and the +/// migration lock to check before each statement that runs outside a +/// transaction. +pub struct PostgresStores { + pub(crate) catalog: PgPool, + pub(crate) data: PgPool, + /// A second connection to the data database, on its server and as its + /// role, from which the index helper watches and cancels a statement + /// the data connection runs. A status session has none. + data_watch: Option, + management: PostgresCatalogStore, + lock: Option, + /// The `application_name` of this runner's sessions. + pub(crate) application: String, +} + +impl PostgresStores { + /// The runner's connection to the catalog database, for a test body that + /// runs its own statements inside the runner's transaction. + #[cfg(feature = "test-util")] + #[must_use] + pub fn catalog_connection(&self) -> &PgPool { + &self.catalog + } + + /// Fail when the migration lock's connection is lost. A status session, + /// or a test driving the runner itself, has no lock to check. + pub(crate) async fn check_lock(&self) -> Result<(), MigrationError> { + match &self.lock { + Some(lock) => lock.check_alive().await, + None => Ok(()), + } + } + + fn pool(&self, kind: StoreKind) -> &PgPool { + match kind { + StoreKind::Catalog => &self.catalog, + StoreKind::Data | StoreKind::Account => &self.data, + } + } + + /// The connection from which a statement running on `pool` is watched + /// and cancelled: [`Self::data_watch`] for the data connection, which + /// reaches the data database's server as its role, and the catalog + /// connection otherwise. Only data bodies run outside a transaction. + pub(crate) fn watch_for(&self, pool: &PgPool) -> &PgPool { + match &self.data_watch { + Some(watch) if std::ptr::eq(pool, &self.data) => watch, + _ => &self.catalog, + } + } +} + +impl HasManagementStore for PostgresStores { + fn management_store(&self) -> &dyn ManagementStore { + &self.management + } +} + +/// How long each statement of a body waits for a table lock. +const LOCK_TIMEOUT_SECS: u64 = 5; + +/// How many times a body is run again after a lock timeout. +const LOCK_RETRIES: u32 = 3; + +/// Why opening failed. +pub(crate) enum OpenError { + /// The catalog database does not exist (SQLSTATE 3D000). + NoCatalogDatabase, + Failed(MigrationError), +} + +/// The PostgreSQL backend of the migration runner. +pub struct PostgresMigrations { + stores: PostgresStores, + catalog_name: String, + data_name: String, + data_connection_row: Option, + read_only: bool, + /// Whether each database accepts [`CONNECTION_CHECK`]: catalog, data. + connection_check: [bool; 2], +} + +impl PostgresMigrations { + /// Open the runner's connections. `lock` is the migration lock the + /// caller holds, checked before every body. This waits until no + /// statement of an earlier migrator runs in either database. + /// + /// # Errors + /// + /// A database cannot be reached, the catalog's data connection rows + /// disagree with the target, or an earlier migrator's statements outlast + /// the lock's timeout. + pub async fn open( + target: &PostgresTarget, + lock: &MigrationLock, + ) -> Result { + Self::open_runner(target, Some(lock.handle())).await + } + + /// [`PostgresMigrations::open`] with no lock, for a test that drives the + /// runner itself; nothing is checked before each body. + /// + /// # Errors + /// + /// As [`PostgresMigrations::open`]. + #[cfg(feature = "test-util")] + pub async fn open_unlocked(target: &PostgresTarget) -> Result { + Self::open_runner(target, None).await + } + + async fn open_runner( + target: &PostgresTarget, + lock: Option, + ) -> Result { + Self::open_with(target, lock, false) + .await + .map_err(|e| match e { + OpenError::NoCatalogDatabase => MigrationError::Store(format!( + "the catalog database {} does not exist; run extenddb init", + target.catalog_database() + )), + OpenError::Failed(e) => e, + }) + } + + pub(crate) async fn open_for_status(target: &PostgresTarget) -> Result { + Self::open_with(target, None, true).await + } + + async fn open_with( + target: &PostgresTarget, + lock: Option, + read_only: bool, + ) -> Result { + let catalog_name = target.catalog_database(); + let application = runner_application(); + let named = |options: sqlx::postgres::PgConnectOptions| { + if read_only { + options + } else { + options.application_name(&application) + } + }; + let catalog = one_connection( + named(target.catalog.clone()), + read_only, + target.connect_timeout, + ) + .await + .map_err(|e| { + if missing_database(&e) { + OpenError::NoCatalogDatabase + } else { + OpenError::Failed(MigrationError::store( + format!("connect to the catalog database {catalog_name}"), + e, + )) + } + })?; + let data_options = match served_data(target, &catalog).await { + Ok(options) => options, + Err(e) => { + catalog.close().await; + return Err(OpenError::Failed(e)); + } + }; + let data_name = data_options.get_database().unwrap_or("postgres").to_owned(); + let data_options_for_watch = data_options.clone(); + let data = + match one_connection(named(data_options), read_only, target.connect_timeout).await { + Ok(pool) => pool, + Err(e) => { + catalog.close().await; + return Err(OpenError::Failed(MigrationError::store( + format!("connect to the data database {data_name}"), + e, + ))); + } + }; + let close_both = |e: MigrationError| { + let (catalog, data) = (catalog.clone(), data.clone()); + async move { + catalog.close().await; + data.close().await; + OpenError::Failed(e) + } + }; + let mut connection_check = [false; 2]; + let mut data_watch = None; + if !read_only { + data_watch = + match one_connection(named(data_options_for_watch), false, target.connect_timeout) + .await + { + Ok(pool) => Some(pool), + Err(e) => { + return Err(close_both(MigrationError::store( + format!("connect a second session to the data database {data_name}"), + e, + )) + .await); + } + }; + if let Some(lock) = &lock { + let databases = [ + (&catalog, catalog_name.as_str()), + (&data, data_name.as_str()), + ]; + // The time the lock's waits left: with none left, as at a + // zero lock timeout, the earlier runners' sessions are + // looked at once. + if let Err(e) = + wait_for_earlier_runners(&databases, &application, lock.remaining()).await + { + if let Some(watch) = &data_watch { + watch.close().await; + } + return Err(close_both(e).await); + } + } + connection_check = [ + connection_check_supported(&catalog).await, + connection_check_supported(&data).await, + ]; + } + Ok(Self { + stores: PostgresStores { + management: PostgresCatalogStore::new(catalog.clone()), + catalog, + data, + data_watch, + lock, + application, + }, + catalog_name, + data_name, + data_connection_row: target.data_connection_row.clone(), + read_only, + connection_check, + }) + } + + /// Close the runner's connections. Callers close them on every exit path. + /// A status session's read-only transaction is rolled back first, so a + /// pooler can hand its backend to the next client. + pub async fn close(self) { + if self.read_only { + let _ = exec(&self.stores.catalog, "ROLLBACK").await; + let _ = exec(&self.stores.data, "ROLLBACK").await; + } + self.stores.catalog.close().await; + self.stores.data.close().await; + if let Some(watch) = self.stores.data_watch { + watch.close().await; + } + } + + /// One attempt: the body and the rows in one transaction. + async fn in_transaction( + &self, + store: &StoreRef, + id: MigrationId, + step: &Step<'_, PostgresStores>, + row: Option, + catalog_version: Option, + ) -> Result { + let pool = self.stores.pool(store.kind); + in_transaction( + exec(pool, "BEGIN"), + self.body_and_rows(store, id, step, row, catalog_version), + exec(pool, "COMMIT"), + async { + let _ = exec(pool, "ROLLBACK").await; + }, + ) + .await + } + + async fn body_and_rows( + &self, + store: &StoreRef, + id: MigrationId, + step: &Step<'_, PostgresStores>, + row: Option, + catalog_version: Option, + ) -> Result { + let pool = self.stores.pool(store.kind); + exec(pool, &self.transaction_settings(store.kind)).await?; + let txid = current_txid(pool).await?; + let mut done = match step { + Step::File { name, text } => { + sqlx::raw_sql(text) + .execute(pool) + .await + .map_err(|e| Failure::sqlx(&format!("migration {id} ({name})"), &e))?; + Done::default() + } + Step::Code(body) => body.run(&self.stores).await?, + Step::Record => Done::default(), + }; + if id == MigrationId::Baseline + && store.kind == StoreKind::Catalog + && let Some(connection) = &self.data_connection_row + { + for (key, value) in [ + ("data_database_connection_string", connection.as_str()), + ("data_database_name", self.data_name.as_str()), + ] { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ($1, $2) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(key) + .bind(value) + .execute(pool) + .await + .map_err(|e| Failure::sqlx(&format!("record {key}"), &e))?; + } + } + if current_txid(pool).await? != txid { + return Err(Failure::Other(MigrationError::Store(format!( + "migration {id} committed the runner's transaction itself, so its changes and its \ + ledger row are no longer one commit; the body must not open or end transactions" + )))); + } + // The first application only: on a store at a later migration the + // objects migration 1 creates may carry what later migrations added. + if id == MigrationId::Baseline && row.is_some() { + let expected = match store.kind { + StoreKind::Catalog => super::CATALOG_EXPECTED, + StoreKind::Data | StoreKind::Account => super::DATA_EXPECTED, + }; + done.notes + .extend(super::postcheck::check(pool, expected, &store.name).await?); + } + record_rows(&Rows(pool), row, &done.notes, catalog_version).await?; + Ok(done) + } + + /// The `SET LOCAL` statements every runner transaction starts with: the + /// lock timeout, no notices (every `IF NOT EXISTS` that finds its object + /// would print one through the log), and, where the server supports it, + /// the check that ends a statement whose client is gone. + fn transaction_settings(&self, kind: StoreKind) -> String { + let mut settings = format!( + "SET LOCAL lock_timeout = '{LOCK_TIMEOUT_SECS}s'; \ + SET LOCAL client_min_messages = warning" + ); + let check = match kind { + StoreKind::Catalog => self.connection_check[0], + StoreKind::Data | StoreKind::Account => self.connection_check[1], + }; + if check { + settings.push_str("; "); + settings.push_str(CONNECTION_CHECK); + } + settings + } + + /// A body that runs outside a transaction, then its rows. + async fn autocommit( + &self, + store: &StoreRef, + id: MigrationId, + step: &Step<'_, PostgresStores>, + row: Option, + catalog_version: Option, + ) -> Result { + let Step::Code(body) = step else { + return Err(MigrationError::Store(format!( + "migration {id} runs outside a transaction and has no code body" + ))); + }; + let done = body.run(&self.stores).await?; + self.stores.check_lock().await?; + let pool = self.stores.pool(store.kind); + let rows = async { + exec(pool, "BEGIN").await?; + exec(pool, &self.transaction_settings(store.kind)).await?; + record_rows(&Rows(pool), row, &done.notes, catalog_version).await?; + exec(pool, "COMMIT").await + }; + if let Err(e) = rows.await { + let _ = exec(pool, "ROLLBACK").await; + return Err(e.into_error()); + } + Ok(done) + } +} + +#[async_trait] +impl MigrationBackend for PostgresMigrations { + type Stores = PostgresStores; + + fn backend_name(&self) -> &'static str { + "postgres" + } + + fn migration(&self, id: MigrationId) -> Migration { + super::migration(id) + } + + fn released(&self) -> &'static [Released] { + super::RELEASED + } + + fn stores(&self) -> &PostgresStores { + &self.stores + } + + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError> { + Ok(match kind { + StoreKind::Catalog => vec![self.catalog_name.clone()], + StoreKind::Data => vec![self.data_name.clone()], + StoreKind::Account => Vec::new(), + }) + } + + async fn read_ledger(&self, store: &StoreRef) -> Result, MigrationError> { + let pool = self.stores.pool(store.kind); + if !relation_exists(pool, "extenddb_migrations").await? { + return Ok(Vec::new()); + } + let rows: Vec> = sqlx::query_as( + "SELECT id, name, kind, checksum, outcome, notes, applied_at::text, applied_by \ + FROM extenddb_migrations ORDER BY id", + ) + .fetch_all(pool) + .await + .map_err(|e| sql_error(&format!("read extenddb_migrations in {}", store.name), &e))?; + rows.into_iter() + .map(|columns| LedgerRow::from_columns(Some(&store.name), columns)) + .collect() + } + + async fn read_traces(&self) -> Result { + let pool = &self.stores.catalog; + if !relation_exists(pool, "settings").await? { + return Ok(CatalogTraces::default()); + } + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT key, value FROM settings \ + WHERE key IN ('catalog_version', 'resource_tag_migration')", + ) + .fetch_all(pool) + .await + .map_err(|e| sql_error("read settings", &e))?; + Ok(CatalogTraces::from_settings(&rows)) + } + + async fn catalog_is_empty(&self) -> Result { + let empty: bool = sqlx::query_scalar( + "SELECT NOT EXISTS (SELECT 1 FROM pg_class c \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p'))", + ) + .fetch_one(&self.stores.catalog) + .await + .map_err(|e| MigrationError::store("count tables", e))?; + Ok(empty) + } + + async fn before_first_write( + &self, + _from: Option, + _to: u32, + ) -> Result, MigrationError> { + self.stores.check_lock().await?; + Ok(Vec::new()) + } + + async fn apply( + &self, + store: &StoreRef, + id: MigrationId, + step: Step<'_, PostgresStores>, + row: Option, + catalog_version: Option, + ) -> Result { + self.stores.check_lock().await?; + if matches!(step, Step::Code(_)) && runs_in(id) == RunsIn::Autocommit { + return self + .autocommit(store, id, &step, row, catalog_version) + .await; + } + let mut retries = 0; + loop { + match self + .in_transaction(store, id, &step, row.clone(), catalog_version) + .await + { + Ok(done) => return Ok(done), + Err(Failure::LockTimeout(e)) if retries < LOCK_RETRIES => { + retries += 1; + tracing::warn!( + "migration {id}: a lock wait in {} passed {LOCK_TIMEOUT_SECS} s \ + ({e}); running it again, {retries} of {LOCK_RETRIES}", + store.name + ); + self.stores.check_lock().await?; + } + Err(Failure::LockTimeout(e)) => { + return Err(lock_timeout_stop( + self.stores.pool(store.kind), + store, + id, + &e.to_string(), + LOCK_TIMEOUT_SECS, + LOCK_RETRIES, + ) + .await); + } + Err(Failure::Privilege(e)) => { + return Err(ownership_stop( + self.stores.pool(store.kind), + store, + id, + &e.to_string(), + ) + .await); + } + Err(Failure::Other(e)) => return Err(e), + } + } + } + + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError> { + Rows(&self.stores.catalog) + .write_version(version) + .await + .map_err(Failure::into_error) + } + + async fn delete_ledger_row(&self, store: &StoreRef, id: u32) -> Result<(), MigrationError> { + sqlx::query("DELETE FROM extenddb_migrations WHERE id = $1") + .bind(i32::try_from(id).unwrap_or(i32::MAX)) + .execute(self.stores.pool(store.kind)) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store(format!("forget migration {id}"), e)) + } +} diff --git a/crates/storage-postgres/src/migrations/commands.rs b/crates/storage-postgres/src/migrations/commands.rs new file mode 100644 index 000000000..dc7822f49 --- /dev/null +++ b/crates/storage-postgres/src/migrations/commands.rs @@ -0,0 +1,350 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The PostgreSQL backend's [`CatalogMigrations`], which `init`, `migrate` +//! and `verify` call through the installed backend, and the target the +//! server factory reads before it serves. +//! +//! A command's [`Target`] gives the catalog database's connection string +//! and the values of the command's flags. The data database is the target's +//! data store (`--data-db`, or the name `init` resolved) when given, else +//! the catalog's `data_database_name` row, else the catalog database's name +//! without its `_catalog` suffix. `migrate --yes` connects as an admin role +//! only for the pgvector attempt and only when `--pg-user` is given; `init` +//! always has one, the role it created the databases with. + +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + AdminRole, CatalogMigrations, CatalogStatus, HeldLock, LedgerRow, Listed, MigrationError, + MigrationId, RunKind, RunReport, Runner, StatusNote, StoreKind, Target, +}; +use sqlx::postgres::PgConnectOptions; + +use super::backend::{OpenError, PostgresMigrations}; +use super::connect::{missing_database, one_connection, relation_exists, sql_error}; +use super::sessions::denied_with_notes; +use super::{MigrationLock, PostgresTarget, catalog_status, migrate_holding}; + +/// SQLSTATE `insufficient_privilege`. +const INSUFFICIENT_PRIVILEGE: &str = "42501"; + +/// How long [`CatalogMigrations::data_store_name`] waits for the catalog. +const NAME_LOOKUP: Duration = Duration::from_secs(2); + +/// The PostgreSQL backend's catalog migrations for the commands. +pub struct PostgresCatalogMigrations; + +/// What the migration lock carries for [`CatalogMigrations::run`] and +/// [`CatalogMigrations::forget`]: the lock, the resolved databases, and the +/// admin role for the pgvector attempt. +struct Held { + lock: MigrationLock, + target: PostgresTarget, + admin: Option, +} + +#[async_trait] +impl CatalogMigrations for PostgresCatalogMigrations { + fn listed(&self, id: MigrationId) -> Listed { + Listed::of(&super::migration(id)) + } + + fn backup_advice(&self, target: &Target) -> String { + let catalog = catalog_options(target).ok(); + let name = catalog + .as_ref() + .and_then(PgConnectOptions::get_database) + .unwrap_or("postgres"); + match target.data_store.as_deref() { + Some(data) => format!( + "back up both PostgreSQL databases, the catalog database {name} and the data \ + database {data}" + ), + None => format!( + "back up both PostgreSQL databases, the catalog database {name} and its data \ + database" + ), + } + } + + /// `--data-db` when given, else the data database [`resolve`] finds: + /// the catalog's `data_database_name` row, or the catalog's name without + /// its `_catalog` suffix. The read is bounded by [`NAME_LOOKUP`], since + /// the name only completes the backup clause and an unreachable catalog + /// is reported by the status read that follows. + async fn data_store_name(&self, target: &Target) -> Option { + if target.data_store.is_some() { + return target.data_store.clone(); + } + tokio::time::timeout(NAME_LOOKUP, resolve(target)) + .await + .ok()? + .ok()? + .map(|resolved| resolved.data_database) + } + + async fn status(&self, target: &Target) -> Result { + let status = async { + match resolve(target).await? { + Some(target) => catalog_status(&target).await, + None => Ok(CatalogStatus::Uninitialized), + } + }; + explain_denied(target, status.await).await + } + + async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError> { + let rows = async { + let Some(target) = resolve(target).await? else { + return Ok(Vec::new()); + }; + let migrations = match PostgresMigrations::open_for_status(&target).await { + Ok(m) => m, + Err(OpenError::NoCatalogDatabase) => return Ok(Vec::new()), + Err(OpenError::Failed(e)) => return Err(e), + }; + let rows = Runner::new(&migrations, "").catalog_ledger().await; + migrations.close().await; + rows + }; + explain_denied(target, rows.await).await + } + + async fn lock(&self, target: &Target, timeout: Duration) -> Result { + let catalog = catalog_options(target)?; + let lock = MigrationLock::acquire(&catalog, timeout).await?; + let resolved = match resolve(target).await { + Ok(Some(resolved)) => Ok(resolved), + Ok(None) => Err(no_catalog_database(&catalog)), + Err(e) => Err(e), + }; + let mut resolved = match resolved { + Ok(resolved) => resolved, + Err(e) => { + lock.release().await; + return Err(e); + } + }; + // `init` hands over the data database's connection string its + // bootstrapper built; migration 1 records it with the catalog rows. + if let Some(row) = &target.data_connection { + resolved = resolved.with_data_connection_row(row.clone()); + } + Ok(HeldLock::new( + target.location.as_str(), + Held { + lock, + target: resolved, + admin: target.admin.clone(), + }, + )) + } + + async fn run( + &self, + lock: &HeldLock, + release: &str, + kind: RunKind, + ) -> Result { + let held = held(lock)?; + // Beside the ledger, on every run: an operator who installs pgvector + // after `init` gets it through the next `migrate --yes`. It reaches + // the data database the server serves, as the runner does; when that + // cannot be read, the catalog's server with the data database's + // name, and the runner then reports why. + let data = super::served_data_options(&held.target) + .await + .unwrap_or_else(|_| held.target.data()); + // The admin role connects to the data database's server, which the + // row may name apart from the catalog's. + let admin = held.admin.as_ref().map(|admin| admin_options(admin, &data)); + let vector = super::try_create_vector_extension(&data, admin.as_ref()).await; + let mut report = match kind { + RunKind::Create | RunKind::Migrate => { + migrate_holding(&held.target, release, &held.lock).await? + } + RunKind::Reconverge => { + let migrations = PostgresMigrations::open(&held.target, &held.lock).await?; + let result = Runner::new(&migrations, release).reconverge().await; + migrations.close().await; + result? + } + }; + report.notes.insert(0, vector); + Ok(report) + } + + async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError> { + let held = held(lock)?; + let migrations = PostgresMigrations::open(&held.target, &held.lock).await?; + let result = Runner::new(&migrations, "").forget(id).await; + migrations.close().await; + result + } + + async fn status_notes(&self, target: &Target) -> Result, MigrationError> { + let notes = async { + match resolve(target).await? { + Some(target) => super::status_notes(&target).await, + None => Ok(Vec::new()), + } + }; + explain_denied(target, notes.await).await + } +} + +/// A read the application role may not make (SQLSTATE 42501) answered with +/// the ownership note of each database it reaches: the catalog database, +/// and the data database the target resolves to. When the data database's +/// name cannot be read, it is not guessed from the catalog's, since a +/// database of the guessed name may be another application's: the notes +/// say how to see the data database's note instead. +async fn explain_denied( + target: &Target, + read: Result, +) -> Result { + let error = match read { + Err(e) if e.sqlstate() == Some(INSUFFICIENT_PRIVILEGE) => e, + other => return other, + }; + let Ok(catalog) = catalog_options(target) else { + return Err(error); + }; + let mut databases = vec![(StoreKind::Catalog, catalog)]; + let resolved = resolve(target).await; + if let Ok(Some(resolved)) = &resolved { + let data = super::served_data_options(resolved) + .await + .unwrap_or_else(|_| resolved.data()); + databases.push((StoreKind::Data, data)); + } + let data_unnamed = resolved.is_err(); + Err(denied_with_notes(error, &databases, data_unnamed, Duration::from_secs(10)).await) +} + +fn held(lock: &HeldLock) -> Result<&Held, MigrationError> { + lock.get::().ok_or_else(|| { + MigrationError::Refused("the migration lock was not taken by the PostgreSQL backend".into()) + }) +} + +fn catalog_options(target: &Target) -> Result { + target.location.parse::().map_err(|e| { + MigrationError::Refused(format!( + "the catalog connection string is not a PostgreSQL URL: {e}" + )) + }) +} + +fn no_catalog_database(catalog: &PgConnectOptions) -> MigrationError { + MigrationError::Store(format!( + "the catalog database {} does not exist; create the deployment with 'extenddb init'", + catalog.get_database().unwrap_or("postgres") + )) +} + +/// The databases `target` names, or `None` when its catalog database does +/// not exist. Reads the `data_database_name` row on a read-only connection. +/// +/// # Errors +/// +/// The connection string is not a PostgreSQL URL, or the catalog cannot be +/// reached or read. +pub async fn resolve(target: &Target) -> Result, MigrationError> { + let catalog = catalog_options(target)?; + let catalog_db = catalog.get_database().unwrap_or("postgres").to_owned(); + if let Some(data) = &target.data_store { + // Given on the command line (`--data-db`, or the name `init` + // resolved), so it wins over the database the catalog's connection + // string row names, on that row's server. + return Ok(Some( + PostgresTarget::new(catalog, data.clone()).with_given_data_database(), + )); + } + let fallback = crate::bootstrapper::derive_data_db_name(&catalog_db); + let pool = match one_connection(catalog.clone(), true, Duration::from_secs(10)).await { + Ok(pool) => pool, + Err(e) if missing_database(&e) => return Ok(None), + Err(e) => { + return Err(MigrationError::store( + format!("connect to the catalog database {catalog_db}"), + e, + )); + } + }; + let recorded = async { + if !relation_exists(&pool, "settings").await? { + return Ok(None); + } + sqlx::query_scalar::<_, String>( + "SELECT value FROM settings WHERE key = 'data_database_name'", + ) + .fetch_optional(&pool) + .await + .map_err(|e| sql_error("read data_database_name", &e)) + } + .await; + pool.close().await; + let data = recorded?.unwrap_or(fallback); + Ok(Some(PostgresTarget::new(catalog, data))) +} + +/// The admin role's connection on the server of `data`, the data database +/// the server serves. +fn admin_options(admin: &AdminRole, data: &PgConnectOptions) -> PgConnectOptions { + let options = PgConnectOptions::new() + .host(data.get_host()) + .port(data.get_port()) + .username(&admin.user); + match &admin.password { + Some(password) => options.password(password), + None => options, + } +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::PostgresCatalogMigrations; + + #[test] + fn the_backup_advice_names_both_databases() { + let mut target = Target::at("postgresql://u:p@localhost:5432/acme_catalog"); + let advice = PostgresCatalogMigrations.backup_advice(&target); + assert_eq!( + advice, + "back up both PostgreSQL databases, the catalog database acme_catalog and its data \ + database" + ); + target.data_store = Some("acme_items".into()); + let advice = PostgresCatalogMigrations.backup_advice(&target); + assert!( + advice.ends_with("and the data database acme_items"), + "{advice}" + ); + assert!(!advice.contains(":p@"), "{advice}"); + } + + /// A catalog that accepts the connection and never answers gives no + /// name, after the lookup's bound and not the connect timeout of 10 s. + #[tokio::test] + async fn a_catalog_that_does_not_answer_gives_no_data_database_name() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a listener"); + let port = listener.local_addr().expect("its address").port(); + let target = Target::at(format!("postgresql://u:p@127.0.0.1:{port}/acme_catalog")); + let started = std::time::Instant::now(); + assert_eq!( + PostgresCatalogMigrations.data_store_name(&target).await, + None + ); + let took = started.elapsed(); + assert!( + took >= super::NAME_LOOKUP && took < std::time::Duration::from_secs(6), + "{took:?}" + ); + drop(listener); + } +} diff --git a/crates/storage-postgres/src/migrations/connect.rs b/crates/storage-postgres/src/migrations/connect.rs new file mode 100644 index 000000000..ca579425f --- /dev/null +++ b/crates/storage-postgres/src/migrations/connect.rs @@ -0,0 +1,115 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Connections the runner, the locks and the status open. +//! +//! No session sets a parameter in the startup packet other than +//! `application_name`, which PgBouncer tracks and passes on: a connection +//! pooler such as PgBouncer refuses a login whose `options` names a +//! parameter it does not track. Every setting is sent with `SET LOCAL` inside the +//! transaction that needs it, so a transaction-mode pooler applies it to the +//! backend that runs the transaction, and it ends with that transaction. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use extenddb_storage::migrations::MigrationError; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, PgPool}; + +/// `application_name` of the status's read-only sessions. +pub(crate) const STATUS_APPLICATION: &str = "extenddb status"; + +/// Prefix of the `application_name` of a runner's sessions, which apply the +/// bodies. Each runner adds its own suffix, so a later migrator can tell an +/// earlier runner's sessions from its own. +pub(crate) const RUNNER_APPLICATION_PREFIX: &str = "extenddb migrate run "; + +/// A new runner session name, unique to one runner. +pub(crate) fn runner_application() -> String { + let id = uuid::Uuid::new_v4().simple().to_string(); + format!("{RUNNER_APPLICATION_PREFIX}{}", &id[..12]) +} + +/// One connection as a pool, opened once. A second connect would mean the +/// first was lost, together with any open transaction, and statements would +/// then commit one by one, so it is refused. +/// +/// `read_only` is the status's session: it is named [`STATUS_APPLICATION`] +/// and opens one `READ ONLY` transaction at connect, so nothing it runs can +/// write; closing the connection ends it. Otherwise the session keeps the +/// `application_name` of `options` and runs in autocommit until the runner +/// sends `BEGIN`. +pub(crate) async fn one_connection( + options: PgConnectOptions, + read_only: bool, + timeout: Duration, +) -> Result { + let options = if read_only { + options.application_name(STATUS_APPLICATION) + } else { + options + }; + // `migrate` prints a line for every wait; sqlx's slow-statement warnings + // would print SQL, with terminal colour codes, between them. + let options = options.disable_statement_logging(); + let opened = Arc::new(AtomicUsize::new(0)); + let pool = PgPoolOptions::new() + .max_connections(1) + .min_connections(1) + .idle_timeout(None) + .max_lifetime(None) + .acquire_timeout(timeout) + .test_before_acquire(false) + .after_connect(move |_conn, _meta| { + let opened = Arc::clone(&opened); + Box::pin(async move { + if opened.fetch_add(1, Ordering::SeqCst) > 0 { + return Err(sqlx::Error::Protocol( + "the migration connection was lost and is not reopened".to_owned(), + )); + } + Ok(()) + }) + }) + .connect_with(options) + .await?; + if read_only { + // The pool holds one connection that is never reopened, so the + // transaction lasts until the pool closes. + if let Err(e) = sqlx::raw_sql("BEGIN READ ONLY").execute(&pool).await { + pool.close().await; + return Err(e); + } + } + Ok(pool) +} + +/// The database named in the connect options does not exist (SQLSTATE +/// 3D000). +pub(crate) fn missing_database(e: &sqlx::Error) -> bool { + sqlstate(e).as_deref() == Some("3D000") +} + +/// The SQLSTATE of a database error. +pub(crate) fn sqlstate(e: &sqlx::Error) -> Option { + e.as_database_error() + .and_then(|d| d.code().map(|c| c.into_owned())) +} + +/// The migration error for a failed statement, keeping its SQLSTATE when +/// the database reported one. Code bodies and the runner map every +/// `sqlx::Error` through this, so the runner can read the code. +pub(crate) fn sql_error(context: &str, e: &sqlx::Error) -> MigrationError { + MigrationError::coded(context, e, sqlstate(e)) +} + +/// Whether a table of this name exists in the public schema. +pub(crate) async fn relation_exists(pool: &PgPool, name: &str) -> Result { + sqlx::query_scalar("SELECT to_regclass($1) IS NOT NULL") + .bind(format!("public.\"{name}\"")) + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store(format!("look up {name}"), e)) +} diff --git a/crates/storage-postgres/src/migrations/failure.rs b/crates/storage-postgres/src/migrations/failure.rs new file mode 100644 index 000000000..3d23f9885 --- /dev/null +++ b/crates/storage-postgres/src/migrations/failure.rs @@ -0,0 +1,101 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How one attempt at a body fails, and the runner statements that report a +//! failure that way. +//! +//! The runner classifies an error by the SQLSTATE it carries +//! ([`MigrationError::sqlstate`]), which every code body and runner +//! statement keeps through [`super::connect::sql_error`]. Its text is never +//! read: a server whose `lc_messages` names another language translates it. + +use extenddb_storage::migrations::MigrationError; +use sqlx::PgPool; + +use super::connect::sql_error; +use crate::data::SQLSTATE_LOCK_NOT_AVAILABLE as LOCK_NOT_AVAILABLE; + +/// SQLSTATE `insufficient_privilege`, which altering a table the role does +/// not own raises. +const INSUFFICIENT_PRIVILEGE: &str = "42501"; + +/// Why one attempt at a body failed. +pub(super) enum Failure { + /// A lock wait passed `lock_timeout`: the body runs again. + LockTimeout(MigrationError), + /// The role may not alter an object: the stop names the owners. + Privilege(MigrationError), + Other(MigrationError), +} + +impl Failure { + pub(super) fn sqlx(context: &str, e: &sqlx::Error) -> Self { + sql_error(context, e).into() + } + + /// The error the attempt failed with. + pub(super) fn into_error(self) -> MigrationError { + match self { + Self::LockTimeout(e) | Self::Privilege(e) | Self::Other(e) => e, + } + } +} + +impl From for Failure { + fn from(e: MigrationError) -> Self { + match e.sqlstate() { + Some(LOCK_NOT_AVAILABLE) => Self::LockTimeout(e), + Some(INSUFFICIENT_PRIVILEGE) => Self::Privilege(e), + _ => Self::Other(e), + } + } +} + +pub(super) async fn exec(pool: &PgPool, sql: &str) -> Result<(), Failure> { + sqlx::raw_sql(sql) + .execute(pool) + .await + .map(|_| ()) + .map_err(|e| Failure::sqlx(sql, &e)) +} + +pub(super) async fn current_txid(pool: &PgPool) -> Result { + sqlx::query_scalar("SELECT txid_current()") + .fetch_one(pool) + .await + .map_err(|e| Failure::sqlx("read the transaction id", &e)) +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::MigrationError; + + use super::Failure; + + fn coded(sqlstate: &str) -> MigrationError { + MigrationError::coded( + "a statement", + "an error in any language", + Some(sqlstate.into()), + ) + } + + #[test] + fn a_failure_is_classified_by_its_sqlstate() { + assert!(matches!( + Failure::from(coded("55P03")), + Failure::LockTimeout(_) + )); + assert!(matches!( + Failure::from(coded("42501")), + Failure::Privilege(_) + )); + assert!(matches!(Failure::from(coded("23505")), Failure::Other(_))); + // The text of a lock timeout without its code is not one. + let text = MigrationError::Store("canceling statement due to lock timeout".into()); + assert!(matches!(Failure::from(text), Failure::Other(_))); + // A stopped run keeps its error's code. + let failure = Failure::from(coded("55P03")).into_error(); + assert_eq!(failure.sqlstate(), Some("55P03")); + } +} diff --git a/crates/storage-postgres/src/migrations/held.rs b/crates/storage-postgres/src/migrations/held.rs new file mode 100644 index 000000000..627f9225f --- /dev/null +++ b/crates/storage-postgres/src/migrations/held.rs @@ -0,0 +1,242 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A connection whose open transaction holds an advisory key, and its +//! renewals: `SELECT 1` every keepalive interval, each waiting at most two +//! thirds of the interval for its answer, and the key counted as lost after +//! twice the interval with no successful renewal. + +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::MigrationError; +use sqlx::postgres::PgConnectOptions; +use sqlx::{ConnectOptions, Connection, PgConnection}; +use tokio::sync::{Mutex, watch}; +use tokio::task::JoinHandle; + +/// How long a lock or registration connection may take to open. +pub(super) const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long one renewal waits for its answer: two thirds of the interval +/// (10 s for 15 s), and at least 1 s. +fn ping_timeout(interval: Duration) -> Duration { + (interval * 2 / 3).max(Duration::from_secs(1)) +} + +/// How long a holder goes without a successful renewal before its lock +/// counts as lost: twice the interval (30 s for 15 s), and at least 2 s. +fn renewal_deadline(interval: Duration) -> Duration { + (interval * 2).max(Duration::from_secs(2)) +} + +/// A connection whose open transaction holds a key. +pub(super) struct Held { + pub(super) conn: Mutex>, + lost: StdMutex>, + pub(super) what: &'static str, + /// What the holder does once the key is lost, appended to the reason: + /// for the migration lock "; nothing more is applied", and nothing for + /// the serving registration, whose server decides. + then: &'static str, + pub(super) ping_timeout: Duration, +} + +impl Held { + pub(super) fn new( + conn: PgConnection, + what: &'static str, + then: &'static str, + interval: Duration, + ) -> Arc { + Arc::new(Self { + conn: Mutex::new(Some(conn)), + lost: StdMutex::new(None), + what, + then, + ping_timeout: ping_timeout(interval), + }) + } + + fn lost_reason(&self) -> Option { + self.lost.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + fn mark_lost(&self, reason: String) -> MigrationError { + let mut lost = self.lost.lock().unwrap_or_else(|e| e.into_inner()); + let reason = lost.get_or_insert(reason).clone(); + MigrationError::Store(reason) + } + + /// `SELECT 1` on the connection, waiting at most `within`. A failed + /// statement marks the connection lost, and every later call returns that + /// reason; a statement with no answer in time returns + /// [`Renewal::NoAnswer`] and marks nothing. + async fn renew(&self, within: Duration) -> Renewal { + if let Some(reason) = self.lost_reason() { + return Renewal::Lost(MigrationError::Store(reason)); + } + match tokio::time::timeout(within, select_one(&self.conn)).await { + Ok(Ok(())) => Renewal::Renewed, + Ok(Err(None)) => Renewal::Lost(MigrationError::Store(format!( + "the {} was released", + self.what + ))), + Ok(Err(Some(e))) => Renewal::Lost(self.mark_lost(format!( + "the {}'s connection was lost ({e}), so another process may hold it now{}", + self.what, self.then + ))), + Err(_) => Renewal::NoAnswer, + } + } + + /// One renewal for a caller about to act under the lock: no answer in + /// time counts as lost, because the caller cannot tell it still holds + /// the key. + pub(super) async fn ping(&self) -> Result<(), MigrationError> { + match self.renew(self.ping_timeout).await { + Renewal::Renewed => Ok(()), + Renewal::Lost(e) => Err(e), + Renewal::NoAnswer => Err(self.mark_lost(format!( + "the {}'s connection did not answer within {} s, so this process cannot tell \ + that it still holds it{}", + self.what, + self.ping_timeout.as_secs_f32(), + self.then + ))), + } + } + + /// Roll back and close, waiting at most the ping timeout for each: a + /// database that does not answer has the connection dropped instead. + pub(super) async fn release(&self) { + let conn = match tokio::time::timeout(self.ping_timeout, self.conn.lock()).await { + Ok(mut guard) => guard.take(), + Err(_) => None, + }; + if let Some(conn) = conn { + let _ = tokio::time::timeout(self.ping_timeout * 2, rollback_and_close(conn)).await; + } + } +} + +/// `SELECT 1` on the held connection; `Err(None)` once it was released. +async fn select_one(conn: &Mutex>) -> Result<(), Option> { + let mut guard = conn.lock().await; + let Some(conn) = guard.as_mut() else { + return Err(None); + }; + statement(conn, "SELECT 1").await.map_err(Some) +} + +async fn rollback_and_close(mut conn: PgConnection) { + if statement(&mut conn, "ROLLBACK").await.is_ok() { + let _ = conn.close().await; + } +} + +/// The outcome of one renewal. +enum Renewal { + Renewed, + /// The statement failed, or the key was released. + Lost(MigrationError), + /// No answer within the ping timeout. + NoAnswer, +} + +/// Renew every `interval`. A renewal that fails, or no successful renewal +/// for the renewal deadline, ends the task and reports the reason on `lost`. +/// After a renewal with no answer the next one starts at once. +pub(super) fn spawn_keepalive( + held: Arc, + interval: Duration, + lost: Option>>, +) -> JoinHandle<()> { + let deadline = renewal_deadline(interval); + tokio::spawn(async move { + let mut last_renewed = Instant::now(); + let mut wait = interval; + loop { + tokio::time::sleep(wait).await; + let left = deadline.saturating_sub(last_renewed.elapsed()); + let within = held.ping_timeout.min(left).max(Duration::from_millis(1)); + let reason = match held.renew(within).await { + Renewal::Renewed => { + last_renewed = Instant::now(); + wait = interval; + continue; + } + Renewal::NoAnswer if last_renewed.elapsed() < deadline => { + wait = Duration::ZERO; + continue; + } + Renewal::NoAnswer => held + .mark_lost(format!( + "the {} could not be renewed for {} s: the database has not answered \ + its connection since, so the database may have freed it", + held.what, + last_renewed.elapsed().as_secs() + )) + .to_string(), + Renewal::Lost(e) => e.to_string(), + }; + if let Some(lost) = &lost { + let _ = lost.send(Some(reason)); + } + return; + } + }) +} + +pub(super) async fn connect( + options: &PgConnectOptions, + app: &str, +) -> Result { + let what = "connect to the catalog database to take a lock"; + // The holder prints its own lines; sqlx's slow-statement warning for a + // lock wait would print SQL with terminal colour codes. + let options = options + .clone() + .application_name(app) + .disable_statement_logging(); + match tokio::time::timeout(CONNECT_TIMEOUT, PgConnection::connect_with(&options)).await { + Ok(Ok(conn)) => Ok(conn), + Ok(Err(e)) => Err(MigrationError::store(what, e)), + Err(_) => Err(MigrationError::Store(format!( + "{what}: no answer within {} s", + CONNECT_TIMEOUT.as_secs() + ))), + } +} + +pub(super) async fn statement(conn: &mut PgConnection, sql: &str) -> Result<(), sqlx::Error> { + sqlx::query(sql).execute(&mut *conn).await.map(|_| ()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{ping_timeout, renewal_deadline}; + + /// At the production interval of 15 s a renewal waits 10 s for its + /// answer, and the key counts as lost 30 s after the last renewal; the + /// floors hold for the 1 s interval tests use. + #[test] + fn the_production_interval_gives_a_30_s_deadline() { + let interval = super::super::KEEPALIVE_INTERVAL; + assert_eq!(interval, Duration::from_secs(15)); + assert_eq!(ping_timeout(interval), Duration::from_secs(10)); + assert_eq!(renewal_deadline(interval), Duration::from_secs(30)); + assert_eq!(ping_timeout(Duration::from_secs(3)), Duration::from_secs(2)); + assert_eq!( + renewal_deadline(Duration::from_secs(3)), + Duration::from_secs(6) + ); + assert_eq!(ping_timeout(Duration::from_secs(1)), Duration::from_secs(1)); + assert_eq!( + renewal_deadline(Duration::from_secs(1)), + Duration::from_secs(2) + ); + } +} diff --git a/crates/storage-postgres/src/migrations/index_helper.rs b/crates/storage-postgres/src/migrations/index_helper.rs new file mode 100644 index 000000000..5115f852f --- /dev/null +++ b/crates/storage-postgres/src/migrations/index_helper.rs @@ -0,0 +1,251 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The index helper for bodies that build indexes outside a transaction. +//! +//! `CREATE INDEX CONCURRENTLY` commits the index's catalog entry before it +//! builds, so a build that is cancelled, times out or loses its connection +//! leaves the index INVALID, and `IF NOT EXISTS` then skips it on every later +//! run. The helper reads `pg_index.indisvalid` first, drops an invalid index +//! with `DROP INDEX CONCURRENTLY`, builds it with `CREATE INDEX +//! CONCURRENTLY`, and checks it is valid afterwards; [`IndexHelper::finish`] +//! checks every index the body owns again, so the body's ledger row is +//! written only when all of them are valid. Before every statement it checks +//! that the migration lock is still held. +//! +//! A statement outside a transaction cannot take `SET LOCAL lock_timeout`, +//! and a session-level setting would reach whichever backend a +//! transaction-mode pooler picks. So while a statement runs, the helper +//! watches it from a second session of the runner on the same database, +//! which reaches that database's server as the statement's role: once it +//! has waited for a lock for as long as the transactional bodies' lock +//! timeout, it is cancelled, its index is dropped and built again, three +//! times, and then the body stops naming the sessions that hold the lock. + +use std::time::Duration; + +use extenddb_storage::migrations::MigrationError; +use sqlx::PgPool; + +use super::PostgresStores; +use super::connect::sqlstate; + +/// How long a statement waits for a lock before it is cancelled: the +/// transactional bodies' `lock_timeout`. +const LOCK_WAIT: Duration = Duration::from_secs(5); + +/// How many times an index is built again after a cancelled wait. +const LOCK_RETRIES: u32 = 3; + +/// How often a running statement is looked at. +const WATCH: Duration = Duration::from_millis(250); + +/// SQLSTATE `query_canceled`. +const QUERY_CANCELED: &str = "57014"; + +/// What [`IndexHelper::ensure`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Built { + /// The index existed and was valid. + Existing, + /// The index was missing, and was built. + Created, + /// The index was invalid, and was dropped and built again. + Rebuilt, +} + +/// One index's state in `pg_index`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + Missing, + Valid, + Invalid, +} + +/// How one statement ended. +enum Ran { + Done, + /// The helper cancelled it after it waited for a lock; the sessions that + /// held the lock. + LockWait(String), +} + +/// Builds the indexes of one body in one database. +pub(crate) struct IndexHelper<'a> { + stores: &'a PostgresStores, + pool: &'a PgPool, + /// Where the statements `pool` runs are watched and cancelled from. + watch: &'a PgPool, + owned: Vec, +} + +impl<'a> IndexHelper<'a> { + pub(crate) fn new(stores: &'a PostgresStores, pool: &'a PgPool) -> Self { + Self { + stores, + pool, + watch: stores.watch_for(pool), + owned: Vec::new(), + } + } + + async fn state(&self, index: &str) -> Result { + let valid: Option = sqlx::query_scalar( + "SELECT i.indisvalid FROM pg_index i \ + WHERE i.indexrelid = to_regclass($1)", + ) + .bind(format!("public.\"{index}\"")) + .fetch_optional(self.pool) + .await + .map_err(|e| MigrationError::store(format!("read the state of index {index}"), e))?; + Ok(match valid { + None => State::Missing, + Some(true) => State::Valid, + Some(false) => State::Invalid, + }) + } + + /// The backend running `sql` for this runner when it waits for a lock, + /// and the sessions it waits for. + async fn lock_wait(&self, sql: &str) -> Result, MigrationError> { + sqlx::query_as( + "SELECT a.pid, coalesce(string_agg(format('pid %s (%s, %s)', b.pid, \ + coalesce(nullif(b.application_name, ''), 'no application name'), \ + coalesce(b.state, 'state not visible to this role')), \ + ', ' ORDER BY b.pid), 'no session found') \ + FROM pg_stat_activity a \ + LEFT JOIN LATERAL unnest(pg_blocking_pids(a.pid)) AS blocker(pid) ON TRUE \ + LEFT JOIN pg_stat_activity b ON b.pid = blocker.pid \ + WHERE a.application_name = $1 AND a.query = $2 AND a.state = 'active' \ + AND a.wait_event_type = 'Lock' \ + GROUP BY a.pid", + ) + .bind(&self.stores.application) + .bind(sql) + .fetch_optional(self.watch) + .await + .map_err(|e| MigrationError::store("read the index build's lock wait", e)) + } + + /// Run one statement, cancelling it once it has waited [`LOCK_WAIT`] for + /// a lock; `attempt` counts from 1, for the line that says so. + async fn run(&self, sql: &str, attempt: u32) -> Result { + self.stores.check_lock().await?; + let statement = sqlx::query(sql).execute(self.pool); + tokio::pin!(statement); + let mut waited = Duration::ZERO; + let mut announced = false; + let mut cancelled: Option = None; + loop { + tokio::select! { + result = &mut statement => { + return match (result, cancelled) { + (Ok(_), _) => Ok(Ran::Done), + (Err(e), Some(holders)) + if sqlstate(&e).as_deref() == Some(QUERY_CANCELED) => + { + Ok(Ran::LockWait(holders)) + } + (Err(e), _) => Err(MigrationError::store(sql, e)), + }; + } + () = tokio::time::sleep(WATCH), if cancelled.is_none() => { + let Some((pid, holders)) = self.lock_wait(sql).await? else { + waited = Duration::ZERO; + continue; + }; + waited += WATCH; + if !announced && waited >= Duration::from_secs(1) { + println!( + "--- {sql} waits for a lock held by {holders}; attempt {attempt} of \ + {} is cancelled once it has waited {} s", + LOCK_RETRIES + 1, + LOCK_WAIT.as_secs() + ); + announced = true; + } + if waited >= LOCK_WAIT { + sqlx::query("SELECT pg_cancel_backend($1)") + .bind(pid) + .execute(self.watch) + .await + .map_err(|e| MigrationError::store("cancel the index build", e))?; + cancelled = Some(holders); + } + } + } + } + } + + /// Make `index` exist and be valid: `on` is the rest of the statement + /// after the index name, such as `ON "_ddb_x" (base_pk, base_sk_s)`. The + /// names are quoted by the caller's identifiers, which must be checked + /// before they reach here. + pub(crate) async fn ensure(&mut self, index: &str, on: &str) -> Result { + self.owned.push(index.to_owned()); + let first = self.state(index).await?; + if first == State::Valid { + return Ok(Built::Existing); + } + let drop = format!("DROP INDEX CONCURRENTLY IF EXISTS \"{index}\""); + let create = format!("CREATE INDEX CONCURRENTLY IF NOT EXISTS \"{index}\" {on}"); + let mut state = first; + let mut holders = String::new(); + for attempt in 1..=LOCK_RETRIES + 1 { + if attempt > 1 { + state = self.state(index).await?; + } + let ran = match state { + State::Valid => Ran::Done, + State::Invalid => match self.run(&drop, attempt).await? { + Ran::Done => self.run(&create, attempt).await?, + waited @ Ran::LockWait(_) => waited, + }, + State::Missing => self.run(&create, attempt).await?, + }; + match ran { + Ran::Done if self.state(index).await? == State::Valid => { + return Ok(if first == State::Invalid { + Built::Rebuilt + } else { + Built::Created + }); + } + Ran::Done => { + return Err(MigrationError::Store(format!( + "index {index} is not valid after its build; the migration is not \ + recorded, and the next run drops and builds it again" + ))); + } + Ran::LockWait(held_by) => holders = held_by, + } + } + Err(MigrationError::Store(format!( + "index {index} could not be built: each of {} attempts waited more than {} s for a \ + lock, held by {holders}; nothing of this migration is recorded in the data \ + database, so stop those sessions and run again", + LOCK_RETRIES + 1, + LOCK_WAIT.as_secs() + ))) + } + + /// Check every index the body asked for is valid now. + pub(crate) async fn finish(self) -> Result<(), MigrationError> { + self.stores.check_lock().await?; + let mut invalid = Vec::new(); + for index in &self.owned { + if self.state(index).await? != State::Valid { + invalid.push(index.clone()); + } + } + if invalid.is_empty() { + Ok(()) + } else { + Err(MigrationError::Store(format!( + "indexes not valid at the end of the migration: {}; it is not recorded, and the \ + next run builds them again", + invalid.join(", ") + ))) + } + } +} diff --git a/crates/storage-postgres/src/migrations/locks.rs b/crates/storage-postgres/src/migrations/locks.rs new file mode 100644 index 000000000..3d0e1b6b9 --- /dev/null +++ b/crates/storage-postgres/src/migrations/locks.rs @@ -0,0 +1,572 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The migration lock and the serving registration, both transaction-level +//! advisory locks in the catalog database, each held by an explicit +//! transaction on a dedicated connection that stays open while it is held. +//! A transaction-pooling proxy keeps one backend for an open transaction, so +//! the lock cannot move between backends. +//! +//! - The migration lock is the `(NAMESPACE, 1)` key releases since v0.1.3 +//! take, exclusive. A migrator takes it and then the serving key +//! exclusively, waiting for another migrator and then for running servers +//! to stop, both within its one timeout. +//! - A server holds the serving key `(NAMESPACE, 2)` shared, then checks that +//! no migrator holds the migration lock. Each side takes its own key before +//! it looks for the other, so two that start together cannot both miss +//! each other. +//! +//! Each of the two transactions turns off `idle_in_transaction_session_timeout` +//! for itself, and on PostgreSQL 17 and later `transaction_timeout`, so a +//! role or database setting of either does not end the session that holds +//! the key. The holder runs `SELECT 1` on the connection every 15 s, and the +//! runner checks the connection before each migration and each statement +//! outside a transaction. Each renewal waits at most 10 s for its answer, and +//! a holder whose last renewal succeeded more than 30 s ago counts its lock +//! as lost: a database that stops answering without closing the connection, +//! such as a paused or vanished host, is noticed within that time. The +//! serving session sets server-side TCP keepalives, which watch the +//! database's connection to its peer: connected directly, the database ends +//! the session, and frees the key, within 30 s of the serving host dying +//! instead of after the operating system's default of hours. Behind +//! PgBouncer that peer is PgBouncer, whose connection stays up when the +//! serving host dies, so the key stays until PgBouncer closes the dead +//! host's connection, which its own keepalive settings decide. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::MigrationError; +use sqlx::postgres::PgConnectOptions; +use sqlx::{Connection, PgConnection}; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use super::held::{Held, connect, spawn_keepalive, statement}; + +/// ExtendDB's advisory-lock namespace, `classid` of every key. +pub(crate) const NAMESPACE: i32 = 0x0045_4442; +/// `objid` of the migration lock. +pub(crate) const MIGRATION_KEY: i32 = 1; +/// `objid` of the serving registration. +pub(crate) const SERVING_KEY: i32 = 2; + +/// How often a holder runs `SELECT 1` on its lock connection. +pub const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15); + +/// Server-side keepalives of the serving session: the first probe after 10 s +/// of silence, then one every 5 s, and the session ends after 4 unanswered. +const TCP_KEEPALIVES: [(&str, &str); 3] = [ + ("tcp_keepalives_idle", "10"), + ("tcp_keepalives_interval", "5"), + ("tcp_keepalives_count", "4"), +]; + +/// The longest wait a lock timeout can ask for that `Instant` can add. +const MAX_LOCK_WAIT: Duration = Duration::from_secs(86_400); + +/// The statement that turns off `idle_in_transaction_session_timeout` for +/// the transaction that holds a key. +const IDLE_OFF: &str = "SET LOCAL idle_in_transaction_session_timeout = 0"; + +/// The statement that turns off `transaction_timeout` (PostgreSQL 17 and +/// later) for the transaction that holds a key. +const AGE_OFF: &str = "SET LOCAL transaction_timeout = 0"; + +/// What the transaction that holds a key sends after `BEGIN`, for a server +/// whose `server_version_num` is `version`. A role or database setting of +/// `idle_in_transaction_session_timeout` shorter than the renewal interval +/// would end the session between two renewals, and `transaction_timeout`, +/// which PostgreSQL 17 added, counts from the transaction's start, so no +/// renewal resets it. +fn timeouts_off(version: i32) -> &'static [&'static str] { + if version >= 170_000 { + &[IDLE_OFF, AGE_OFF] + } else { + &[IDLE_OFF] + } +} + +/// Send [`timeouts_off`] for this server in the transaction open on `conn`. +/// They are `SET LOCAL`s, so a transaction-mode pooler applies them to the +/// backend the key lives on, and they last as long as the key is held. +async fn keep_open(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + let version: String = sqlx::query_scalar("SELECT current_setting('server_version_num')") + .fetch_one(&mut *conn) + .await?; + for sql in timeouts_off(version.trim().parse().unwrap_or(0)) { + statement(conn, sql).await?; + } + Ok(()) +} + +/// What the runner keeps of the migration lock: a way to check that it is +/// still held, and the deadline its waits share. +#[derive(Clone)] +pub(crate) struct LockHandle { + held: Arc, + deadline: Instant, +} + +impl LockHandle { + pub(crate) async fn check_alive(&self) -> Result<(), MigrationError> { + self.held.ping().await + } + + /// The time left of the `--lock-timeout` the lock was taken with, + /// counted from when it was asked for: the bound of the run's later + /// waits, so every wait of a run together stays within the timeout. + pub(crate) fn remaining(&self) -> Duration { + self.deadline.saturating_duration_since(Instant::now()) + } +} + +/// `timeout` from now, or a day when the clock cannot add it. +fn deadline_after(timeout: Duration) -> Instant { + let now = Instant::now(); + now.checked_add(timeout).unwrap_or(now + MAX_LOCK_WAIT) +} + +/// Whole seconds for a message about a wait, rounded up, so a wait with +/// time left never reads as 0 s. +fn shown_secs(wait: Duration) -> u128 { + wait.as_millis().div_ceil(1000) +} + +async fn try_key(conn: &mut PgConnection, function: &str, key: i32) -> Result { + sqlx::query_scalar(&format!("SELECT {function}($1, $2)")) + .bind(NAMESPACE) + .bind(key) + .fetch_one(&mut *conn) + .await +} + +/// The sessions holding a key in this database (advisory keys are per +/// database), for messages: pid, application, client and connection time. +/// PostgreSQL shows a role the client and connection time of another role's +/// session only with `pg_read_all_stats`; such a session is named as one of +/// that role whose details this role cannot see. +async fn holders(conn: &mut PgConnection, key: i32) -> Result, sqlx::Error> { + type Row = (i32, Option, String, Option, Option); + let rows: Vec = sqlx::query_as( + "SELECT a.pid, a.usename::text, coalesce(a.application_name, ''), host(a.client_addr), \ + to_char(a.backend_start AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') \ + FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid \ + WHERE l.locktype = 'advisory' AND l.classid = $1 AND l.objid = $2 \ + AND l.objsubid = 2 AND l.granted AND l.pid <> pg_backend_pid() \ + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) \ + ORDER BY a.pid", + ) + .bind(NAMESPACE) + .bind(key) + .fetch_all(&mut *conn) + .await?; + Ok(rows + .into_iter() + .map(|(pid, role, app, client, since)| match since { + Some(since) => format!( + "pid {pid} ({app:?} from {}, connected {since})", + client.unwrap_or_else(|| "a local socket".to_owned()) + ), + None => format!( + "pid {pid}, a session of {} ({app:?}) whose address and connection time this \ + role cannot see", + role.map_or_else(|| "another role".to_owned(), |r| format!("the role {r}")) + ), + }) + .collect()) +} + +fn listed(names: &[String]) -> String { + if names.is_empty() { + "no session holds it now".to_owned() + } else { + names.join(", ") + } +} + +/// The migration lock, held for a run. Dropping it without +/// [`MigrationLock::release`] closes the connection, which also frees it. +pub struct MigrationLock { + held: Arc, + keepalive: JoinHandle<()>, + deadline: Instant, +} + +impl MigrationLock { + /// Take the migration lock, then the serving key exclusively, and keep + /// the transaction alive every [`KEEPALIVE_INTERVAL`]. The waits for the + /// two keys, and the runner's later wait for an earlier migrator's + /// statements, share one deadline: `timeout` from this call. A key that + /// is held when no time is left, as at a zero `timeout`, is not waited + /// for. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] (exit 6) naming the holders when another + /// migrator or a running server outlasts `timeout`. + pub async fn acquire( + catalog: &PgConnectOptions, + timeout: Duration, + ) -> Result { + Self::acquire_keepalive(catalog, timeout, KEEPALIVE_INTERVAL).await + } + + /// [`MigrationLock::acquire`] with another keepalive interval, for tests. + /// + /// # Errors + /// + /// As [`MigrationLock::acquire`]. + #[cfg(feature = "test-util")] + pub async fn acquire_with( + catalog: &PgConnectOptions, + timeout: Duration, + keepalive: Duration, + ) -> Result { + Self::acquire_keepalive(catalog, timeout, keepalive).await + } + + async fn acquire_keepalive( + catalog: &PgConnectOptions, + timeout: Duration, + keepalive: Duration, + ) -> Result { + let deadline = deadline_after(timeout); + let mut conn = connect(catalog, "extenddb migrate").await?; + if let Err(e) = take_migration_keys(&mut conn, timeout, deadline).await { + let _ = conn.close().await; + return Err(e); + } + let held = Held::new( + conn, + "migration lock", + "; nothing more is applied", + keepalive, + ); + let keepalive = spawn_keepalive(Arc::clone(&held), keepalive, None); + Ok(Self { + held, + keepalive, + deadline, + }) + } + + pub(crate) fn handle(&self) -> LockHandle { + LockHandle { + held: Arc::clone(&self.held), + deadline: self.deadline, + } + } + + /// Whether the lock's connection still answers. + /// + /// # Errors + /// + /// The connection was lost. + pub async fn check_alive(&self) -> Result<(), MigrationError> { + self.held.ping().await + } + + /// Roll the lock's transaction back and close its connection. + pub async fn release(self) { + self.keepalive.abort(); + self.held.release().await; + } +} + +impl Drop for MigrationLock { + fn drop(&mut self) { + self.keepalive.abort(); + } +} + +/// One of the two keys a migrator takes, as its messages name it. +struct Key { + key: i32, + /// What holds the key, in the error after the wait. + held_by: &'static str, + /// The line printed when the run starts to wait, given the holders and + /// the seconds left. + waiting: fn(&str, u128) -> String, +} + +const MIGRATION: Key = Key { + key: MIGRATION_KEY, + held_by: "another migrator holds the migration lock", + waiting: migrator_waiting, +}; + +const SERVING: Key = Key { + key: SERVING_KEY, + held_by: "servers are serving this catalog", + waiting: servers_waiting, +}; + +fn migrator_waiting(names: &str, secs: u128) -> String { + format!("--- Another migrator holds the migration lock ({names}); waiting up to {secs} s...") +} + +fn servers_waiting(names: &str, secs: u128) -> String { + format!("--- Waiting up to {secs} s for the servers serving this catalog to stop: {names}") +} + +async fn take_migration_keys( + conn: &mut PgConnection, + timeout: Duration, + deadline: Instant, +) -> Result<(), MigrationError> { + let fail = |e: sqlx::Error| MigrationError::store("take the migration lock", e); + statement(conn, "BEGIN").await.map_err(fail)?; + keep_open(conn).await.map_err(fail)?; + take_key(conn, &MIGRATION, timeout, deadline).await?; + take_key(conn, &SERVING, timeout, deadline).await?; + let both: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND classid = $1 \ + AND objid IN ($2, $3) AND objsubid = 2 AND mode = 'ExclusiveLock' AND granted \ + AND pid = pg_backend_pid()", + ) + .bind(NAMESPACE) + .bind(MIGRATION_KEY) + .bind(SERVING_KEY) + .fetch_one(&mut *conn) + .await + .map_err(fail)?; + if both != 2 { + return Err(MigrationError::Store( + "PostgreSQL did not report the migration lock's advisory keys after taking them; \ + refusing to migrate without them" + .to_owned(), + )); + } + Ok(()) +} + +/// Take `key` exclusively in the lock's transaction: at once when it is +/// free, else by waiting for it until `deadline`, with `lock_timeout` set to +/// the time left. With no time left, as at a zero `timeout` or after an +/// earlier wait used the time up, a held key is `Locked` at once: +/// PostgreSQL reads a `lock_timeout` of 0 as no limit, and counts it in +/// whole milliseconds. +async fn take_key( + conn: &mut PgConnection, + key: &Key, + timeout: Duration, + deadline: Instant, +) -> Result<(), MigrationError> { + let fail = |e: sqlx::Error| MigrationError::store("take the migration lock", e); + if try_key(conn, "pg_try_advisory_xact_lock", key.key) + .await + .map_err(fail)? + { + return Ok(()); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.as_millis() == 0 { + return Err(still_held(conn, key, timeout).await); + } + let names = holders(conn, key.key).await.unwrap_or_default(); + println!("{}", (key.waiting)(&listed(&names), shown_secs(left))); + let _: String = sqlx::query_scalar("SELECT set_config('lock_timeout', $1, true)") + .bind(format!("{}ms", left.as_millis())) + .fetch_one(&mut *conn) + .await + .map_err(fail)?; + let waited = sqlx::query("SELECT pg_advisory_xact_lock($1, $2)") + .bind(NAMESPACE) + .bind(key.key) + .execute(&mut *conn) + .await; + match waited { + Ok(_) => Ok(()), + Err(e) if crate::data::lock_not_available(&e) => Err(still_held(conn, key, timeout).await), + Err(e) => { + let _ = statement(conn, "ROLLBACK").await; + Err(fail(e)) + } + } +} + +/// The error when `key` is still held at the deadline: the lock's +/// transaction is rolled back, and the holders are named. +async fn still_held(conn: &mut PgConnection, key: &Key, timeout: Duration) -> MigrationError { + let _ = statement(conn, "ROLLBACK").await; + let names = match holders(conn, key.key).await { + Ok(names) => listed(&names), + Err(e) => format!("(pg_locks could not be read: {e})"), + }; + MigrationError::Locked(format!( + "{}, still after {} s: {names}", + key.held_by, + timeout.as_secs() + )) +} + +/// What a server holds while it serves: the serving key, shared. A +/// migration cannot start while it is held. Once +/// [`ServingRegistration::lost`] reports the connection lost, the server +/// admits no new request until it holds a registration again +/// ([`super::ServerRegistration`]). +pub struct ServingRegistration { + held: Arc, + keepalive: JoinHandle<()>, + lost: watch::Receiver>, +} + +impl ServingRegistration { + /// Register, with the session named `extenddb serve`. The key is taken + /// with a try: a migration that holds or waits for it is never waited + /// for. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] (exit 6) when a migration is running, so a + /// restart loop retries. + pub async fn register(catalog: &PgConnectOptions) -> Result { + Self::register_keepalive(catalog, KEEPALIVE_INTERVAL).await + } + + /// [`ServingRegistration::register`] with another keepalive interval, for + /// tests. + /// + /// # Errors + /// + /// As [`ServingRegistration::register`]. + #[cfg(feature = "test-util")] + pub async fn register_with( + catalog: &PgConnectOptions, + keepalive: Duration, + ) -> Result { + Self::register_keepalive(catalog, keepalive).await + } + + pub(crate) async fn register_keepalive( + catalog: &PgConnectOptions, + keepalive: Duration, + ) -> Result { + let mut conn = connect(catalog, "extenddb serve").await?; + match register_on(&mut conn).await { + Ok(()) => {} + Err(e) => { + let _ = statement(&mut conn, "ROLLBACK").await; + let _ = conn.close().await; + return Err(e); + } + } + let held = Held::new(conn, "serving registration", "", keepalive); + let (tx, lost) = watch::channel(None); + let keepalive = spawn_keepalive(Arc::clone(&held), keepalive, Some(tx)); + Ok(Self { + held, + keepalive, + lost, + }) + } + + /// Changes to `Some(reason)` when the registration's connection is lost. + #[must_use] + pub fn lost(&self) -> watch::Receiver> { + self.lost.clone() + } + + /// Whether the registration's connection still answers. + /// + /// # Errors + /// + /// The connection was lost. + pub async fn check_alive(&self) -> Result<(), MigrationError> { + self.held.ping().await + } + + /// The serving session's `application_name` and TCP keepalive settings, + /// by name, as the database reports them, for diagnostics. + /// + /// # Errors + /// + /// The connection was lost or released. + #[cfg(feature = "test-util")] + pub async fn session_settings(&self) -> Result, MigrationError> { + let mut guard = self.held.conn.lock().await; + let Some(conn) = guard.as_mut() else { + return Err(MigrationError::Store( + "the serving registration was released".to_owned(), + )); + }; + sqlx::query_as( + "SELECT name, setting FROM pg_settings WHERE name IN ('application_name', \ + 'tcp_keepalives_idle', 'tcp_keepalives_interval', 'tcp_keepalives_count') \ + ORDER BY name", + ) + .fetch_all(&mut *conn) + .await + .map_err(|e| MigrationError::store("read the serving session's settings", e)) + } + + /// End the registration. + pub async fn release(self) { + self.keepalive.abort(); + self.held.release().await; + } +} + +impl Drop for ServingRegistration { + fn drop(&mut self) { + self.keepalive.abort(); + } +} + +async fn register_on(conn: &mut PgConnection) -> Result<(), MigrationError> { + let fail = |e: sqlx::Error| MigrationError::store("register the server", e); + statement(conn, "BEGIN").await.map_err(fail)?; + // Inside the transaction that holds the key, so a transaction-mode pooler + // applies them to the backend the key lives on; they last as long as the + // transaction, which is the registration's life. + keep_open(conn).await.map_err(fail)?; + for (setting, value) in TCP_KEEPALIVES { + statement(conn, &format!("SET LOCAL {setting} = {value}")) + .await + .map_err(fail)?; + } + if !try_key(conn, "pg_try_advisory_xact_lock_shared", SERVING_KEY) + .await + .map_err(fail)? + { + let names = holders(conn, SERVING_KEY).await.unwrap_or_default(); + return Err(MigrationError::Locked(format!( + "a migration is running on this catalog ({}); start the server when it has finished", + listed(&names) + ))); + } + let migrators = holders(conn, MIGRATION_KEY).await.map_err(fail)?; + if !migrators.is_empty() { + return Err(MigrationError::Locked(format!( + "a migration holds the migration lock on this catalog ({}); start the server when \ + it has finished", + listed(&migrators) + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{AGE_OFF, IDLE_OFF, shown_secs, timeouts_off}; + use std::time::Duration; + + #[test] + fn transaction_timeout_is_turned_off_from_postgresql_17() { + assert_eq!(timeouts_off(160_010), [IDLE_OFF]); + assert_eq!(timeouts_off(169_999), [IDLE_OFF]); + assert_eq!(timeouts_off(170_000), [IDLE_OFF, AGE_OFF]); + assert_eq!(timeouts_off(180_001), [IDLE_OFF, AGE_OFF]); + // A version that does not parse gets the setting every server has. + assert_eq!(timeouts_off(0), [IDLE_OFF]); + } + + #[test] + fn a_wait_with_time_left_never_reads_as_zero_seconds() { + assert_eq!(shown_secs(Duration::from_millis(1)), 1); + assert_eq!(shown_secs(Duration::from_millis(1000)), 1); + assert_eq!(shown_secs(Duration::from_millis(1001)), 2); + assert_eq!(shown_secs(Duration::ZERO), 0); + } +} diff --git a/crates/storage-postgres/src/migrations/m0003_sort_key_definitions.rs b/crates/storage-postgres/src/migrations/m0003_sort_key_definitions.rs new file mode 100644 index 000000000..6b12999b8 --- /dev/null +++ b/crates/storage-postgres/src/migrations/m0003_sort_key_definitions.rs @@ -0,0 +1,254 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 3, `sort_key_definitions`, PostgreSQL: restore sort key +//! attribute definitions that the pre-fix `UpdateTable` deleted (#259), in +//! `tables` rows and in the definitions stored with backups. +//! +//! Before the fix, `UpdateTable` replaced a table's stored attribute +//! definitions with the request's subset, so adding a GSI dropped the table's +//! own key definitions. A table's sort key type is read back from its data +//! table's PRIMARY KEY column (`sk_s`, `sk_n` or `sk_b`) in the data +//! database. A backup's is read from the typed value in one of its items, or +//! else from its source table. A partition key definition the same write +//! dropped cannot be recovered, because the partition key column is always +//! TEXT; it is a note. Nothing here stops the migration: a definition it +//! cannot repair is a note, and the table or backup keeps failing loudly +//! until a person fixes it. The writes go to the catalog, in the runner's +//! transaction; the data database is only read. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit, and keeps +//! its own small helpers instead of calling the crate's, which may change. + +use async_trait::async_trait; +use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType}; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use extenddb_storage::util::recover_sort_key_definitions; +use serde_json::Value; + +use super::PostgresStores; +use super::connect::sql_error; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0003_sort_key_definitions.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0003_sort_key_definitions/postgres/1"; + +pub(crate) struct SortKeyDefinitions; + +/// What one pass repaired, and what it could not. +#[derive(Default)] +struct Repair { + repaired: Vec, + needs_attention: Vec, +} + +#[async_trait] +impl CodeBody for SortKeyDefinitions { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &PostgresStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &PostgresStores) -> Result { + let tables = repair_tables(stores).await?; + let backups = repair_backups(stores).await?; + let mut notes = Vec::new(); + for report in [tables, backups] { + notes.extend(report.repaired.into_iter().map(|line| format!("restored {line}"))); + notes.extend(report.needs_attention); + } + Ok(Done { notes }) + } +} + +/// The sort key elements of `key_schema` with no attribute definition. +fn missing_sort_keys<'a>(key_schema: &'a [KeySchemaElement], attr_defs: &[AttributeDefinition]) -> Vec<&'a KeySchemaElement> { + key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter(|ks| !attr_defs.iter().any(|ad| ad.attribute_name == ks.attribute_name)) + .collect() +} + +fn names(elements: &[&KeySchemaElement]) -> String { + elements.iter().map(|ks| ks.attribute_name.as_str()).collect::>().join(", ") +} + +/// The PRIMARY KEY columns of the data table of `table_id`, in key order; +/// empty when the data table does not exist. `to_regclass` returns NULL for +/// a missing table rather than raising. +async fn primary_key_columns(stores: &PostgresStores, table_id: &str) -> Result, MigrationError> { + sqlx::query_scalar( + "SELECT a.attname::text \ + FROM pg_index i \ + JOIN unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON TRUE \ + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum \ + WHERE i.indrelid = to_regclass($1) AND i.indisprimary \ + ORDER BY k.ord", + ) + .bind(format!("public.\"_ddb_{table_id}\"")) + .fetch_all(&stores.data) + .await + .map_err(|e| sql_error(&format!("read the PRIMARY KEY of _ddb_{table_id}"), &e)) +} + +/// Repair the `tables` rows. The partition key note is reported on every run +/// while the definition stays absent. +async fn repair_tables(stores: &PostgresStores) -> Result { + let mut report = Repair::default(); + let rows: Vec<(String, String, String, Value, Value)> = sqlx::query_as( + "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ + FROM tables ORDER BY table_name, account_id", + ) + .fetch_all(&stores.catalog) + .await + .map_err(|e| sql_error("read tables", &e))?; + + for (account_id, table_name, table_id, ks_json, ad_json) in rows { + let key_schema: Vec = match serde_json::from_value(ks_json) { + Ok(v) => v, + Err(e) => { + report.needs_attention.push(format!("{table_name}: unreadable key_schema ({e})")); + continue; + } + }; + let attr_defs: Vec = serde_json::from_value(ad_json).unwrap_or_default(); + + let pk_missing: Vec<&str> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .filter(|ks| !attr_defs.iter().any(|ad| ad.attribute_name == ks.attribute_name)) + .map(|ks| ks.attribute_name.as_str()) + .collect(); + if !pk_missing.is_empty() { + report.needs_attention.push(format!( + "{table_name}: partition key definition(s) [{}] are absent; reads and writes are \ + unaffected, but index key type validation cannot check them", + pk_missing.join(", ") + )); + } + + let missing = missing_sort_keys(&key_schema, &attr_defs); + if missing.is_empty() { + continue; + } + let columns = primary_key_columns(stores, &table_id).await?; + let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); + if recovered.is_empty() { + report.needs_attention.push(format!( + "{table_name}: sort key(s) [{}] have no attribute definition and no matching \ + PRIMARY KEY column was found to recover the type from", + names(&missing) + )); + continue; + } + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_value(&merged) + .map_err(|e| MigrationError::Store(format!("serialize the definitions of {table_name}: {e}")))?; + sqlx::query("UPDATE tables SET attribute_definitions = $1 WHERE account_id = $2 AND table_name = $3") + .bind(&merged_json) + .bind(&account_id) + .bind(&table_name) + .execute(&stores.catalog) + .await + .map_err(|e| sql_error(&format!("repair {table_name}"), &e))?; + for def in &recovered { + report.repaired.push(format!("{table_name}: {} ({:?})", def.attribute_name, def.attribute_type)); + } + } + Ok(report) +} + +/// The type of `attribute` in one of the backup's items, from its typed +/// value (`{"S": ...}`, `{"N": ...}` or `{"B": ...}`). +async fn type_from_items(stores: &PostgresStores, backup_arn: &str, attribute: &str) -> Result, MigrationError> { + let item: Option = sqlx::query_scalar("SELECT item_data FROM backup_items WHERE backup_arn = $1 LIMIT 1") + .bind(backup_arn) + .fetch_optional(&stores.catalog) + .await + .map_err(|e| sql_error(&format!("read an item of backup {backup_arn}"), &e))?; + let descriptor = item + .as_ref() + .and_then(|value| value.get(attribute)) + .and_then(Value::as_object) + .and_then(|typed| typed.keys().next().cloned()); + Ok(match descriptor.as_deref() { + Some("S") => Some(ScalarAttributeType::S), + Some("N") => Some(ScalarAttributeType::N), + Some("B") => Some(ScalarAttributeType::B), + _ => None, + }) +} + +/// Repair the definitions stored with backups (`backups.attribute_definitions`), +/// so a backup taken before the fix restores like its table. +/// RestoreTableFromBackup validates a stored definition as CreateTable does, +/// so a backup left unrepaired fails to restore rather than bringing the +/// damage back. +async fn repair_backups(stores: &PostgresStores) -> Result { + let mut report = Repair::default(); + let rows: Vec<(String, String, String, Value, Value)> = sqlx::query_as( + "SELECT backup_arn, table_name, table_id, key_schema, attribute_definitions \ + FROM backups ORDER BY backup_arn", + ) + .fetch_all(&stores.catalog) + .await + .map_err(|e| sql_error("read backups", &e))?; + + for (backup_arn, table_name, table_id, ks_json, ad_json) in rows { + let label = format!("backup {backup_arn} of {table_name}"); + let Ok(key_schema) = serde_json::from_value::>(ks_json) else { + report.needs_attention.push(format!("{label}: unreadable key_schema")); + continue; + }; + let attr_defs: Vec = serde_json::from_value(ad_json).unwrap_or_default(); + let missing = missing_sort_keys(&key_schema, &attr_defs); + if missing.is_empty() { + continue; + } + let mut recovered = Vec::new(); + for element in &missing { + if let Some(attribute_type) = type_from_items(stores, &backup_arn, &element.attribute_name).await? { + recovered.push(AttributeDefinition { + attribute_name: element.attribute_name.clone(), + attribute_type, + }); + } + } + if recovered.len() < missing.len() { + let columns = primary_key_columns(stores, &table_id).await?; + let known: Vec = attr_defs.iter().chain(recovered.iter()).cloned().collect(); + recovered.extend(recover_sort_key_definitions(&key_schema, &known, &columns)); + } + if recovered.len() < missing.len() { + report.needs_attention.push(format!( + "{label}: sort key(s) [{}] have no attribute definition, and neither its items nor \ + its source table give the type; restoring it fails until the definition is added", + names(&missing) + )); + continue; + } + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_value(&merged) + .map_err(|e| MigrationError::Store(format!("serialize the definitions of {label}: {e}")))?; + sqlx::query("UPDATE backups SET attribute_definitions = $1 WHERE backup_arn = $2") + .bind(&merged_json) + .bind(&backup_arn) + .execute(&stores.catalog) + .await + .map_err(|e| sql_error(&format!("repair {label}"), &e))?; + for def in &recovered { + report.repaired.push(format!("{label}: {} ({:?})", def.attribute_name, def.attribute_type)); + } + } + Ok(report) +} diff --git a/crates/storage-postgres/src/migrations/m0004_index_table_base_key.rs b/crates/storage-postgres/src/migrations/m0004_index_table_base_key.rs new file mode 100644 index 000000000..01ccf235c --- /dev/null +++ b/crates/storage-postgres/src/migrations/m0004_index_table_base_key.rs @@ -0,0 +1,118 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 4, `index_table_base_key`, PostgreSQL: give every GSI and LSI +//! index table a valid `_ddb__base_key_idx`. +//! +//! Index propagation finds an item's old index row by its base table key +//! (`base_pk` and the base sort key columns); without an index leading with +//! those columns that is a scan of the whole index table. CreateTable has +//! created the index with each index table since v0.1.8; index tables created +//! by v0.1.5 and v0.1.6 lack it until a later release's `migrate` built it, +//! and a build that was cancelled left it INVALID. +//! +//! Tables are listed from the catalog rows, never by name in the data +//! database, where vector tables share the `_ddb_` prefix. The index's +//! columns are the index table's PRIMARY KEY columns after `pk`, read from the +//! table itself, so a damaged attribute definition cannot choose the wrong +//! column; the statement is the one CreateTable writes. A catalog row whose +//! table is being deleted, or whose index table is missing, is skipped and +//! noted. The builds run outside a transaction through the index helper, and +//! the ledger row is written only when every index is valid. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; + +use super::index_helper::{Built, IndexHelper}; +use super::PostgresStores; +use super::connect::sql_error; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0004_index_table_base_key.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0004_index_table_base_key/postgres/1"; + +pub(crate) struct IndexTableBaseKey; + +#[async_trait] +impl CodeBody for IndexTableBaseKey { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &PostgresStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &PostgresStores) -> Result { + add_base_key_indexes(stores).await.map(|notes| Done { notes }) + } +} + +/// Build the missing and invalid indexes; one note per index built and per +/// catalog row skipped. +async fn add_base_key_indexes(stores: &PostgresStores) -> Result, MigrationError> { + let mut notes = Vec::new(); + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT t.table_name, t.table_status, i.index_name, i.index_id \ + FROM indexes i JOIN tables t ON t.table_id = i.table_id \ + WHERE i.index_type IN ('GSI', 'LSI') \ + ORDER BY t.table_name, i.index_name", + ) + .fetch_all(&stores.catalog) + .await + .map_err(|e| sql_error("list secondary indexes", &e))?; + + let mut helper = IndexHelper::new(stores, &stores.data); + for (table_name, table_status, index_name, index_id) in rows { + let label = format!("{table_name} index {index_name}"); + // The id is interpolated into identifiers, so it must be a UUID. + if uuid::Uuid::parse_str(&index_id).is_err() { + notes.push(format!("{label}: skipped, index_id {index_id:?} is not a UUID")); + continue; + } + if table_status == "DELETING" { + notes.push(format!("{label}: skipped, the table is being deleted")); + continue; + } + let index_table = format!("_ddb_{index_id}"); + let columns: Vec = sqlx::query_scalar( + "SELECT a.attname::text \ + FROM pg_index i \ + JOIN unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON TRUE \ + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum \ + WHERE i.indrelid = to_regclass($1) AND i.indisprimary \ + ORDER BY k.ord", + ) + .bind(format!("public.\"{index_table}\"")) + .fetch_all(&stores.data) + .await + .map_err(|e| sql_error(&format!("read the PRIMARY KEY of {index_table}"), &e))?; + if columns.is_empty() { + notes.push(format!("{label}: skipped, its index table {index_table} does not exist")); + continue; + } + let base_columns: Vec = columns.into_iter().filter(|c| c.starts_with("base_")).collect(); + if base_columns.first().map(String::as_str) != Some("base_pk") { + notes.push(format!("{label}: skipped, the PRIMARY KEY of {index_table} has no base_pk column")); + continue; + } + let index = format!("{index_table}_base_key_idx"); + let on = format!("ON \"{index_table}\" ({})", base_columns.join(", ")); + match helper.ensure(&index, &on).await? { + Built::Existing => {} + Built::Created => notes.push(format!("{label}: built {index} on ({})", base_columns.join(", "))), + Built::Rebuilt => notes.push(format!( + "{label}: rebuilt {index} on ({}), which an earlier build left invalid", + base_columns.join(", ") + )), + } + } + helper.finish().await?; + Ok(notes) +} diff --git a/crates/storage-postgres/src/migrations/mod.rs b/crates/storage-postgres/src/migrations/mod.rs new file mode 100644 index 000000000..af6b6d1b4 --- /dev/null +++ b/crates/storage-postgres/src/migrations/mod.rs @@ -0,0 +1,355 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The PostgreSQL backend's catalog migrations. +//! +//! Two stores: the catalog database ([`StoreKind::Catalog`]) and the data +//! database ([`StoreKind::Data`]). [`migration`] is the table, an exhaustive +//! `match` from every [`MigrationId`] to this backend's bodies. [`RELEASED`] +//! pins the released bodies; the release PR that raises `RELEASED_THROUGH` +//! pastes the lines the checksum test prints. +//! +//! Entry points: [`catalog_status`] reads with no lock and no write; +//! [`migrate`] takes the migration lock (which also waits for running +//! servers to stop), applies and releases; [`migrate_holding`] applies for a +//! caller that took [`MigrationLock`] itself (the commands, which take it +//! through [`PostgresCatalogMigrations`]); [`ServingRegistration`] is what a +//! server holds while it serves, and [`ServerRegistration`] is the server's +//! startup check around it and the backend's side of registering again; +//! [`try_create_vector_extension`] is the pgvector attempt `migrate` makes +//! on every run. [`PostgresCatalogMigrations`] is how the commands reach +//! them. + +mod backend; +mod commands; +mod connect; +mod failure; +mod held; +mod index_helper; +mod locks; +// Frozen once released: neither rustfmt nor a later clippy lint may require +// an edit that changes their checksums. +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0003_sort_key_definitions; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0004_index_table_base_key; +mod postcheck; +mod registrar; +mod rows; +mod sessions; +mod vector_extension; + +use std::time::Duration; + +use extenddb_storage::file_body; +use extenddb_storage::migrations::m0002_resource_tag_condition_keys::{ + self, ResourceTagConditionKeys, +}; +use extenddb_storage::migrations::{ + CatalogStatus, Kind, Migration, MigrationBackend, MigrationError, MigrationId, Released, + RunReport, Runner, StatusNote, StoreBody, StoreKind, +}; +use sqlx::postgres::PgConnectOptions; + +pub use backend::{PostgresMigrations, PostgresStores}; +pub use commands::{PostgresCatalogMigrations, resolve}; +pub use locks::{KEEPALIVE_INTERVAL, MigrationLock, ServingRegistration}; +#[cfg(feature = "test-util")] +pub use postcheck::{Column, Definitions, Index, compare, describe}; +pub use registrar::ServerRegistration; +pub use sessions::{require_data_connection_row, served_data_options}; +pub use vector_extension::try_create_vector_extension; + +/// The definitions migration 1 creates in an empty catalog database, which +/// the post-check compares the database with the first time migration 1 is +/// applied to it. +pub(crate) const CATALOG_EXPECTED: &str = + include_str!("../../migrations/0001_baseline.expected.json"); + +/// The same for the data database. +pub(crate) const DATA_EXPECTED: &str = + include_str!("../../data_migrations/0001_baseline.expected.json"); + +/// Checksums of the released bodies (none until v1.0). +pub(crate) const RELEASED: &[Released] = &[]; + +/// This backend's entry for every id. Public so `migrate --list` can print +/// each migration's kind without connecting. +#[must_use] +pub fn migration(id: MigrationId) -> Migration { + match id { + MigrationId::Baseline => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![ + file_body!(StoreKind::Catalog, "migrations", "0001_baseline.sql"), + file_body!(StoreKind::Data, "data_migrations", "0001_baseline.sql"), + ], + }, + MigrationId::ResourceTagConditionKeys => Migration::Applies { + kind: Kind::Breaking, + // A `dynamodb:ResourceTag` key written through the API after the + // rewrite has no effect, as on the service; running the rewrite + // again would give it one. + convergent: false, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &ResourceTagConditionKeys, + m0002_resource_tag_condition_keys::SOURCE, + )], + }, + MigrationId::SortKeyDefinitions => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0003_sort_key_definitions::SortKeyDefinitions, + m0003_sort_key_definitions::SOURCE, + )], + }, + MigrationId::IndexTableBaseKey => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Data, + &m0004_index_table_base_key::IndexTableBaseKey, + m0004_index_table_base_key::SOURCE, + )], + }, + } +} + +/// How a migration's code bodies run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RunsIn { + /// Inside the runner's transaction, committed with the ledger row. + Transaction, + /// Statement by statement outside a transaction (`CREATE INDEX + /// CONCURRENTLY` cannot run in one), through the index helper; the + /// ledger row is written after the last statement. + Autocommit, +} + +/// How each id's code bodies run. File bodies always run in the runner's +/// transaction. +pub(crate) const fn runs_in(id: MigrationId) -> RunsIn { + match id { + MigrationId::Baseline + | MigrationId::ResourceTagConditionKeys + | MigrationId::SortKeyDefinitions => RunsIn::Transaction, + MigrationId::IndexTableBaseKey => RunsIn::Autocommit, + } +} + +/// Where the runner connects: the catalog database, and the data database. +/// +/// The runner reaches the data database with the catalog's +/// `data_database_connection_string` row, which the server connects with, +/// when the catalog has it, and refuses when that row names another database +/// than `data_database` unless `data_database_given`; without the row it +/// uses the catalog's server and credentials with `data_database`. +#[derive(Debug, Clone)] +pub struct PostgresTarget { + /// The catalog database, with the credentials the command was given. + pub catalog: PgConnectOptions, + /// The data database's name. + pub data_database: String, + /// Whether `data_database` was given on the command line (`--data-db`), + /// so it overrides the database the connection string row names. + pub data_database_given: bool, + /// For `init`: the value of the `data_database_connection_string` row, + /// which migration 1's catalog body writes with `data_database_name`. + pub data_connection_row: Option, + /// Timeout for each connection attempt. + pub(crate) connect_timeout: Duration, +} + +impl PostgresTarget { + /// A target with the default connect timeout of 10 s and no data + /// connection rows to write. + #[must_use] + pub fn new(catalog: PgConnectOptions, data_database: impl Into) -> Self { + Self { + catalog, + data_database: data_database.into(), + data_database_given: false, + data_connection_row: None, + connect_timeout: Duration::from_secs(10), + } + } + + /// Mark the data database as given on the command line. + #[must_use] + pub fn with_given_data_database(mut self) -> Self { + self.data_database_given = true; + self + } + + /// Have migration 1's catalog body write the data connection rows, as + /// `init` needs. + #[must_use] + pub fn with_data_connection_row(mut self, connection_string: impl Into) -> Self { + self.data_connection_row = Some(connection_string.into()); + self + } + + /// The catalog database's name. + #[must_use] + pub fn catalog_database(&self) -> String { + self.catalog.get_database().unwrap_or("postgres").to_owned() + } + + /// The data database on the catalog's server, with the catalog's + /// credentials: where the runner connects when the catalog records no + /// data connection string. + #[must_use] + pub fn data(&self) -> PgConnectOptions { + self.catalog.clone().database(&self.data_database) + } +} + +/// The catalog status. Read-only: the connections are read-only sessions, no +/// lock is taken, and nothing is written, so it runs beside a server. A +/// catalog database that does not exist is `Uninitialized`. +/// +/// # Errors +/// +/// A connection fails, or a ledger cannot be read. +pub async fn catalog_status(target: &PostgresTarget) -> Result { + let migrations = match PostgresMigrations::open_for_status(target).await { + Ok(m) => m, + Err(backend::OpenError::NoCatalogDatabase) => return Ok(CatalogStatus::Uninitialized), + Err(backend::OpenError::Failed(e)) => return Err(e), + }; + // A read records nothing, so no release is named. + let status = Runner::new(&migrations, "").status().await; + migrations.close().await; + status +} + +/// The status notes: for each database, the tables, views and sequences the +/// application role does not own ([`sessions::ownership_note`]). Read-only, +/// like [`catalog_status`]; a catalog database that does not exist has +/// none. +/// +/// # Errors +/// +/// A connection fails, or the owners cannot be read. +pub async fn status_notes(target: &PostgresTarget) -> Result, MigrationError> { + let migrations = match PostgresMigrations::open_for_status(target).await { + Ok(m) => m, + Err(backend::OpenError::NoCatalogDatabase) => return Ok(Vec::new()), + Err(backend::OpenError::Failed(e)) => return Err(e), + }; + let notes = async { + let stores = migrations.stores(); + let mut notes = Vec::new(); + for (pool, kind) in [ + (&stores.catalog, StoreKind::Catalog), + (&stores.data, StoreKind::Data), + ] { + for name in migrations.store_names(kind).await? { + notes.extend(sessions::ownership_note(pool, kind, &name).await?); + } + } + Ok(notes) + } + .await; + migrations.close().await; + notes +} + +/// Apply every pending migration: take the migration lock, waiting for +/// another migrator, then for running servers, then for an earlier +/// migrator's statements, all within `lock_timeout`, apply, and release the +/// lock and close every connection on every exit path. +/// +/// # Errors +/// +/// [`MigrationError::Locked`] (exit 6) when the lock or a running server +/// outlasts `lock_timeout`, or as [`Runner::migrate`]. +pub async fn migrate( + target: &PostgresTarget, + release: &str, + lock_timeout: Duration, +) -> Result { + let lock = MigrationLock::acquire(&target.catalog, lock_timeout).await?; + let result = migrate_holding(target, release, &lock).await; + lock.release().await; + result +} + +/// [`migrate`] for a caller that holds the migration lock. +/// +/// # Errors +/// +/// As [`Runner::migrate`]. +pub async fn migrate_holding( + target: &PostgresTarget, + release: &str, + lock: &MigrationLock, +) -> Result { + let migrations = PostgresMigrations::open(target, lock).await?; + let result = Runner::new(&migrations, release).migrate().await; + migrations.close().await; + result +} + +/// Bring a fresh catalog database and a fresh data database to the current +/// schema, for the test harnesses in this crate's `tests/` directory, so they +/// do not carry their own copy of the migration table. +/// +/// With two databases this is [`migrate`], as `extenddb migrate --yes` runs +/// it: every migration through the runner, each recorded in the ledger of +/// the database it changed. When `catalog` and `data` reach one database +/// (the layout the engine uses when the catalog has no +/// `data_database_connection_string` row) the runner cannot apply both +/// stores, because each database holds one ledger and both stores record +/// migration 1 in it. There the file bodies of every migration are executed +/// in id order, catalog first, as this crate's one-database tests execute +/// the shipped files, and no ledger row is written. No code body runs on +/// that layout: each one converts rows or tables that a fresh database does +/// not have. +/// +/// # Errors +/// +/// Returns the first error, formatted. +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub async fn apply_all_migrations( + catalog: &sqlx::PgPool, + data: &sqlx::PgPool, +) -> Result<(), String> { + let catalog_options = (*catalog.connect_options()).clone(); + let data_options = data.connect_options(); + let one_database = catalog_options.get_host() == data_options.get_host() + && catalog_options.get_port() == data_options.get_port() + && catalog_options.get_database() == data_options.get_database(); + if one_database { + for id in MigrationId::ALL { + for body in migration(id).bodies() { + if let extenddb_storage::migrations::Body::File { name, text } = &body.body { + sqlx::raw_sql(text) + .execute(catalog) + .await + .map_err(|e| format!("{name}: {e:?}"))?; + } + } + } + return Ok(()); + } + let data_database = data_options.get_database().unwrap_or("postgres").to_owned(); + migrate( + &PostgresTarget::new(catalog_options, data_database), + "test", + Duration::ZERO, + ) + .await + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/storage-postgres/src/migrations/postcheck.rs b/crates/storage-postgres/src/migrations/postcheck.rs new file mode 100644 index 000000000..a2f8df4d5 --- /dev/null +++ b/crates/storage-postgres/src/migrations/postcheck.rs @@ -0,0 +1,520 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The post-check of migration 1: the database's definitions compared with +//! the ones migration 1 creates in an empty database, embedded from +//! `0001_baseline.expected.json` beside each file. +//! +//! A definition is a column's type, nullability, default and collation, an +//! index's `pg_get_indexdef` and `indisvalid`, a constraint's +//! `pg_get_constraintdef`, and a sequence's existence; column order is +//! ignored. Every object the expected file names is checked, and objects it +//! does not name are ignored, because the same databases hold per-table +//! objects no migration creates. A difference stops migration 1, and the +//! refusal names each object with what it is, what v1.0 has and what to do; +//! the check runs inside migration 1's transaction, so nothing is +//! committed. One pre-release shape is accepted with a note: +//! `table_restores.source_table_arn` nullable, which a build from before the +//! release of `table_restores` left. + +use std::collections::{BTreeMap, BTreeSet}; + +use extenddb_storage::migrations::MigrationError; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; + +/// One column's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Column { + #[serde(rename = "type")] + pub data_type: String, + pub not_null: bool, + pub default: Option, + /// `None` for the type's default collation. + pub collation: Option, +} + +/// One index's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Index { + pub table: String, + pub definition: String, + pub valid: bool, +} + +/// A database's definitions, keyed for comparison: columns as +/// `table.column`, constraints as `table.constraint`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Definitions { + pub columns: BTreeMap, + pub indexes: BTreeMap, + pub constraints: BTreeMap, + pub sequences: BTreeSet, +} + +/// table, column, type, not null, default, collation. +type ColumnRow = (String, String, String, bool, Option, Option); + +/// The runner's own ledger, which no migration creates. +const LEDGER: &str = "extenddb_migrations"; + +fn read_error(e: sqlx::Error) -> MigrationError { + MigrationError::store("read the schema's definitions", e) +} + +/// The definitions of every object in the public schema, the runner's ledger +/// excluded. +/// +/// # Errors +/// +/// The catalog tables of the database cannot be read. +pub async fn describe(pool: &PgPool) -> Result { + let mut defs = Definitions::default(); + let columns: Vec = sqlx::query_as( + "SELECT c.relname::text, a.attname::text, format_type(a.atttypid, a.atttypmod), \ + a.attnotnull, pg_get_expr(d.adbin, d.adrelid), \ + CASE WHEN a.attcollation <> 0 AND a.attcollation <> t.typcollation \ + THEN co.collname::text END \ + FROM pg_attribute a \ + JOIN pg_class c ON c.oid = a.attrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + JOIN pg_type t ON t.oid = a.atttypid \ + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum \ + LEFT JOIN pg_collation co ON co.oid = a.attcollation \ + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') \ + AND a.attnum > 0 AND NOT a.attisdropped", + ) + .fetch_all(pool) + .await + .map_err(read_error)?; + for (table, column, data_type, not_null, default, collation) in columns { + if table != LEDGER { + defs.columns.insert( + format!("{table}.{column}"), + Column { + data_type, + not_null, + default, + collation, + }, + ); + } + } + let indexes: Vec<(String, String, String, bool)> = sqlx::query_as( + "SELECT ic.relname::text, c.relname::text, pg_get_indexdef(i.indexrelid), i.indisvalid \ + FROM pg_index i \ + JOIN pg_class ic ON ic.oid = i.indexrelid \ + JOIN pg_class c ON c.oid = i.indrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public'", + ) + .fetch_all(pool) + .await + .map_err(read_error)?; + for (name, table, definition, valid) in indexes { + if table != LEDGER { + defs.indexes.insert( + name, + Index { + table, + definition, + valid, + }, + ); + } + } + // PostgreSQL 18 records each NOT NULL as a constraint (`contype = 'n'`) + // whose name embeds the table; the column's `not_null` already compares + // it, and earlier servers have no such rows. + let constraints: Vec<(String, String, String)> = sqlx::query_as( + "SELECT c.relname::text, con.conname::text, pg_get_constraintdef(con.oid) \ + FROM pg_constraint con \ + JOIN pg_class c ON c.oid = con.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND con.contype <> 'n'", + ) + .fetch_all(pool) + .await + .map_err(read_error)?; + for (table, name, definition) in constraints { + if table != LEDGER { + defs.constraints + .insert(format!("{table}.{name}"), definition); + } + } + let sequences: Vec = sqlx::query_scalar( + "SELECT c.relname::text FROM pg_class c \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND c.relkind = 'S'", + ) + .fetch_all(pool) + .await + .map_err(read_error)?; + defs.sequences = sequences.into_iter().collect(); + Ok(defs) +} + +/// Whether a difference in `key` from `expected` to `actual` is the accepted +/// pre-release shape. +fn accepted(key: &str, expected: &Column, actual: &Column) -> bool { + key == "table_restores.source_table_arn" + && expected.not_null + && !actual.not_null + && Column { + not_null: true, + ..actual.clone() + } == *expected +} + +/// A column's definition as `ALTER TABLE ... ADD COLUMN` writes it after the +/// name: `bigint NOT NULL DEFAULT 0`. +fn column_sql(column: &Column) -> String { + let mut sql = column.data_type.clone(); + if let Some(collation) = &column.collation { + sql.push_str(&format!(" COLLATE \"{collation}\"")); + } + if column.not_null { + sql.push_str(" NOT NULL"); + } + if let Some(default) = &column.default { + sql.push_str(&format!(" DEFAULT {default}")); + } + sql +} + +/// How `found` differs from `expected`, one clause per attribute, such as +/// `has type integer where v1.0 has bigint`. +fn column_differences(found: &Column, expected: &Column) -> Vec { + let collation = |c: &Option| { + c.as_ref() + .map_or_else(|| "the type's default".to_owned(), |c| format!("\"{c}\"")) + }; + let mut clauses = Vec::new(); + if found.data_type != expected.data_type { + clauses.push(format!( + "has type {} where v1.0 has {}", + found.data_type, expected.data_type + )); + } + if found.not_null != expected.not_null { + clauses.push(if found.not_null { + "is NOT NULL where v1.0 allows NULL".to_owned() + } else { + "allows NULL where v1.0 has NOT NULL".to_owned() + }); + } + match (&found.default, &expected.default) { + (Some(f), Some(e)) if f != e => { + clauses.push(format!("has default {f} where v1.0 has {e}")); + } + (Some(f), None) => clauses.push(format!("has default {f} where v1.0 has none")), + (None, Some(e)) => clauses.push(format!("has no default where v1.0 has {e}")), + _ => {} + } + if found.collation != expected.collation { + clauses.push(format!( + "has collation {} where v1.0 has {}", + collation(&found.collation), + collation(&expected.collation) + )); + } + clauses +} + +/// `items` joined as a sentence lists them: `a`, `a and b`, `a, b and c`. +fn listed(items: &[String]) -> String { + match items { + [] => String::new(), + [one] => one.clone(), + [rest @ .., last] => format!("{} and {last}", rest.join(", ")), + } +} + +/// A `table.name` key split into its table and its name. +fn split(key: &str) -> (&str, &str) { + key.split_once('.').unwrap_or(("", key)) +} + +/// The differences between `expected` and `actual`, each naming the object, +/// what it is and what v1.0 has, and what to do about it; and the notes for +/// the accepted shape. +#[must_use] +pub fn compare(expected: &Definitions, actual: &Definitions) -> (Vec, Vec) { + let mut differences = Vec::new(); + let mut notes = Vec::new(); + let named_tables: BTreeSet<&str> = expected.columns.keys().map(|k| split(k).0).collect(); + for (key, column) in &expected.columns { + match actual.columns.get(key) { + None => differences.push(format!( + "column {key} is missing; add it as {}", + column_sql(column) + )), + Some(found) if found == column => {} + Some(found) if accepted(key, column, found) => notes.push(format!( + "{key} is nullable, as a build from before the release of table_restores left it; \ + its rows may hold NULL" + )), + Some(found) => differences.push(format!( + "column {key} {}; change it to {}", + listed(&column_differences(found, column)), + column_sql(column) + )), + } + } + for key in actual.columns.keys() { + if named_tables.contains(split(key).0) && !expected.columns.contains_key(key) { + differences.push(format!( + "column {key} is not part of the v1.0 schema; drop it, after saving any data in \ + it that you need" + )); + } + } + for (name, index) in &expected.indexes { + let on = format!("index {name} on table {}", index.table); + match actual.indexes.get(name) { + None => differences.push(format!( + "{on} is missing; create it with {}", + index.definition + )), + Some(found) if found == index => {} + Some(found) if found.definition != index.definition => differences.push(format!( + "{on} is defined as {} where v1.0 defines it as {}; drop it and create it with \ + the v1.0 definition", + found.definition, index.definition + )), + Some(_) => differences.push(format!( + "{on} is not valid, as an index build that did not finish leaves it; rebuild it \ + with REINDEX INDEX {name}" + )), + } + } + for (key, definition) in &expected.constraints { + let (table, constraint) = split(key); + let on = format!("constraint {constraint} on table {table}"); + match actual.constraints.get(key) { + None => differences.push(format!("{on} is missing; add it as {definition}")), + Some(found) if found == definition => {} + Some(found) => differences.push(format!( + "{on} is {found} where v1.0 has {definition}; drop it and add it as {definition}" + )), + } + } + for sequence in &expected.sequences { + if !actual.sequences.contains(sequence) { + differences.push(format!("sequence {sequence} is missing; create it")); + } + } + (differences, notes) +} + +/// The refusal for `differences` found in `database`: one line per +/// difference, then what to do. +fn refusal(database: &str, differences: &[String]) -> String { + let lines: Vec = differences.iter().map(|d| format!(" {d}")).collect(); + format!( + "after migration 1 the database {database} differs from the v1.0 schema in {} place(s), \ + so nothing was committed:\n{}\nMake each change in the database {database}, then run \ + migrate --yes again; migration 1 checks the schema again before it commits.", + differences.len(), + lines.join("\n") + ) +} + +/// Compare the database with `expected_json`; returns the notes, or stops +/// naming every difference. +pub(crate) async fn check( + pool: &PgPool, + expected_json: &str, + database: &str, +) -> Result, MigrationError> { + let expected: Definitions = serde_json::from_str(expected_json).map_err(|e| { + MigrationError::Store(format!( + "migration 1's expected definitions do not parse: {e}" + )) + })?; + let actual = describe(pool).await?; + let (differences, notes) = compare(&expected, &actual); + if differences.is_empty() { + Ok(notes) + } else { + Err(MigrationError::Store(refusal(database, &differences))) + } +} + +#[cfg(test)] +mod tests { + use super::{Column, Definitions, Index, compare, refusal}; + + fn definitions() -> Definitions { + let mut defs = Definitions::default(); + defs.columns.insert( + "t.id".to_owned(), + Column { + data_type: "integer".to_owned(), + not_null: true, + default: None, + collation: None, + }, + ); + defs.indexes.insert( + "t_pkey".to_owned(), + Index { + table: "t".to_owned(), + definition: "CREATE UNIQUE INDEX t_pkey ON public.t USING btree (id)".to_owned(), + valid: true, + }, + ); + defs.constraints + .insert("t.t_pkey".to_owned(), "PRIMARY KEY (id)".to_owned()); + defs.sequences.insert("t_seq".to_owned()); + defs + } + + #[test] + fn identical_definitions_have_no_difference() { + let (differences, notes) = compare(&definitions(), &definitions()); + assert!( + differences.is_empty() && notes.is_empty(), + "{differences:?}" + ); + } + + #[test] + fn an_index_on_other_columns_is_a_difference() { + let mut actual = definitions(); + actual.indexes.get_mut("t_pkey").unwrap().definition = + "CREATE UNIQUE INDEX t_pkey ON public.t USING btree (id, other)".to_owned(); + let (differences, _) = compare(&definitions(), &actual); + assert_eq!( + differences, + [ + "index t_pkey on table t is defined as CREATE UNIQUE INDEX t_pkey ON public.t \ + USING btree (id, other) where v1.0 defines it as CREATE UNIQUE INDEX t_pkey ON \ + public.t USING btree (id); drop it and create it with the v1.0 definition" + ] + ); + let mut invalid = definitions(); + invalid.indexes.get_mut("t_pkey").unwrap().valid = false; + assert_eq!( + compare(&definitions(), &invalid).0, + [ + "index t_pkey on table t is not valid, as an index build that did not finish \ + leaves it; rebuild it with REINDEX INDEX t_pkey" + ] + ); + } + + #[test] + fn a_constraint_with_another_definition_is_a_difference() { + let mut actual = definitions(); + actual + .constraints + .insert("t.t_pkey".to_owned(), "PRIMARY KEY (id, other)".to_owned()); + let (differences, _) = compare(&definitions(), &actual); + assert_eq!( + differences, + [ + "constraint t_pkey on table t is PRIMARY KEY (id, other) where v1.0 has PRIMARY \ + KEY (id); drop it and add it as PRIMARY KEY (id)" + ] + ); + } + + #[test] + fn a_missing_sequence_index_or_constraint_is_a_difference() { + let mut actual = definitions(); + actual.sequences.clear(); + actual.indexes.clear(); + actual.constraints.clear(); + let (differences, _) = compare(&definitions(), &actual); + assert_eq!( + differences, + [ + "index t_pkey on table t is missing; create it with CREATE UNIQUE INDEX t_pkey \ + ON public.t USING btree (id)", + "constraint t_pkey on table t is missing; add it as PRIMARY KEY (id)", + "sequence t_seq is missing; create it" + ] + ); + } + + /// Each attribute of a column that differs is named in words with what + /// v1.0 has, followed by the definition to change it to; a missing + /// column gives the definition to add, and a column v1.0 does not have + /// is to be dropped. + #[test] + fn a_column_difference_names_each_attribute_and_the_v1_definition() { + let mut actual = definitions(); + actual.columns.insert( + "t.id".to_owned(), + Column { + data_type: "double precision".to_owned(), + not_null: false, + default: Some("1".to_owned()), + collation: Some("C".to_owned()), + }, + ); + let (differences, _) = compare(&definitions(), &actual); + assert_eq!( + differences, + [ + "column t.id has type double precision where v1.0 has integer, allows NULL where \ + v1.0 has NOT NULL, has default 1 where v1.0 has none and has collation \"C\" \ + where v1.0 has the type's default; change it to integer NOT NULL" + ] + ); + let mut actual = definitions(); + actual.columns.get_mut("t.id").unwrap().default = None; + let mut expected = definitions(); + expected.columns.get_mut("t.id").unwrap().default = Some("0".to_owned()); + assert_eq!( + compare(&expected, &actual).0, + [ + "column t.id has no default where v1.0 has 0; change it to integer NOT NULL \ + DEFAULT 0" + ] + ); + let mut actual = definitions(); + actual.columns.clear(); + actual.columns.insert( + "t.extra".to_owned(), + Column { + data_type: "text".to_owned(), + not_null: false, + default: None, + collation: None, + }, + ); + assert_eq!( + compare(&definitions(), &actual).0, + [ + "column t.id is missing; add it as integer NOT NULL", + "column t.extra is not part of the v1.0 schema; drop it, after saving any data \ + in it that you need" + ] + ); + } + + /// The refusal prints no Rust structure: one line per difference, then + /// what to do. + #[test] + fn the_refusal_lists_each_difference_and_what_to_do() { + let mut actual = definitions(); + actual.columns.get_mut("t.id").unwrap().data_type = "bigint".to_owned(); + actual.sequences.clear(); + let (differences, _) = compare(&definitions(), &actual); + let message = refusal("acme_catalog", &differences); + assert_eq!( + message, + "after migration 1 the database acme_catalog differs from the v1.0 schema in 2 \ + place(s), so nothing was committed:\n column t.id has type bigint where v1.0 has \ + integer; change it to integer NOT NULL\n sequence t_seq is missing; create it\n\ + Make each change in the database acme_catalog, then run migrate --yes again; \ + migration 1 checks the schema again before it commits." + ); + assert!( + !message.contains("Column {") && !message.contains("Some("), + "{message}" + ); + } +} diff --git a/crates/storage-postgres/src/migrations/registrar.rs b/crates/storage-postgres/src/migrations/registrar.rs new file mode 100644 index 000000000..5bfbaeb52 --- /dev/null +++ b/crates/storage-postgres/src/migrations/registrar.rs @@ -0,0 +1,164 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The registration a server holds for its life: the check `serve` makes +//! before it opens the engine, and the backend's side of registering again +//! after the database ended the registration's session +//! ([`ServingRegistrar`]). +//! +//! The database ends the session on a restart, a failover, a +//! `pg_terminate_backend`, or a pooler that drops it, and frees the serving +//! key with it, so a migration can start. The server then asks for a new +//! registration, which is taken on a new session with a try of the shared +//! key, never a wait: a migration that holds or waits for the exclusive key +//! makes the attempt fail at once. The catalog's status is read again under +//! the new registration, and the server serves again only when it is the +//! status the server started with; otherwise a migration ran in between, +//! and the server stops. + +use std::sync::{Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +use extenddb_storage::migrations::{CatalogStatus, MigrationError, Target}; + +use super::locks::{KEEPALIVE_INTERVAL, ServingRegistration}; +use super::{PostgresTarget, catalog_status, require_data_connection_row, resolve}; + +/// The serving registration of a running server, with what it needs to +/// take it again: the catalog's databases and the status the server started +/// with. +pub struct ServerRegistration { + target: PostgresTarget, + started: CatalogStatus, + keepalive: Duration, + current: Mutex, +} + +impl ServerRegistration { + /// Register as the catalog's server and read its status, before anything + /// opens the engine: a server that refuses has written nothing. The + /// registration is taken before the status is read, so a migration that + /// starts in between waits for this server or makes it refuse. + /// + /// # Errors + /// + /// [`MigrationError::Status`] for a catalog the server does not serve, + /// [`MigrationError::Locked`] (exit 6) while a migration runs, or the + /// error of a database that cannot be reached or read. + pub async fn register(connection_string: &str) -> Result { + Self::register_keepalive(connection_string, KEEPALIVE_INTERVAL).await + } + + /// [`ServerRegistration::register`] with another keepalive interval, for + /// tests. + /// + /// # Errors + /// + /// As [`ServerRegistration::register`]. + #[cfg(feature = "test-util")] + pub async fn register_with( + connection_string: &str, + keepalive: Duration, + ) -> Result { + Self::register_keepalive(connection_string, keepalive).await + } + + async fn register_keepalive( + connection_string: &str, + keepalive: Duration, + ) -> Result { + let Some(target) = resolve(&Target::at(connection_string)).await? else { + return Err(MigrationError::Status(CatalogStatus::Uninitialized)); + }; + let registration = + ServingRegistration::register_keepalive(&target.catalog, keepalive).await?; + let status = match catalog_status(&target).await { + Ok(status) => status, + Err(e) => { + registration.release().await; + return Err(e); + } + }; + match status { + CatalogStatus::Current { .. } | CatalogStatus::AheadCompatible { .. } => { + if let Err(e) = require_data_connection_row(&target).await { + registration.release().await; + return Err(e); + } + if matches!(status, CatalogStatus::AheadCompatible { .. }) { + tracing::warn!( + "{status}; serving it, because a binary may serve migrations it does \ + not know when every one of them is additive" + ); + } + Ok(Self { + target, + started: status, + keepalive, + current: Mutex::new(registration), + }) + } + status => { + registration.release().await; + Err(MigrationError::Status(status)) + } + } + } +} + +#[async_trait] +impl ServingRegistrar for ServerRegistration { + async fn lost(&self) -> String { + let mut lost = self + .current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .lost(); + loop { + let reason = lost.borrow_and_update().clone(); + if let Some(reason) = reason { + return reason; + } + if lost.changed().await.is_err() { + // Released without being lost: nothing more is reported on + // this registration. + std::future::pending::<()>().await; + } + } + } + + async fn register_again(&self) -> Reregistration { + let registration = + match ServingRegistration::register_keepalive(&self.target.catalog, self.keepalive) + .await + { + Ok(registration) => registration, + Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), + Err(e) => return Reregistration::Unavailable(e.to_string()), + }; + let status = match catalog_status(&self.target).await { + Ok(status) => status, + Err(e) => { + // Dropping it closes the connection, which frees the key. + drop(registration); + return Reregistration::Unavailable(format!( + "the catalog's status could not be read: {e}" + )); + } + }; + if status != self.started { + registration.release().await; + return Reregistration::Refused(format!( + "the catalog's status changed while the server held no registration (when it \ + started: {}; now: {status}); start the server again to serve the catalog as it \ + is now", + self.started + )); + } + // The lost registration's connection closes as it is dropped here. + *self.current.lock().unwrap_or_else(PoisonError::into_inner) = registration; + Reregistration::Registered + } +} diff --git a/crates/storage-postgres/src/migrations/rows.rs b/crates/storage-postgres/src/migrations/rows.rs new file mode 100644 index 000000000..e4d8bf7cb --- /dev/null +++ b/crates/storage-postgres/src/migrations/rows.rs @@ -0,0 +1,57 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The rows the runner writes with a body, in PostgreSQL's statements: the +//! ledger row and the version row. + +use async_trait::async_trait; +use extenddb_storage::migrations::{LedgerRow, LedgerWrites, sql_ledger_ddl}; +use sqlx::PgPool; + +use super::failure::{Failure, exec}; + +/// The ledger's `applied_at` column in PostgreSQL. +const APPLIED_AT: &str = "TIMESTAMPTZ NOT NULL DEFAULT now()"; + +/// The ledger rows of one of the runner's connections, the same in both +/// databases. +pub(super) struct Rows<'a>(pub(super) &'a PgPool); + +#[async_trait] +impl LedgerWrites for Rows<'_> { + type Error = Failure; + + async fn create_ledger(&self) -> Result<(), Failure> { + exec(self.0, &sql_ledger_ddl(APPLIED_AT)).await + } + + async fn insert_row(&self, row: &LedgerRow) -> Result<(), Failure> { + sqlx::query( + "INSERT INTO extenddb_migrations (id, name, kind, checksum, outcome, notes, applied_by) \ + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(i32::try_from(row.id).unwrap_or(i32::MAX)) + .bind(&row.name) + .bind(row.kind.as_str()) + .bind(&row.checksum) + .bind(row.outcome.as_str()) + .bind(&row.notes) + .bind(&row.applied_by) + .execute(self.0) + .await + .map(|_| ()) + .map_err(|e| Failure::sqlx(&format!("record migration {}", row.id), &e)) + } + + async fn write_version(&self, version: u32) -> Result<(), Failure> { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('catalog_version', $1) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(version.to_string()) + .execute(self.0) + .await + .map(|_| ()) + .map_err(|e| Failure::sqlx("write catalog_version", &e)) + } +} diff --git a/crates/storage-postgres/src/migrations/sessions.rs b/crates/storage-postgres/src/migrations/sessions.rs new file mode 100644 index 000000000..aa6fb12b3 --- /dev/null +++ b/crates/storage-postgres/src/migrations/sessions.rs @@ -0,0 +1,600 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What a runner and the status read from the database about other +//! sessions, about the data connection, and about who owns the objects +//! migrations alter, and the stops and notes built from it. +//! +//! - A migrator that dies while a statement runs leaves that statement +//! running: the database notices a client is gone only when it next reads +//! from or writes to the connection. Its lock connection is idle in its +//! transaction, so that session ends at once and frees the migration lock, +//! while a body session blocked on a lock or building an index goes on. +//! [`wait_for_earlier_runners`] makes the next migrator wait until no +//! earlier runner's session runs a statement in either database, looking +//! at each database through the runner's own connection to it, so a data +//! database on another server or reached as another role is seen too; and +//! the runner's own transactions set `client_connection_check_interval`, +//! so a statement whose client died ends within a second. +//! - [`served_data`] reaches the data database with the connection string +//! the server connects with; [`served_data_options`] reads it for a caller +//! outside the runner, such as the pgvector attempt. +//! - [`ownership_note`] names the objects the application role does not +//! own and the statements that give it them; [`ownership_stop`] prints the +//! same after a body failed with SQLSTATE 42501, and [`denied_with_notes`] +//! after a status read did. [`lock_timeout_stop`] names the tables other +//! sessions hold locks on after a body's last lock timeout. + +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::{MigrationError, MigrationId, StatusNote, StoreKind, StoreRef}; +use sqlx::PgPool; +use sqlx::postgres::PgConnectOptions; + +use super::PostgresTarget; +use super::connect::{RUNNER_APPLICATION_PREFIX, one_connection, relation_exists, sql_error}; + +/// How often the wait for an earlier runner looks again. +const POLL: Duration = Duration::from_millis(200); + +/// The setting the runner's transactions send so that a statement whose +/// client is gone ends instead of running on. +pub(crate) const CONNECTION_CHECK: &str = "SET LOCAL client_connection_check_interval = '1s'"; + +/// Whether the server accepts [`CONNECTION_CHECK`]: PostgreSQL 14 and later, +/// on a platform that can watch a socket for the peer closing it. +pub(crate) async fn connection_check_supported(pool: &PgPool) -> bool { + let probe = sqlx::raw_sql(&format!("BEGIN; {CONNECTION_CHECK}; ROLLBACK")) + .execute(pool) + .await; + if probe.is_err() { + let _ = sqlx::raw_sql("ROLLBACK").execute(pool).await; + } + probe.is_ok() +} + +/// Wait until no session of another runner runs a statement in any of +/// `databases`, each a database and the runner's connection to it, up to +/// `timeout`, the time the run's lock timeout has left (with none left, the +/// sessions are looked at once); `own` is this runner's `application_name`. +/// Each database is looked at through its own connection, which reaches its +/// server as the role an earlier runner's sessions there used. An idle +/// session is not waited for: a dead client's idle session has already +/// ended, and a pooler's idle backend keeps the last client's name. +/// +/// # Errors +/// +/// [`MigrationError::Blocked`] (exit 6) naming the sessions when they +/// outlast `timeout`. +pub(crate) async fn wait_for_earlier_runners( + databases: &[(&PgPool, &str)], + own: &str, + timeout: Duration, +) -> Result<(), MigrationError> { + type Row = (i32, String, Option, Option, Option); + let started = Instant::now(); + let mut announced = false; + loop { + let mut rows: Vec = Vec::new(); + for (pool, database) in databases { + let found: Vec = sqlx::query_as( + "SELECT pid, datname::text, state, wait_event_type, left(query, 60) \ + FROM pg_stat_activity \ + WHERE datname = $1 AND starts_with(application_name, $2) \ + AND application_name <> $3 AND coalesce(state, '') <> 'idle' \ + ORDER BY pid", + ) + .bind(database) + .bind(RUNNER_APPLICATION_PREFIX) + .bind(own) + .fetch_all(*pool) + .await + .map_err(|e| sql_error("look for an earlier migrator's sessions", &e))?; + rows.extend(found); + } + if rows.is_empty() { + return Ok(()); + } + let sessions = rows + .iter() + .map(|(pid, database, state, wait, query)| { + let waiting = wait + .as_deref() + .map_or_else(String::new, |w| format!(", waiting on a {w}")); + format!( + "pid {pid} in {database} ({}{waiting}: {})", + state.as_deref().unwrap_or("state not visible to this role"), + query.as_deref().unwrap_or("").trim() + ) + }) + .collect::>() + .join("; "); + // Whole seconds rounded up: the run's lock timeout may leave this + // wait a part of a second. + let secs = timeout.as_millis().div_ceil(1000); + if started.elapsed() >= timeout { + return Err(MigrationError::Blocked(format!( + "statements of an earlier migrator that stopped are still running, still after \ + {secs} s, the part of this run's lock timeout its earlier waits left: \ + {sessions}. They do not end by themselves; end them with SELECT \ + pg_cancel_backend(), as their role or a superuser, then run again" + ))); + } + if !announced { + println!( + "--- Waiting up to {secs} s for the statements of an earlier migrator that \ + stopped: {sessions}" + ); + announced = true; + } + tokio::time::sleep(POLL).await; + } +} + +/// The catalog's `data_database_connection_string` row; `None` when the +/// catalog has no settings table or no such row. +async fn connection_row(catalog: &PgPool) -> Result, MigrationError> { + if !relation_exists(catalog, "settings").await? { + return Ok(None); + } + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'data_database_connection_string'") + .fetch_optional(catalog) + .await + .map_err(|e| sql_error("read data_database_connection_string", &e)) +} + +/// A status session (read-only, named `extenddb status`) on `options`. +pub(crate) async fn status_session( + options: &PgConnectOptions, + timeout: Duration, +) -> Result { + one_connection(options.clone(), true, timeout) + .await + .map_err(|e| { + sql_error( + &format!( + "connect to the database {}", + options.get_database().unwrap_or("postgres") + ), + &e, + ) + }) +} + +/// Roll a status session's transaction back and close it. +pub(crate) async fn close_status(pool: PgPool) { + let _ = sqlx::raw_sql("ROLLBACK").execute(&pool).await; + pool.close().await; +} + +/// [`connection_row`] on a status session of its own. +async fn read_connection_row(target: &PostgresTarget) -> Result, MigrationError> { + let pool = status_session(&target.catalog, target.connect_timeout).await?; + let row = connection_row(&pool).await; + close_status(pool).await; + row +} + +/// The data database's connect options: those of the catalog's +/// `data_database_connection_string` row, which the server connects with, +/// when the catalog has the row; otherwise the catalog's server and +/// credentials with the target's data database. A target whose data +/// database was given on the command line keeps that name on the row's +/// server. +/// +/// # Errors +/// +/// The row is not a PostgreSQL URL, or it names another data database than +/// the target's, which was not given on the command line. +pub(crate) async fn served_data( + target: &PostgresTarget, + catalog: &PgPool, +) -> Result { + served_from(target, connection_row(catalog).await?) +} + +/// [`served_data`] from the row's value. +fn served_from( + target: &PostgresTarget, + row: Option, +) -> Result { + let Some(row) = row.filter(|r| !r.is_empty()) else { + return Ok(target.data()); + }; + let served: PgConnectOptions = row.parse().map_err(|e| { + MigrationError::Refused(format!( + "the catalog's data_database_connection_string row is not a PostgreSQL URL ({e}); \ + the server cannot connect with it either; correct the row" + )) + })?; + let served_name = served.get_database().unwrap_or("postgres").to_owned(); + if served_name == target.data_database { + Ok(served) + } else if target.data_database_given { + Ok(served.database(&target.data_database)) + } else { + Err(MigrationError::Refused(format!( + "the catalog's data_database_connection_string row names the data database \ + {served_name}, which the server serves, and its data_database_name row (or the \ + catalog database's name, when that row is missing) gives {}; make the \ + data_database_name row name {served_name}, or correct the connection string, then \ + run again", + target.data_database + ))) + } +} + +/// [`served_data`] on a status session of its own, for a caller outside the +/// runner: the connect options of the data database the server serves. +/// +/// # Errors +/// +/// As [`served_data`], or the catalog cannot be reached. +pub async fn served_data_options( + target: &PostgresTarget, +) -> Result { + served_from(target, read_connection_row(target).await?) +} + +/// For the server: refuse a catalog with no `data_database_connection_string` +/// row. The engine would then keep table data in the catalog database, while +/// the status read the ledger of the data database `target` names. +/// +/// # Errors +/// +/// The row is missing or empty, or the catalog cannot be read. +pub async fn require_data_connection_row(target: &PostgresTarget) -> Result<(), MigrationError> { + match read_connection_row(target).await? { + Some(value) if !value.is_empty() => Ok(()), + _ => Err(MigrationError::Refused(format!( + "the catalog database {} has no data_database_connection_string row, which \ + extenddb init writes; without it the server would keep table data in the catalog \ + database instead of the data database {}, whose migrations the status read. Record \ + the data database's connection string in that row, then start the server", + target.catalog_database(), + target.data_database + ))), + } +} + +/// The stop after a body failed with SQLSTATE 42501: the database's +/// [`ownership_note`], which names the objects the application role does +/// not own and the statements that give them to it. Runs after the body's +/// transaction rolled back. +pub(crate) async fn ownership_stop( + pool: &PgPool, + store: &StoreRef, + id: MigrationId, + message: &str, +) -> MigrationError { + let committed = format!( + "{message}. Nothing of migration {id} was committed in the {} database {}", + store.kind.as_str(), + store.name + ); + match ownership_note(pool, store.kind, &store.name).await { + Ok(Some(note)) => MigrationError::Store(format!( + "{committed}. Migrations run as the application role, which must own every table, \ + view and sequence in the public schema of the catalog and data databases and may \ + create tables there: {}", + note.text + )), + _ => MigrationError::Store(committed), + } +} + +/// How many objects a note names before it counts the rest. +const NAMED: usize = 5; + +/// What [`not_owned`] reads in one database. +struct NotOwned { + /// The session's role, the application role. + role: String, + /// Whether the role may create objects in the public schema. + may_create: bool, + /// The role that owns the public schema (the database's owner when the + /// schema belongs to `pg_database_owner`), and whether it is a superuser. + schema_owner: (String, bool), + /// (owner, object, whether the owner is a superuser), in order. + objects: Vec<(String, String, bool)>, +} + +/// The objects in the public schema of the database `pool` reaches whose +/// owner's privileges the session's role does not have, which is the check +/// PostgreSQL makes before it lets a role alter an object; whether the role +/// may create objects in the public schema; and who owns that schema. +/// Objects that follow another's owner (a column's sequence, an extension's +/// members) are left out, as the statements that give the others to the +/// role leave them out. +async fn not_owned(pool: &PgPool) -> Result { + let (role, may_create, schema_owner, schema_owner_super): (String, bool, String, bool) = + sqlx::query_as( + "WITH s AS (SELECT CASE WHEN pg_get_userbyid(n.nspowner) = 'pg_database_owner' \ + THEN (SELECT d.datdba FROM pg_database d WHERE d.datname = current_database()) \ + ELSE n.nspowner END AS owner, has_schema_privilege(n.oid, 'CREATE') AS may_create \ + FROM pg_namespace n WHERE n.nspname = 'public') \ + SELECT current_user::text, coalesce((SELECT may_create FROM s), true), \ + coalesce((SELECT r.rolname::text FROM s JOIN pg_roles r ON r.oid = s.owner), ''), \ + coalesce((SELECT r.rolsuper FROM s JOIN pg_roles r ON r.oid = s.owner), false)", + ) + .fetch_one(pool) + .await + .map_err(|e| sql_error("read the role's privileges", &e))?; + let objects: Vec<(String, String, bool)> = sqlx::query_as( + "SELECT pg_get_userbyid(c.relowner)::text, c.relname::text, coalesce(r.rolsuper, false) \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + LEFT JOIN pg_roles r ON r.oid = c.relowner \ + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'S', 'v', 'm') \ + AND NOT pg_has_role(c.relowner, 'USAGE') \ + AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass \ + AND d.objid = c.oid AND d.deptype IN ('a', 'i', 'e')) \ + ORDER BY 1, 2", + ) + .fetch_all(pool) + .await + .map_err(|e| sql_error("read the owners of the objects", &e))?; + Ok(NotOwned { + role, + may_create, + schema_owner: (schema_owner, schema_owner_super), + objects, + }) +} + +/// `name` as a quoted SQL identifier. +fn quoted(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// The statements that give `role` the objects of each of `owners` (name, +/// whether a superuser) in the public schema and, unless `may_create`, the +/// right to create objects there, in the order they must run, each with the +/// role that runs it. PostgreSQL lets a role that is not a superuser give +/// an object it owns only to a role it may `SET ROLE` to, and only when that +/// role may create objects in the object's schema, so for such an owner a +/// role that may grant `role` (one with its ADMIN option, which the +/// administrator that created it holds; a role cannot grant itself) first +/// runs `GRANT "" TO ""`, and the schema grant, run by the +/// schema's owner, comes before the `DO` blocks. A superuser needs neither +/// and gets the same result in this order. Each owner's `DO` block changes +/// the owner of that owner's objects with `ALTER ... OWNER TO` and of +/// nothing else: not its objects in other schemas or databases, which +/// `REASSIGN OWNED` would also move. The membership is needed only by the +/// `DO` block, so the last steps take it back with `REVOKE "" FROM +/// ""`; an owner that was a member before keeps it by skipping that +/// step. +fn give_to( + role: &str, + owners: &[(&str, bool)], + may_create: bool, + schema_owner: (&str, bool), +) -> Vec<(String, String)> { + let ident = quoted(role); + let literal = role.replace('\'', "''"); + let mut steps = Vec::new(); + for (owner, _) in owners.iter().filter(|(_, superuser)| !superuser) { + steps.push(( + format!("an administrator that may grant {role}"), + format!("GRANT {ident} TO {};", quoted(owner)), + )); + } + if !may_create { + let who = match schema_owner { + (owner, false) if !owner.is_empty() => { + format!("{owner}, which owns the public schema, or a superuser") + } + _ => "a superuser".to_owned(), + }; + steps.push((who, format!("GRANT CREATE ON SCHEMA public TO {ident};"))); + } + for (owner, superuser) in owners { + let who = if *superuser { + "a superuser".to_owned() + } else { + format!("{owner}, or a superuser") + }; + let owner = owner.replace('\'', "''"); + steps.push(( + who, + format!( + "DO $$DECLARE o record; BEGIN FOR o IN SELECT c.relname, CASE c.relkind WHEN \ + 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE \ + 'TABLE' END AS kind FROM pg_class c JOIN pg_namespace n ON n.oid = \ + c.relnamespace WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'S', 'v', \ + 'm') AND pg_get_userbyid(c.relowner) = '{owner}' AND NOT \ + pg_has_role('{literal}', c.relowner, 'USAGE') AND NOT EXISTS (SELECT 1 FROM \ + pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid AND \ + d.deptype IN ('a', 'i', 'e')) LOOP EXECUTE format('ALTER %s public.%I OWNER TO \ + %I', o.kind, o.relname, '{literal}'); END LOOP; END $$;" + ), + )); + } + for (owner, _) in owners.iter().filter(|(_, superuser)| !superuser) { + steps.push(( + format!( + "an administrator that may grant {role}, unless {owner} was a member of {role} \ + before these steps" + ), + format!("REVOKE {ident} FROM {};", quoted(owner)), + )); + } + steps +} + +/// The status note for the database `pool` reaches when the session's role +/// does not own some of the tables, views and sequences in its public +/// schema, or may not create objects in it: the objects are named with +/// their owners, with the statements that fix both, numbered in the order +/// they run and each with the role that runs it ([`give_to`]). Every +/// migration that alters such an object fails with SQLSTATE 42501, and so +/// does one that creates a table where the role may not, so the note +/// refuses `migrate --yes` before it takes the lock. +/// +/// # Errors +/// +/// The catalog of objects cannot be read. +pub(crate) async fn ownership_note( + pool: &PgPool, + kind: StoreKind, + name: &str, +) -> Result, MigrationError> { + let NotOwned { + role, + may_create, + schema_owner, + objects, + } = not_owned(pool).await?; + if objects.is_empty() && may_create { + return Ok(None); + } + let mut owners: Vec<(&str, Vec<&str>)> = Vec::new(); + let mut superusers: Vec<(&str, bool)> = Vec::new(); + for (owner, object, superuser) in &objects { + match owners.last_mut() { + Some((last, list)) if last == owner => list.push(object), + _ => { + owners.push((owner, vec![object])); + superusers.push((owner, *superuser)); + } + } + } + let held = owners + .iter() + .map(|(owner, list)| { + let mut shown = list + .iter() + .take(NAMED) + .copied() + .collect::>() + .join(", "); + if list.len() > NAMED { + shown.push_str(&format!(" and {} more", list.len() - NAMED)); + } + format!("{} owned by {owner} ({shown})", list.len()) + }) + .collect::>() + .join("; "); + let kind = kind.as_str(); + let mut found = Vec::new(); + if !objects.is_empty() { + found.push(format!( + "the application role {role} does not own these tables, views or sequences in the \ + public schema of the {kind} database {name}: {held}" + )); + } + if !may_create { + found.push(format!( + "the application role {role} may not create tables in the public schema of the \ + {kind} database {name}" + )); + } + let steps = give_to( + &role, + &superusers, + may_create, + (schema_owner.0.as_str(), schema_owner.1), + ) + .iter() + .enumerate() + .map(|(i, (who, statement))| format!("({}) as {who}: {statement}", i + 1)) + .collect::>() + .join(" "); + Ok(Some(StatusNote { + text: format!( + "{}. Migrations run as {role}, so migrate --yes refuses to apply any while this \ + holds. Connected to {name}, run these statements in this order, each as the role \ + named before it: {steps}", + found.join("; "), + ), + refuses_runs: true, + })) +} + +/// A status read that failed with SQLSTATE 42501, such as `permission +/// denied for table settings` on a catalog another role restored without +/// grants, with the ownership note of each database `databases` reaches: +/// the note reads only the system catalogs, which every role may read. When +/// the data database's name could not be read (`data_unnamed`), a line says +/// so after the catalog database's note, since that name is never guessed. +/// The error is returned as it is when no note explains it. +pub(crate) async fn denied_with_notes( + error: MigrationError, + databases: &[(StoreKind, PgConnectOptions)], + data_unnamed: bool, + timeout: Duration, +) -> MigrationError { + let mut notes = Vec::new(); + for (kind, options) in databases { + let Ok(pool) = status_session(options, timeout).await else { + continue; + }; + let name = options.get_database().unwrap_or("postgres"); + if let Ok(Some(note)) = ownership_note(&pool, *kind, name).await { + notes.push(note.text); + } + close_status(pool).await; + } + if notes.is_empty() { + return error; + } + if data_unnamed { + notes.push( + "the data database's name is in the catalog's settings, which the application role \ + cannot read, so the data database has no note here: run the catalog database's \ + statements first, or pass migrate --data-db , and the next run prints its \ + note" + .to_owned(), + ); + } + MigrationError::Refused(format!( + "{error}. The status reads these objects as the application role, which has to own \ + them:\n{}", + notes + .iter() + .map(|n| format!("Note: {n}")) + .collect::>() + .join("\n") + )) +} + +/// The stop after the last lock timeout: which tables other sessions +/// hold locks on in that database. +pub(crate) async fn lock_timeout_stop( + pool: &PgPool, + store: &StoreRef, + id: MigrationId, + last: &str, + lock_timeout_secs: u64, + retries: u32, +) -> MigrationError { + let holders: Vec<(String, i32, String)> = sqlx::query_as( + "SELECT DISTINCT c.relname::text, l.pid, coalesce(a.application_name, '') \ + FROM pg_locks l JOIN pg_class c ON c.oid = l.relation \ + LEFT JOIN pg_stat_activity a ON a.pid = l.pid \ + WHERE l.granted AND l.pid <> pg_backend_pid() \ + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) \ + AND c.relkind IN ('r', 'p') \ + ORDER BY 1, 2", + ) + .fetch_all(pool) + .await + .unwrap_or_default(); + let held = if holders.is_empty() { + "no other session holds a table lock now".to_owned() + } else { + holders + .iter() + .map(|(table, pid, app)| format!("{table} (pid {pid}, {app:?})")) + .collect::>() + .join(", ") + }; + MigrationError::Store(format!( + "migration {id} could not lock a table in the {} database {} within \ + {lock_timeout_secs} s, {} times ({last}); tables other sessions hold locks on: \ + {held}. Nothing of migration {id} was committed there; stop those sessions and run \ + again", + store.kind.as_str(), + store.name, + retries + 1 + )) +} diff --git a/crates/storage-postgres/src/migrations/tests.rs b/crates/storage-postgres/src/migrations/tests.rs new file mode 100644 index 000000000..f23e7f3bd --- /dev/null +++ b/crates/storage-postgres/src/migrations/tests.rs @@ -0,0 +1,95 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Tests of the PostgreSQL migration table that need no database. + +use extenddb_storage::migrations::{ + Body, MigrationId, RELEASED_THROUGH, StoreKind, check_released, +}; + +use super::{RELEASED, RunsIn, migration, runs_in}; + +#[test] +fn the_released_bodies_match_their_checksums() { + let lines = check_released("postgres", migration, RELEASED, RELEASED_THROUGH) + .unwrap_or_else(|e| panic!("{e}")); + println!("{lines}"); +} + +#[test] +fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { + for id in MigrationId::ALL { + let kinds = migration(id).store_kinds(); + assert_eq!(kinds.first(), Some(&StoreKind::Catalog), "{id}"); + assert!(!kinds.contains(&StoreKind::Account), "{id}"); + } + assert_eq!( + migration(MigrationId::Baseline).store_kinds(), + [StoreKind::Catalog, StoreKind::Data] + ); + assert_eq!( + migration(MigrationId::IndexTableBaseKey).store_kinds(), + [StoreKind::Catalog, StoreKind::Data] + ); +} + +#[test] +fn only_the_index_builds_run_outside_a_transaction() { + for id in MigrationId::ALL { + let autocommit = runs_in(id) == RunsIn::Autocommit; + assert_eq!(autocommit, id == MigrationId::IndexTableBaseKey, "{id}"); + if autocommit { + // A body outside a transaction is code that uses the index helper. + for body in migration(id).bodies() { + assert!(matches!(body.body, Body::Code { .. }), "{id}"); + } + } + } +} + +#[test] +fn the_baseline_files_keep_the_legacy_index_names_and_guards() { + let baseline = migration(MigrationId::Baseline); + let Body::File { text, .. } = &baseline.bodies()[0].body else { + panic!("migration 1's catalog body is a file"); + }; + // A release before 1.0 that replays legacy 001 stops at its first index, + // before it changes anything, only while the names stay the same. + for name in [ + "idx_tables_pending_transition", + "idx_stream_shards_table", + "idx_stream_records_created", + "idx_idempotency_tokens_created", + "idx_metrics_bucket", + "idx_login_attempts_principal_time", + "idx_login_attempts_source_ip_time", + "idx_backups_table", + "idx_backup_items_arn", + ] { + assert!( + text.contains(&format!("CREATE INDEX IF NOT EXISTS {name}")), + "{name}" + ); + } + let unguarded = text + .lines() + .filter(|l| { + let l = l.trim_start(); + (l.starts_with("CREATE INDEX ") || l.starts_with("CREATE UNIQUE INDEX ")) + && !l.contains("IF NOT EXISTS") + }) + .count(); + assert_eq!(unguarded, 0); + for column in [ + "table_class TEXT", + "sse_specification JSONB", + "on_demand_throughput JSONB", + ] { + assert!( + text.contains(&format!( + "ALTER TABLE tables ADD COLUMN IF NOT EXISTS {column}" + )), + "{column}" + ); + } +} diff --git a/crates/storage-postgres/src/migrations/vector_extension.rs b/crates/storage-postgres/src/migrations/vector_extension.rs new file mode 100644 index 000000000..d8e40a3d0 --- /dev/null +++ b/crates/storage-postgres/src/migrations/vector_extension.rs @@ -0,0 +1,104 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The pgvector extension attempt `migrate` makes on every run, beside the +//! ledger, for an operator who installs pgvector after `init`. +//! +//! pgvector is not a trusted extension (its control file has no `trusted = +//! true`), so a stock server refuses `CREATE EXTENSION vector` from any role +//! that is not a superuser, including the application role that owns the +//! data database. The attempt uses an admin connection only when the caller +//! has one (`--pg-user`), then the application role, which is enough on +//! managed platforms that allow the extension for a database owner. Failure +//! is a notice: a deployment without pgvector initialises and upgrades, and +//! the server refuses vector operations. + +use std::time::Duration; + +use sqlx::postgres::PgConnectOptions; +use sqlx::{ConnectOptions, Connection, PgConnection}; + +/// How long the attempt's connection may take to open, as every other +/// connection a run opens. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long `CREATE EXTENSION` may take once connected: its lock wait is +/// bounded by `lock_timeout`, and this bounds a database that stops +/// answering. +const CREATE_TIMEOUT: Duration = Duration::from_secs(30); + +async fn create_as(options: &PgConnectOptions) -> Result<(), String> { + let options = options.clone().disable_statement_logging(); + let mut conn = + match tokio::time::timeout(CONNECT_TIMEOUT, PgConnection::connect_with(&options)).await { + Ok(connected) => connected.map_err(|e| e.to_string())?, + Err(_) => { + return Err(format!( + "no connection within {} s", + CONNECT_TIMEOUT.as_secs() + )); + } + }; + let created = tokio::time::timeout(CREATE_TIMEOUT, create_in_transaction(&mut conn)).await; + let Ok(created) = created else { + // The connection is dropped unclosed: the database did not answer. + return Err(format!( + "CREATE EXTENSION did not finish within {} s", + CREATE_TIMEOUT.as_secs() + )); + }; + let _ = conn.close().await; + created.map(|_| ()).map_err(|e| { + let hint = crate::vector::create_extension_hint(&e); + format!("{e} ({hint})") + }) +} + +/// `CREATE EXTENSION` in a transaction that first sets `client_min_messages` +/// and `lock_timeout`, so the settings reach the backend that runs the +/// statement behind a transaction-mode pooler. No notice for an extension +/// that exists: the line `migrate` prints says so. +async fn create_in_transaction(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + let mut tx = conn.begin().await?; + sqlx::query("SET LOCAL client_min_messages = warning") + .execute(&mut *tx) + .await?; + sqlx::query("SET LOCAL lock_timeout = '5s'") + .execute(&mut *tx) + .await?; + sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") + .execute(&mut *tx) + .await?; + tx.commit().await +} + +/// Try to create pgvector in the data database: as `admin` when given (the +/// admin role, with the data database selected), then as the application +/// role (`data`). Returns the line to print. +pub async fn try_create_vector_extension( + data: &PgConnectOptions, + admin: Option<&PgConnectOptions>, +) -> String { + let mut failures = Vec::new(); + if let Some(admin) = admin { + let admin = admin + .clone() + .database(data.get_database().unwrap_or("postgres")); + match create_as(&admin).await { + Ok(()) => return "pgvector available; vector indexes are supported.".to_owned(), + Err(e) => failures.push(format!("as the admin role: {e}")), + } + } + match create_as(data).await { + Ok(()) => "pgvector available; vector indexes are supported.".to_owned(), + Err(e) => { + failures.push(format!("as the application role: {e}")); + format!( + "NOTICE: could not create the pgvector extension ({}). Vector indexes are \ + unavailable until it is installed; pass --pg-user with a role that may create \ + it to retry.", + failures.join("; ") + ) + } + } +} diff --git a/crates/storage-postgres/src/operations.rs b/crates/storage-postgres/src/operations.rs index 234eb520e..65171799f 100644 --- a/crates/storage-postgres/src/operations.rs +++ b/crates/storage-postgres/src/operations.rs @@ -58,8 +58,27 @@ impl OperationsEngine for PostgresOperationsEngine { Ok(()) } + /// The highest migration this binary carries: the catalog is versioned + /// by its ledger, and its `catalog_version` row holds the same integer. fn catalog_version(&self) -> String { - crate::CATALOG_VERSION.to_string() + extenddb_storage::migrations::MigrationId::LATEST + .number() + .to_string() + } + + fn setting_default(&self, key: &str) -> Option { + use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; + match key { + // The `COALESCE` fallback of CreateTable and DeleteTable, and the + // control-plane worker's. + "control_plane_delay_seconds" => { + Some(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS.to_string()) + } + INDEX_PROPAGATION_DELAY_MS => { + Some(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS.to_string()) + } + _ => None, + } } fn is_sensitive_key(&self, key: &str) -> bool { diff --git a/crates/storage-postgres/src/workers.rs b/crates/storage-postgres/src/workers.rs index 7fcc33339..ee3296a75 100644 --- a/crates/storage-postgres/src/workers.rs +++ b/crates/storage-postgres/src/workers.rs @@ -150,7 +150,7 @@ async fn read_control_plane_delay(store: &S) -> f64 { .flatten() .and_then(|v| v.parse::().ok()) .filter(|&v| v >= 0.0) - .unwrap_or(0.25) + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS) } pub(crate) async fn table_size_refresh_worker( diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index 783b8a04c..5c0a6f588 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -29,9 +29,11 @@ use extenddb_core::types::{ }; use extenddb_core::validation::validate_create_table; use extenddb_storage::error::StorageError; +use extenddb_storage::migrations::Runner; use extenddb_storage::{ BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, }; +use extenddb_storage_postgres::migrations::{PostgresMigrations, PostgresTarget}; use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; use futures::FutureExt; use sqlx::PgPool; @@ -121,9 +123,22 @@ impl Drop for DbGuard { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } async fn connect(url: &str) -> PgPool { @@ -134,15 +149,6 @@ async fn connect(url: &str) -> PgPool { .expect("connect to a scratch database") } -async fn apply(pool: &PgPool, migrations: &[&str]) { - for sql in migrations { - sqlx::raw_sql(sql) - .execute(pool) - .await - .expect("apply a shipped migration"); - } -} - async fn scratch() -> Scratch { scratch_with_pool(4).await } @@ -168,35 +174,31 @@ async fn scratch_with_pool(pool_size: u32) -> Scratch { } let catalog_url = format!("{base}/{catalog_name}"); let data_url = format!("{base}/{data_name}"); + // A fresh install, migrated by the runner as `init` does. + let target = PostgresTarget::new(catalog_url.parse().expect("a postgres URL"), &data_name) + .with_data_connection_row(&data_url); + let runner = PostgresMigrations::open_unlocked(&target) + .await + .expect("open the migration runner"); + Runner::new(&runner, "test") + .migrate() + .await + .expect("migrate the scratch databases"); + runner.close().await; let catalog = connect(&catalog_url).await; let db = connect(&data_url).await; - apply( - &catalog, - &[ - include_str!("../migrations/001_schema.sql"), - include_str!("../migrations/002_vector_indexes.sql"), - include_str!("../migrations/003_backup_definitions.sql"), - ], - ) - .await; - apply( - &db, - &[ - include_str!("../data_migrations/001_data_schema.sql"), - include_str!("../data_migrations/002_gsi_pending.sql"), - include_str!("../data_migrations/003_idempotency_account_scope.sql"), - include_str!("../data_migrations/004_vector_index_state.sql"), - ], - ) - .await; - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&catalog) - .await - .expect("pin the control-plane delay to zero"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") + for key in ["control_plane_delay_seconds", "index_propagation_delay_ms"] { + // Zero: no control-plane delay, and synchronous index propagation, + // as no queue worker runs here. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ($1, '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(key) .execute(&catalog) .await - .expect("make index propagation synchronous, as no queue worker runs here"); + .expect("set a delay to zero"); + } sqlx::query( "INSERT INTO settings (key, value) VALUES ('data_database_connection_string', $1) \ ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", @@ -1372,8 +1374,8 @@ async fn simultaneous_restores_of_one_backup_admit_exactly_one() { } /// A backup taken before catalog 0.0.4 has no definition row. Rolling the -/// catalog back to that shape, replaying migration 003 twice (as an upgrade -/// interrupted between applying and recording it would), and restoring must +/// catalog back to that shape, replaying migration 1's catalog file twice +/// (as a rerun after an interrupted upgrade would), and restoring must /// give the table as restores did before: keys and items, no secondary /// indexes, and 5/5 throughput for a provisioned table. #[tokio::test] @@ -1393,18 +1395,20 @@ async fn pre_0_0_4_backup_restores_after_migrating() { .execute(&s.catalog) .await .expect("roll the catalog back to 0.0.3"); + // Migration 1's catalog file, twice, as a rerun after an interrupted + // upgrade runs it. for _ in 0..2 { - sqlx::raw_sql(include_str!("../migrations/003_backup_definitions.sql")) + sqlx::raw_sql(include_str!("../migrations/0001_baseline.sql")) .execute(&s.catalog) .await - .expect("apply migration 003"); + .expect("apply migration 1"); } - let version: String = - sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_version'") + let recreated: bool = + sqlx::query_scalar("SELECT to_regclass('public.backup_definitions') IS NOT NULL") .fetch_one(&s.catalog) .await - .expect("version"); - assert_eq!(version, "0.0.4"); + .expect("look up backup_definitions"); + assert!(recreated); // A backup taken before 0.0.4 recorded every attribute definition of its // table, including those only its GSI and LSI used. It restores with no diff --git a/crates/storage-postgres/tests/catalog_converge.rs b/crates/storage-postgres/tests/catalog_converge.rs deleted file mode 100644 index feb761aef..000000000 --- a/crates/storage-postgres/tests/catalog_converge.rs +++ /dev/null @@ -1,612 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -//! Catalogs stamped with the current version whose shape or ledger does not -//! match this binary's migration files. -//! -//! - A catalog that ran the earlier revision of `003_backup_definitions.sql` -//! (no `table_restores.source_table_arn`, `DEFAULT NOW()` on -//! `restore_date_time`): `serve` adds the column at startup -//! (`PostgresEngine::prepare_catalog_for_serve`), waiting at most a few -//! seconds for the table lock, and keeps the default; `extenddb migrate` -//! reports and applies both steps. -//! - A catalog stamped 0.0.4 by a different `003_*` file: `serve` refuses it -//! naming this binary's 003; `extenddb migrate` reports that file as -//! pending and applies it. -//! -//! Requires `EXTENDDB_TEST_PG_CONNECTION_STRING` (a base URL -//! `postgres://user:pass@host:port`, no database); without it every test -//! reports a skip and passes. Each test creates and drops its own databases. - -use extenddb_core::types::{ - AttributeDefinition, BillingMode, CreateTableInput, DescribeTableInput, KeySchemaElement, - KeyType, ScalarAttributeType, -}; -use extenddb_storage::TableEngine; -use extenddb_storage::bootstrapper::{BootstrapConfig, Bootstrapper}; -use extenddb_storage::server_components::BackendError; -use extenddb_storage_postgres::{PostgresBootstrapper, PostgresConfig, PostgresEngine}; -use sqlx::PgPool; -use sqlx::postgres::PgPoolOptions; - -const ACCOUNT: &str = "123456789012"; -const EARLIER_003: &str = include_str!("fixtures/003_backup_definitions_earlier.sql"); - -fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) -} - -/// `postgres://user:pass@host:port` into a bootstrap config for `catalog` -/// and `data`, with the same role as admin and application user. -fn bootstrap_config(base: &str, catalog: &str, data: &str) -> BootstrapConfig { - let rest = base - .strip_prefix("postgres://") - .or_else(|| base.strip_prefix("postgresql://")) - .expect("a postgres:// URL"); - let (creds, addr) = rest.split_once('@').expect("user:pass@host:port"); - let (user, pass) = creds.split_once(':').expect("user:pass"); - let (host, port) = addr.split_once(':').unwrap_or((addr, "5432")); - BootstrapConfig { - host: host.to_owned(), - port: port.parse().expect("port"), - admin_user: user.to_owned(), - admin_password: Some(pass.to_owned()), - app_user: user.to_owned(), - app_password: pass.to_owned(), - catalog_db: catalog.to_owned(), - data_db: data.to_owned(), - } -} - -struct Scratch { - base: String, - admin: PgPool, - catalog: PgPool, - names: [String; 2], - _guard: DbGuard, -} - -/// Drops the scratch databases if the test panics, from the moment they -/// are created. -struct DbGuard { - base: String, - names: [String; 2], -} - -impl Drop for DbGuard { - fn drop(&mut self) { - if !std::thread::panicking() { - return; - } - // The test's runtime is unwinding: use a separate thread and runtime. - let (base, names) = (self.base.clone(), self.names.clone()); - let _ = std::thread::spawn(move || { - let Ok(rt) = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - else { - return; - }; - rt.block_on(async { - if let Ok(admin) = PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{base}/postgres")) - .await - { - for n in &names { - let _ = - sqlx::query(&format!("DROP DATABASE IF EXISTS \"{n}\" WITH (FORCE)")) - .execute(&admin) - .await; - } - admin.close().await; - } - }); - }) - .join(); - } -} - -impl Scratch { - /// A catalog with 001 and 002 applied and recorded, then `third` - /// applied and recorded under `third_name`; and a data database with - /// every data migration applied and recorded. - async fn new(third_name: &str, third: &str) -> Self { - let base = base_conn().expect("caller checks base_conn() first"); - let stem = format!("eddb_conv_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); - let names = [format!("{stem}_c"), format!("{stem}_d")]; - let guard = DbGuard { - base: base.clone(), - names: names.clone(), - }; - let admin = PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{base}/postgres")) - .await - .expect("admin"); - for n in &names { - sqlx::query(&format!("CREATE DATABASE \"{n}\"")) - .execute(&admin) - .await - .expect("create scratch database"); - } - let catalog = PgPoolOptions::new() - .max_connections(2) - .connect(&format!("{base}/{}", names[0])) - .await - .expect("catalog"); - for (file, sql) in [ - ( - "001_schema.sql", - include_str!("../migrations/001_schema.sql"), - ), - ( - "002_vector_indexes.sql", - include_str!("../migrations/002_vector_indexes.sql"), - ), - (third_name, third), - ] { - sqlx::raw_sql(sql).execute(&catalog).await.expect(file); - sqlx::query("INSERT INTO schema_history (filename) VALUES ($1)") - .bind(file) - .execute(&catalog) - .await - .expect("record"); - } - let s = Self { - base, - admin, - catalog, - names, - _guard: guard, - }; - // The data database the way `init` leaves it, applied directly: the - // bootstrapper's data step would also converge the catalog. - let data = PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{}/{}", s.base, s.names[1])) - .await - .expect("data"); - sqlx::query( - "CREATE TABLE schema_history (filename TEXT PRIMARY KEY, \ - applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())", - ) - .execute(&data) - .await - .expect("data ledger"); - for (file, sql) in [ - ( - "001_data_schema.sql", - include_str!("../data_migrations/001_data_schema.sql"), - ), - ( - "002_gsi_pending.sql", - include_str!("../data_migrations/002_gsi_pending.sql"), - ), - ( - "003_idempotency_account_scope.sql", - include_str!("../data_migrations/003_idempotency_account_scope.sql"), - ), - ( - "004_vector_index_state.sql", - include_str!("../data_migrations/004_vector_index_state.sql"), - ), - ("003_gsi_base_key_index", ""), - ] { - if !sql.is_empty() { - sqlx::raw_sql(sql).execute(&data).await.expect(file); - } - sqlx::query("INSERT INTO schema_history (filename) VALUES ($1)") - .bind(file) - .execute(&data) - .await - .expect("record"); - } - data.close().await; - sqlx::query( - "INSERT INTO settings (key, value) VALUES ('data_database_connection_string', $1) \ - ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", - ) - .bind(format!("{}/{}", s.base, s.names[1])) - .execute(&s.catalog) - .await - .expect("point the catalog at the data database"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&s.catalog) - .await - .expect("no control-plane delay"); - sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") - .bind(ACCOUNT) - .bind(format!("acct-{stem}")) - .execute(&s.catalog) - .await - .expect("account"); - s - } - - fn bootstrapper(&self) -> PostgresBootstrapper { - PostgresBootstrapper::new(bootstrap_config(&self.base, &self.names[0], &self.names[1])) - } - - async fn engine(&self) -> PostgresEngine { - PostgresEngine::new( - &PostgresConfig { - connection_string: format!("{}/{}", self.base, self.names[0]), - pool_size: 10, - max_item_size_bytes: 400_000, - }, - "us-east-1", - ) - .await - .expect("engine") - } - - async fn catalog_version(&self) -> String { - sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_one(&self.catalog) - .await - .expect("catalog_version") - } - - async fn column_default(&self, column: &str) -> Option> { - sqlx::query_scalar( - "SELECT column_default::text FROM information_schema.columns \ - WHERE table_schema = 'public' AND table_name = 'table_restores' \ - AND column_name = $1", - ) - .bind(column) - .fetch_optional(&self.catalog) - .await - .expect("information_schema") - } - - async fn cleanup(self) { - self.catalog.close().await; - for n in &self.names { - let _ = sqlx::query(&format!("DROP DATABASE IF EXISTS \"{n}\" WITH (FORCE)")) - .execute(&self.admin) - .await; - } - self.admin.close().await; - } -} - -fn describe(name: &str) -> DescribeTableInput { - DescribeTableInput { - table_name: name.to_owned(), - } -} - -fn table(name: &str) -> CreateTableInput { - CreateTableInput { - table_name: name.to_owned(), - key_schema: vec![KeySchemaElement { - attribute_name: "pk".to_owned(), - key_type: KeyType::Hash, - }], - attribute_definitions: vec![AttributeDefinition { - attribute_name: "pk".to_owned(), - attribute_type: ScalarAttributeType::S, - }], - billing_mode: Some(BillingMode::PayPerRequest), - ..Default::default() - } -} - -/// The earlier 0.0.4 shape: DescribeTable fails until converged; serve's -/// startup step adds the column (an earlier row kept and reported without a -/// SourceTableArn) and keeps the `restore_date_time` default, so an -/// earlier-build server's INSERT, which omits that column, still works; a -/// second serve converge does nothing. `migrate` reports both steps before, -/// the default's step after, and `migrate --yes` drops the default. -#[tokio::test] -async fn an_earlier_0_0_4_catalog_converges_at_serve_and_describe_table_works() { - if base_conn().is_none() { - eprintln!("SKIP an_earlier_0_0_4_catalog_converges_at_serve: no PostgreSQL"); - return; - } - let s = Scratch::new("003_backup_definitions.sql", EARLIER_003).await; - assert_eq!(s.catalog_version().await, "0.0.4"); - assert_eq!(s.column_default("source_table_arn").await, None); - assert!( - matches!(s.column_default("restore_date_time").await, Some(Some(_))), - "the earlier shape has a default on restore_date_time" - ); - - let engine = s.engine().await; - engine - .check_catalog_version() - .await - .expect("the version gate passes on this catalog"); - engine - .create_table(ACCOUNT, table("plain")) - .await - .expect("create a table"); - let before = engine.describe_table(ACCOUNT, describe("plain")).await; - assert!( - format!("{:?}", before.as_ref().err()).contains("source_table_arn"), - "DescribeTable on the unconverged shape: {before:?}" - ); - // A restore registration the earlier build wrote. - let restored = engine - .create_table(ACCOUNT, table("restored")) - .await - .expect("create a second table"); - sqlx::query("INSERT INTO table_restores (table_id, source_backup_arn) VALUES ($1, 'arn:b')") - .bind(&restored.table_id) - .execute(&s.catalog) - .await - .expect("an earlier-shape registration row"); - - let pending = s - .bootstrapper() - .pending_data_migrations() - .await - .expect("pending"); - assert_eq!( - pending, - vec![ - "catalog: add table_restores.source_table_arn".to_owned(), - "catalog: drop the default of table_restores.restore_date_time".to_owned(), - ] - ); - - let applied = engine - .prepare_catalog_for_serve() - .await - .expect("serve's startup converge"); - assert_eq!( - applied, - vec!["add table_restores.source_table_arn".to_owned()], - "serve adds the column and nothing else" - ); - assert_eq!(s.column_default("source_table_arn").await, Some(None)); - assert!( - matches!(s.column_default("restore_date_time").await, Some(Some(_))), - "serve keeps the default an earlier-build server still relies on" - ); - assert!( - engine - .prepare_catalog_for_serve() - .await - .expect("second serve converge") - .is_empty() - ); - // The earlier build's registration INSERT, still accepted. - let third = engine - .create_table(ACCOUNT, table("restored_by_earlier")) - .await - .expect("create a third table"); - sqlx::query("INSERT INTO table_restores (table_id, source_backup_arn) VALUES ($1, 'arn:c')") - .bind(&third.table_id) - .execute(&s.catalog) - .await - .expect("an earlier-build registration after serve converged"); - - let plain = engine - .describe_table(ACCOUNT, describe("plain")) - .await - .expect("DescribeTable after converging"); - assert!(plain.restore_summary.is_none()); - let earlier = engine - .describe_table(ACCOUNT, describe("restored")) - .await - .expect("DescribeTable of the earlier restored table"); - let summary = earlier.restore_summary.expect("its registration is kept"); - assert_eq!(summary.source_backup_arn.as_deref(), Some("arn:b")); - assert_eq!(summary.source_table_arn, None); - assert_eq!( - s.bootstrapper() - .pending_data_migrations() - .await - .expect("pending"), - vec!["catalog: drop the default of table_restores.restore_date_time".to_owned()], - "migrate still reports the step serve does not run" - ); - s.bootstrapper() - .run_data_migrations() - .await - .expect("migrate --yes"); - assert_eq!(s.column_default("restore_date_time").await, Some(None)); - assert!( - s.bootstrapper() - .pending_data_migrations() - .await - .expect("pending") - .is_empty() - ); - drop(engine); - s.cleanup().await; -} - -/// Serve's startup ALTER waits at most a few seconds for its lock: with a -/// transaction open on `table_restores` (another server mid-request), startup -/// fails with the migrate advice and changes nothing, and once the -/// transaction ends the table is not left queued behind it. When the lock is -/// free, the same call converges. -#[tokio::test] -async fn serve_converge_gives_up_on_a_held_lock_and_refuses_to_start() { - if base_conn().is_none() { - eprintln!("SKIP serve_converge_gives_up_on_a_held_lock: no PostgreSQL"); - return; - } - let s = Scratch::new("003_backup_definitions.sql", EARLIER_003).await; - let engine = s.engine().await; - let mut holder = s.catalog.begin().await.expect("holder transaction"); - sqlx::query("SELECT count(*) FROM table_restores") - .execute(&mut *holder) - .await - .expect("touch table_restores"); - - let started = std::time::Instant::now(); - let outcome = tokio::time::timeout( - std::time::Duration::from_secs(30), - engine.prepare_catalog_for_serve(), - ) - .await - .expect("startup must not wait for the lock beyond its timeout"); - let waited = started.elapsed(); - match outcome { - Err(BackendError::InitializationFailed(m)) => { - assert!(m.contains("could not lock the table within 5 s"), "{m}"); - assert!(m.contains("extenddb migrate --yes"), "{m}"); - } - other => panic!("startup must refuse while the lock is held: {other:?}"), - } - assert!( - waited >= std::time::Duration::from_secs(4), - "it waited for the lock first: {waited:?}" - ); - assert_eq!( - s.column_default("source_table_arn").await, - None, - "nothing changed" - ); - // The holder's session still works on the table: the ALTER is gone - // from the lock queue. - sqlx::query("SELECT count(*) FROM table_restores") - .execute(&mut *holder) - .await - .expect("the holder keeps working"); - holder.commit().await.expect("release"); - - let applied = engine - .prepare_catalog_for_serve() - .await - .expect("converge once the lock is free"); - assert_eq!( - applied, - vec!["add table_restores.source_table_arn".to_owned()] - ); - drop(engine); - s.cleanup().await; -} - -/// The same shape converged by `extenddb migrate --yes` (its -/// `run_data_migrations` step) rather than by serve. -#[tokio::test] -async fn migrate_converges_an_earlier_0_0_4_catalog() { - if base_conn().is_none() { - eprintln!("SKIP migrate_converges_an_earlier_0_0_4_catalog: no PostgreSQL"); - return; - } - let s = Scratch::new("003_backup_definitions.sql", EARLIER_003).await; - s.bootstrapper() - .run_data_migrations() - .await - .expect("migrate --yes"); - assert_eq!(s.column_default("source_table_arn").await, Some(None)); - assert_eq!(s.column_default("restore_date_time").await, Some(None)); - let engine = s.engine().await; - engine - .create_table(ACCOUNT, table("plain")) - .await - .expect("create a table"); - engine - .describe_table(ACCOUNT, describe("plain")) - .await - .expect("DescribeTable after migrate"); - drop(engine); - s.cleanup().await; -} - -/// A catalog stamped 0.0.4 by another branch's `003_*` (recorded under its -/// own filename). The version matches, but this binary's 003 is unrecorded: -/// `serve` refuses to start, naming the file and `extenddb migrate --yes`, -/// and changes nothing; `migrate` reports it and applies it; afterwards -/// nothing is pending, serve starts, and DescribeTable works. -#[tokio::test] -async fn an_unrecorded_catalog_migration_is_pending_even_at_the_current_version() { - if base_conn().is_none() { - eprintln!("SKIP an_unrecorded_catalog_migration_is_pending: no PostgreSQL"); - return; - } - let s = Scratch::new( - "003_other_branch.sql", - "UPDATE settings SET value = '0.0.4' WHERE key = 'catalog_version';", - ) - .await; - assert_eq!(s.catalog_version().await, "0.0.4"); - let b = s.bootstrapper(); - assert_eq!( - b.read_catalog_version().await.expect("version").as_deref(), - Some(b.expected_catalog_version().as_str()), - "the version string alone shows nothing pending" - ); - let engine = s.engine().await; - match engine.prepare_catalog_for_serve().await { - Err(BackendError::InitializationFailed(m)) => { - assert!(m.contains("003_backup_definitions.sql"), "{m}"); - assert!(m.contains("extenddb migrate --yes"), "{m}"); - assert!( - !m.contains("001_schema.sql"), - "only the unrecorded file: {m}" - ); - } - other => panic!("serve must refuse an unrecorded catalog file: {other:?}"), - } - let restores_exist: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM information_schema.tables \ - WHERE table_schema = 'public' AND table_name = 'table_restores')", - ) - .fetch_one(&s.catalog) - .await - .expect("information_schema"); - assert!(!restores_exist, "the refusal changed nothing"); - assert_eq!( - b.pending_data_migrations().await.expect("pending"), - vec!["catalog 003_backup_definitions.sql".to_owned()] - ); - b.run_data_migrations().await.expect("migrate --yes"); - assert!( - b.pending_data_migrations() - .await - .expect("pending") - .is_empty() - ); - assert!( - engine - .prepare_catalog_for_serve() - .await - .expect("serve starts after migrate") - .is_empty() - ); - engine - .create_table(ACCOUNT, table("plain")) - .await - .expect("create a table"); - engine - .describe_table(ACCOUNT, describe("plain")) - .await - .expect("DescribeTable after migrate"); - drop(engine); - s.cleanup().await; -} - -/// `serve` once refused an unreadable stored version with -/// "Internal error: "; it now gives the shared guard's refusal, -/// the text `extenddb migrate` prints for the same catalog, unprefixed. -#[tokio::test] -async fn serve_refuses_an_unreadable_catalog_version_with_the_shared_refusal() { - if base_conn().is_none() { - eprintln!("SKIP serve_refuses_an_unreadable_catalog_version: no PostgreSQL"); - return; - } - let s = Scratch::new( - "003_tampered_version.sql", - "UPDATE settings SET value = 'garbage' WHERE key = 'catalog_version';", - ) - .await; - assert_eq!(s.catalog_version().await, "garbage"); - let expected = extenddb_storage::util::check_stored_catalog_version( - Some("garbage"), - extenddb_storage_postgres::CATALOG_VERSION, - ) - .expect_err("unreadable"); - let engine = s.engine().await; - match engine.prepare_catalog_for_serve().await { - Err(BackendError::InitializationFailed(m)) => assert_eq!(m, expected), - other => panic!("serve must refuse an unreadable version: {other:?}"), - } - drop(engine); - s.cleanup().await; -} diff --git a/crates/storage-postgres/tests/doc_version_literals.rs b/crates/storage-postgres/tests/doc_version_literals.rs index 8328f8747..2e00eedb9 100644 --- a/crates/storage-postgres/tests/doc_version_literals.rs +++ b/crates/storage-postgres/tests/doc_version_literals.rs @@ -1,19 +1,14 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Version literals in the documentation must match the compiled constants. +//! Binary version literals in the documentation must match the compiled version. //! -//! The documentation carries sample output containing the binary version and the -//! catalog version. Those literals go stale silently: nothing fails, nothing warns, -//! and the next reader trusts them. This stage found five such blocks still showing -//! catalog `0.0.2` long after `0.0.3` shipped, and the binary version wrong in the -//! same lines, so the fix has to leave a check behind rather than five fresh -//! literals that expire at the next release. -//! -//! The same guard pattern already exists for the schema side: the migration runner -//! asserts the final migration writes the expected version, and the SQLite schema -//! asserts its seeded literal matches the constant. This is that pattern applied to -//! prose. +//! The documentation carries sample output containing the binary version. Those +//! literals go stale silently: nothing fails, nothing warns, and the next reader +//! trusts them, so the check stays behind rather than literals that expire at the +//! next release. Sample lines show the catalog as `catalog migration `, the +//! highest migration id the binary carries; that number is not checked, so adding a +//! migration needs no edit here. //! //! Two escapes are deliberate. Files that document a past version, an upgrade //! between versions, or another backend's version are exempt by path, each with a @@ -75,9 +70,9 @@ fn markdown_files(dir: &Path, out: &mut Vec) { /// How far past a keyword a version literal may sit and still be about it. /// -/// Wide enough for the real forms, `Catalog version 0.0.3` and -/// `catalog 0.0.3 (postgres)`, and narrow enough that an unrelated number later in a -/// sentence is not attributed to the keyword. +/// Wide enough for the real forms, `extenddb 0.1.14` and +/// `ExtendDB 0.1.14 needs migration 4`, and narrow enough that an unrelated number +/// later in a sentence is not attributed to the keyword. const LOOKAHEAD: usize = 24; /// Pull version literals that belong to `keyword` out of one line. @@ -153,14 +148,7 @@ fn exact_version(text: &str) -> Option { } } -/// Which literal a finding is about. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Keyword { - Catalog, - Binary, -} - -/// Every stale literal in one document, as `(line number, which, value)`. +/// Every stale binary version literal in one document, as `(line number, value)`. /// /// Extracted from the corpus test so the rules below can be fed the forms they claim /// to cover. The reason is a finding against this file: it had been "verified failing @@ -170,11 +158,7 @@ enum Keyword { /// enumerated in `the_matcher_sees_every_form_it_claims_to_cover` and /// `a_marker_above_a_fence_covers_the_block` rather than left to a corpus that happens /// to contain them today. -fn stale_literals( - text: &str, - expected_catalog: &str, - expected_binary: &str, -) -> Vec<(usize, Keyword, String)> { +fn stale_literals(text: &str, expected_binary: &str) -> Vec<(usize, String)> { // A marker inside a fenced block would render as sample output rather than as a // comment, and one did: it shipped in a manual as part of an error message an // operator would try to match. So a marker on the line immediately before a fence @@ -207,14 +191,9 @@ fn stale_literals( if in_marked_fence { continue; } - for found in versions_near(line, "catalog") { - if found != expected_catalog { - out.push((number, Keyword::Catalog, found)); - } - } for found in versions_near(line, "extenddb") { if found != expected_binary { - out.push((number, Keyword::Binary, found)); + out.push((number, found)); } } } @@ -230,14 +209,13 @@ fn stale_literals( fn the_matcher_sees_every_form_it_claims_to_cover() { // Must be seen: a stale value on each of these lines has to be reported. let seen = [ - " OK: Catalog version 0.0.2", - "# extenddb 0.1.5 (catalog 0.0.2) starting on 127.0.0.1:18443", - "# catalog 0.0.2 (postgres)", - "The catalog version is 0.0.2 on PostgreSQL and SQLite, stored in the `settings` table", + "# extenddb 0.1.5 (catalog migration 4) starting on 127.0.0.1:18443", + "# extenddb 0.1.5", + "ExtendDB 0.1.5 needs migration 4.", ]; for line in seen { assert!( - !stale_literals(line, "0.0.3", "0.1.6").is_empty(), + !stale_literals(line, "0.1.6").is_empty(), "the matcher must see a stale literal in this form: {line}" ); } @@ -247,11 +225,12 @@ fn the_matcher_sees_every_form_it_claims_to_cover() { "| crc-catalog | 2.4.0 | MIT OR Apache-2.0 | Yes |", "./target/release/extenddb init --bind-addr 10.0.1.5", "connection_string = \"postgresql://extenddb:***@localhost:5432/extenddb_catalog\"", - "under the key `catalog_version` and checked at startup.", + " OK: Catalog version 0.0.2", + "# catalog 0.0.2 (postgres)", ]; for line in ignored { assert_eq!( - stale_literals(line, "0.0.3", "0.1.6"), + stale_literals(line, "0.1.6"), vec![], "the matcher must ignore this line: {line}" ); @@ -269,26 +248,26 @@ fn the_matcher_sees_every_form_it_claims_to_cover() { /// one the matcher does see, so breaking the fence rule breaks these assertions. #[test] fn a_marker_above_a_fence_covers_the_block() { - let marked = "\n```\ncatalog 0.0.2\n```\n"; + let marked = "\n```\nextenddb 0.1.5\n```\n"; assert_eq!( - stale_literals(marked, "0.0.3", "0.1.6"), + stale_literals(marked, "0.1.6"), vec![], "a marker on the line before a fence must cover the whole block" ); - let unmarked = "```\ncatalog 0.0.2\n```\n"; + let unmarked = "```\nextenddb 0.1.5\n```\n"; assert_eq!( - stale_literals(unmarked, "0.0.3", "0.1.6"), - vec![(2, Keyword::Catalog, "0.0.2".to_owned())], + stale_literals(unmarked, "0.1.6"), + vec![(2, "0.1.5".to_owned())], "an unmarked fence must still be checked, or the escape becomes a blanket" ); // The marker must stop at the closing fence. Asserting the exact finding rather // than "not empty" is what makes this fail if the exemption leaks past the block. - let leaky = "\n```\ncatalog 0.0.2\n```\ncatalog 0.0.1\n"; + let leaky = "\n```\nextenddb 0.1.5\n```\nextenddb 0.1.4\n"; assert_eq!( - stale_literals(leaky, "0.0.3", "0.1.6"), - vec![(5, Keyword::Catalog, "0.0.1".to_owned())], + stale_literals(leaky, "0.1.6"), + vec![(5, "0.1.4".to_owned())], "exactly the line after the fence is reported: the block is covered, the rest is not" ); @@ -297,17 +276,16 @@ fn a_marker_above_a_fence_covers_the_block() { // code block inserted after that sentence would have been silenced with no marker // visible above it and nothing to fail. let inline = - "Some prose. \n```\ncatalog 0.0.2\n```\n"; + "Some prose. \n```\nextenddb 0.1.5\n```\n"; assert_eq!( - stale_literals(inline, "0.0.3", "0.1.6"), - vec![(3, Keyword::Catalog, "0.0.2".to_owned())], + stale_literals(inline, "0.1.6"), + vec![(3, "0.1.5".to_owned())], "a marker at the end of a sentence covers that sentence, not the next block" ); } #[test] -fn documentation_version_literals_match_the_compiled_constants() { - let expected_catalog = extenddb_storage_postgres::CATALOG_VERSION.to_string(); +fn documentation_version_literals_match_the_compiled_version() { let expected_binary = env!("CARGO_PKG_VERSION"); // inherited from the workspace let root = repo_root(); @@ -335,13 +313,9 @@ fn documentation_version_literals_match_the_compiled_constants() { continue; } let text = std::fs::read_to_string(&file).expect("read a documentation file"); - for (number, kind, found) in stale_literals(&text, &expected_catalog, expected_binary) { - let (label, expected) = match kind { - Keyword::Catalog => ("catalog", expected_catalog.as_str()), - Keyword::Binary => ("extenddb", expected_binary), - }; + for (number, found) in stale_literals(&text, expected_binary) { stale.push(format!( - "{rel}:{number}: {label} {found}, compiled constant is {expected}" + "{rel}:{number}: extenddb {found}, the compiled version is {expected_binary}" )); } } diff --git a/crates/storage-postgres/tests/fixtures/golden/0002_resource_tag_condition_keys.txt b/crates/storage-postgres/tests/fixtures/golden/0002_resource_tag_condition_keys.txt new file mode 100644 index 000000000..683f0e60a --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/golden/0002_resource_tag_condition_keys.txt @@ -0,0 +1,15 @@ +boundary role r1: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +boundary user tester: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +policy group g1 gp: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +policy role r1 rp: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +policy user tester SelfServicePolicy: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +policy user tester ddb: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +session tok: {"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Deny", "Resource": "*", "Condition": {"StringEquals": {"aws:ResourceTag/env": "prod"}}}]} +notes: +rewrote account 123456789012 assumed-role/r1/s policy (1 key(s)) +rewrote account 123456789012 group/g1 policy gp (1 key(s)) +rewrote account 123456789012 role/r1 policy (1 key(s)) +rewrote account 123456789012 role/r1 policy rp (1 key(s)) +rewrote account 123456789012 user/tester policy (1 key(s)) +rewrote account 123456789012 user/tester policy SelfServicePolicy (1 key(s)) +rewrote account 123456789012 user/tester policy ddb (1 key(s)) diff --git a/crates/storage-postgres/tests/fixtures/golden/0003_sort_key_definitions.txt b/crates/storage-postgres/tests/fixtures/golden/0003_sort_key_definitions.txt new file mode 100644 index 000000000..aec631c12 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/golden/0003_sort_key_definitions.txt @@ -0,0 +1,9 @@ +backup orphan: [] +backup seed_backup: [{"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}, {"AttributeName": "sk", "AttributeType": "S"}] +table seed_simple: [{"AttributeName": "id", "AttributeType": "N"}] +table seed_tbl: [{"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}, {"AttributeName": "sk", "AttributeType": "S"}] +notes: +restored seed_tbl: sk (S) +seed_tbl: partition key definition(s) [pk] are absent; reads and writes are unaffected, but index key type validation cannot check them +restored backup of seed_tbl: sk (S) +backup arn:orphan of gone_tbl: sort key(s) [sk] have no attribute definition, and neither its items nor its source table give the type; restoring it fails until the definition is added diff --git a/crates/storage-postgres/tests/fixtures/golden/0004_index_table_base_key.txt b/crates/storage-postgres/tests/fixtures/golden/0004_index_table_base_key.txt new file mode 100644 index 000000000..b2ed864ef --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/golden/0004_index_table_base_key.txt @@ -0,0 +1,7 @@ +CREATE INDEX "_ddb_I:seed_tbl/gsi1_base_key_idx" ON public."_ddb_I:seed_tbl/gsi1" USING btree (base_pk, base_sk_s) valid=true +CREATE INDEX "_ddb_I:seed_tbl/lsi1_base_key_idx" ON public."_ddb_I:seed_tbl/lsi1" USING btree (base_pk, base_sk_s) valid=true +notes: +going index going_gsi: skipped, the table is being deleted +lost index lost_gsi: skipped, its index table _ddb_I:lost/lost_gsi does not exist +seed_tbl index gsi1: built _ddb_I:seed_tbl/gsi1_base_key_idx on (base_pk, base_sk_s) +seed_tbl index lsi1: built _ddb_I:seed_tbl/lsi1_base_key_idx on (base_pk, base_sk_s) diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.10/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.10/catalog.sql new file mode 100644 index 000000000..3b0080232 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.10/catalog.sql @@ -0,0 +1,1020 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBYEEDXE6K', '\x592ed41fc6e7f712b5f9f70b6f5659475f16704a4cb66159619b6e78e410f30b966ef61586493d0b7a78a20ae3abf4b632a47e91889d64a09626424636286c144649549c', '123456789012', 'tester', true, '2026-10-29 07:19:59.875478+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('656239493330', 'default', '2026-10-29 07:19:53.535715+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:58.926929+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:53.775872+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403297-d4aef5c8', 'seed_backup', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:03.302669+00'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:59.244905+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:59.560627+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:59.244905+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'ac0047b4-5863-4a76-ba47-387dbad39f9e', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'f94c8496-1649-425b-b94e-cca8d7555271', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 56195, 1, 56195, 56195); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 10719, 1, 10719, 10719); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 8983, 1, 8983, 8983); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 2519, 3, 546, 1308); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 4737, 3, 1305, 2048); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 30444, 10, 1907, 5034); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 73231, 9, 413, 49403); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2116, 3, 595, 893); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 10446, 1, 10446, 10446); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 42462, 14, 1271, 8849); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:53.425334+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '656239493330'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:00.178726+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'exp', false, NULL, '2026-10-29T07:20:00', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:03.307844+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '06416496-35f7-46d4-8293-efbc905c9886', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258393189801, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.10/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.10/data.sql new file mode 100644 index 000000000..6bf7959bb --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.10/data.sql @@ -0,0 +1,449 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_06416496-35f7-46d4-8293-efbc905c9886; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_06416496-35f7-46d4-8293-efbc905c9886" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_f94c8496-1649-425b-b94e-cca8d7555271; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_06416496-35f7-46d4-8293-efbc905c9886; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_06416496-35f7-46d4-8293-efbc905c9886" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_06416496-35f7-46d4-8293-efbc905c9886" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_06416496-35f7-46d4-8293-efbc905c9886" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_f94c8496-1649-425b-b94e-cca8d7555271; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:53.486285+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:53.500526+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:53.516717+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:19:53.523655+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452073', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "23989214-88bb-4005-95d3-39cb2d842252", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258393452073", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.254253+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393452074', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "0672afbf-ec27-48ee-927a-938a8ac39e81", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452074", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.259877+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452075', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "6ad6eb4d-2045-4fab-99e6-5ffd62d8c39e", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452075", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.265454+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452076', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "56b2f15a-eed6-4630-9963-1d716a851d4e", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452076", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.269007+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393452077', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "ab0f556b-5ec0-4a7f-904a-a55714003052", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452077", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.273878+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452078', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "99928130-f928-42b8-9e4e-c4225d135af1", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452078", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.279573+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452079', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "e6677b79-c5e4-4093-bdc2-3cb1cd9a56cf", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452079", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.282748+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393452080', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "7e30fbe1-4e08-4e8c-b53b-12db2de15e8e", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452080", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.285812+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452081', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "3b9b9acd-74fc-4ced-8afc-3ca4cea7618f", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452081", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.288946+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393452082', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', 'Insert', '{"eventID": "f5208219-7bb3-4520-bfb5-9da760d533e4", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393452082", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.292024+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.181205+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.181205+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.181205+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', 'fdea6b52-24e1-48ae-8e5d-81d6b68f3c88', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.181205+00'); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258393452082, true); + + +-- +-- Name: _ddb_06416496-35f7-46d4-8293-efbc905c9886 _ddb_06416496-35f7-46d4-8293-efbc905c9886_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_06416496-35f7-46d4-8293-efbc905c9886" + ADD CONSTRAINT "_ddb_06416496-35f7-46d4-8293-efbc905c9886_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e _ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" + ADD CONSTRAINT "_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_f94c8496-1649-425b-b94e-cca8d7555271 _ddb_f94c8496-1649-425b-b94e-cca8d7555271_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" + ADD CONSTRAINT "_ddb_f94c8496-1649-425b-b94e-cca8d7555271_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88 _ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" + ADD CONSTRAINT "_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: _ddb_ac0047b4-5863-4a76-ba47-387d_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_ac0047b4-5863-4a76-ba47-387d_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e_base_key_idx" ON public."_ddb_ac0047b4-5863-4a76-ba47-387dbad39f9e" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_f94c8496-1649-425b-b94e-cca8_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_f94c8496-1649-425b-b94e-cca8_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_f94c8496-1649-425b-b94e-cca8d7555271_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_f94c8496-1649-425b-b94e-cca8d7555271_base_key_idx" ON public."_ddb_f94c8496-1649-425b-b94e-cca8d7555271" USING btree (base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" ON public."_ddb_fdea6b52-24e1-48ae-8e5d-81d6b68f3c88" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.10/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.10/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.10/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.10/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.10/release-version.txt new file mode 100644 index 000000000..56cf1c7ac --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.10/release-version.txt @@ -0,0 +1,4 @@ +extenddb 0.1.10 +catalog 0.0.2 (postgres) +commit 7ecc51630167640907eb58ae0b2f3734c8553b3b +built 2026-08-26T10:55:25Z diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.10/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.10/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.10/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.11/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.11/catalog.sql new file mode 100644 index 000000000..ffe6a73fa --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.11/catalog.sql @@ -0,0 +1,1074 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBKSFBXT8F', '\xdcef82f5b59064ff5779777f88c56d37c47ee334f3998bedfa85a590b16863391bdbd7795123ae80f6bd6039ce3f454caddac473e15ff1b0a20e93d30183399ad31720c5', '123456789012', 'tester', true, '2026-10-29 07:19:59.856469+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('611005726111', 'default', '2026-10-29 07:19:53.07403+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:58.901185+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:53.313438+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403285-0300e58a', 'seed_backup', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:03.290884+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:59.219895+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:59.539457+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:59.219895+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('f98cd4da-fd59-4d78-b38b-074badab0471', '02d624d6-cbe9-47b1-b29a-660e55b59e95', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('f98cd4da-fd59-4d78-b38b-074badab0471', 'c6f40bd3-a950-472d-825d-012dfa26de08', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 46379, 14, 1435, 9307); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 9, 1, 9, 9); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 33200, 10, 2144, 5390); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2527, 3, 716, 947); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5561, 3, 1476, 2138); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 3328, 3, 651, 1421); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 76123, 9, 463, 49768); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 56614, 1, 56614, 56614); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 11770, 1, 11770, 11770); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9440, 1, 9440, 9440); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 10821, 1, 10821, 10821); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:52.932166+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:19:52.946842+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '611005726111'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:00.161915+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'exp', false, NULL, '2026-10-29T07:20:00', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:03.296283+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '98a10bcc-94ea-46c4-8659-e924ad1b28f6', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258392703726, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.11/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.11/data.sql new file mode 100644 index 000000000..b36b9e4fe --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.11/data.sql @@ -0,0 +1,482 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_c6f40bd3-a950-472d-825d-012dfa26de08; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_f98cd4da-fd59-4d78-b38b-074badab0471; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_c6f40bd3-a950-472d-825d-012dfa26de08; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_f98cd4da-fd59-4d78-b38b-074badab0471; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:53.012472+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:53.026985+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:53.041932+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:19:53.055008+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:19:53.061913+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978072', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "4626921b-4fd1-496b-8b57-eabf4221b98c", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258392978072", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.239438+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258392978073', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "9e498ba3-1f03-46d0-b02c-5c6a2d227855", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978073", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.24558+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978074', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "0a44d4b0-a8a7-4a1c-a514-5801477f9ac4", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978074", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.250074+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978075', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "915dd686-b5d7-4270-8e71-e1828648fc2f", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978075", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.255102+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258392978076', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "f323f250-1946-4182-9893-1fb04d409c5e", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978076", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.259774+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978077', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "d9d58e4c-a3db-4942-b0d2-88241a79679f", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978077", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.263342+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978078', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "01b8b921-da98-40df-bc45-630f4d798658", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978078", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.267194+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258392978079', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "624ffeb8-1798-49d1-b40c-f49e404c9478", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978079", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.271974+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978080', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "f1088f65-a82b-4f5f-b18e-1dde7c7519c0", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978080", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.276135+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258392978081', 'f98cd4da-fd59-4d78-b38b-074badab0471', 'Insert', '{"eventID": "dfaae2e4-b23d-483d-828f-3ad3b8445655", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258392978081", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.280509+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', 'f98cd4da-fd59-4d78-b38b-074badab0471', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.164402+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', 'f98cd4da-fd59-4d78-b38b-074badab0471', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.164402+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', 'f98cd4da-fd59-4d78-b38b-074badab0471', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.164402+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', 'f98cd4da-fd59-4d78-b38b-074badab0471', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.164402+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258392978081, true); + + +-- +-- Name: _ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95 _ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" + ADD CONSTRAINT "_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6 _ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6" + ADD CONSTRAINT "_ddb_98a10bcc-94ea-46c4-8659-e924ad1b28f6_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_c6f40bd3-a950-472d-825d-012dfa26de08 _ddb_c6f40bd3-a950-472d-825d-012dfa26de08_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" + ADD CONSTRAINT "_ddb_c6f40bd3-a950-472d-825d-012dfa26de08_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_f98cd4da-fd59-4d78-b38b-074badab0471 _ddb_f98cd4da-fd59-4d78-b38b-074badab0471_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" + ADD CONSTRAINT "_ddb_f98cd4da-fd59-4d78-b38b-074badab0471_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95_base_key_idx" ON public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_02d624d6-cbe9-47b1-b29a-660e_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_02d624d6-cbe9-47b1-b29a-660e_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_02d624d6-cbe9-47b1-b29a-660e55b59e95" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_c6f40bd3-a950-472d-825d-012d_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_c6f40bd3-a950-472d-825d-012d_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_c6f40bd3-a950-472d-825d-012dfa26de08_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_c6f40bd3-a950-472d-825d-012dfa26de08_base_key_idx" ON public."_ddb_c6f40bd3-a950-472d-825d-012dfa26de08" USING btree (base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_f98cd4da-fd59-4d78-b38b-074badab0471; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_f98cd4da-fd59-4d78-b38b-074badab0471" ON public."_ddb_f98cd4da-fd59-4d78-b38b-074badab0471" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.11/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.11/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.11/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.11/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.11/release-version.txt new file mode 100644 index 000000000..295a236c8 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.11/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.11 +catalog 0.0.3 (postgres) +commit df37a77e3c96633e10c361956c43e07756756134 +built 2026-09-09T19:04:24Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.11/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.11/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.11/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.12/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.12/catalog.sql new file mode 100644 index 000000000..9cc2df8a1 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.12/catalog.sql @@ -0,0 +1,1079 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBTKI41F39', '\x6524247726a0f12d5e607a146939e02e1aca016655c85ffb0246e1a379c7cb695a99d5b4c97d78bf16f15f21f0fc40cdf256d2f09822609358ae36c9ae6cce70a82082d9', '123456789012', 'tester', true, '2026-10-29 07:19:59.302557+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('583713090701', 'default', '2026-10-29 07:19:54.876907+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:58.340742+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:55.118997+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402720-e183f760', 'seed_backup', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:02.726472+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:58.661175+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:58.984672+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:58.661175+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'fa93c5ee-d148-407d-bd04-f88e417d22d0', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'b0510e58-91bd-445c-bd2c-da61162138cf', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 43121, 14, 1528, 9263); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2682, 3, 769, 1060); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5466, 3, 1566, 2214); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 5, 5, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 2198, 2, 679, 1519); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 7, 7, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'DescribeTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 10686, 1, 10686, 10686); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 44109, 2, 867, 43242); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 29965, 10, 2093, 5158); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 10558, 1, 10558, 10558); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 50355, 1, 50355, 50355); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 9, 1, 9, 9); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 24628, 7, 475, 10503); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9406, 1, 9406, 9406); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 1277, 1, 1277, 1277); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:54.669141+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:19:54.689674+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '583713090701'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:19:59.60702+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'exp', false, NULL, '2026-10-29T07:19:59', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:02.731533+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '06587d2c-4cbc-4c97-96e4-d55d7e451773', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394318825, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.12/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.12/data.sql new file mode 100644 index 000000000..00f1e6bbe --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.12/data.sql @@ -0,0 +1,482 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_b0510e58-91bd-445c-bd2c-da61162138cf; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_b0510e58-91bd-445c-bd2c-da61162138cf; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:54.790523+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:54.810823+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:54.830658+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:19:54.856975+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:19:54.863633+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394726997', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "100ee800-e9a2-42dd-91f6-7ba0a0ecfc72", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258394726997", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.677827+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394726998', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "320b53cb-46e1-44bf-a24f-c78b9d841571", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394726998", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.684056+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394726999', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "66fd0799-f1f9-4d14-abf9-df1927b97b1c", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394726999", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.689723+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394727000', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "eaa4c326-d9d6-4ff0-b32b-982be0e6091e", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727000", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.693106+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394727001', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "8f942e89-0022-49dc-96a4-3a8ebf0df274", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727001", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.697655+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394727002', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "09be7765-3616-450e-affa-4e1ed6936529", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727002", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.701162+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394727003', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "5bfe5ce7-c659-4d95-9041-12a258eb2c5d", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727003", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.70446+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394727004', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "f1421b89-adb1-43a7-94b0-15928b024e58", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727004", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.709231+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394727005', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "39a8878e-a194-499f-a779-d570e9beaee2", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727005", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.712536+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394727006', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', 'Insert', '{"eventID": "1c8324e6-6303-4852-bd17-fef1e692567f", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394727006", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.715953+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.609731+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.609731+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.609731+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', '2ba73e4b-7be3-4c5d-9c56-2b769ab930c6', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.609731+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394727006, true); + + +-- +-- Name: _ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773 _ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773" + ADD CONSTRAINT "_ddb_06587d2c-4cbc-4c97-96e4-d55d7e451773_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6 _ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" + ADD CONSTRAINT "_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_b0510e58-91bd-445c-bd2c-da61162138cf _ddb_b0510e58-91bd-445c-bd2c-da61162138cf_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" + ADD CONSTRAINT "_ddb_b0510e58-91bd-445c-bd2c-da61162138cf_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0 _ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" + ADD CONSTRAINT "_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_b0510e58-91bd-445c-bd2c-da61162138cf_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_b0510e58-91bd-445c-bd2c-da61162138cf_base_key_idx" ON public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_b0510e58-91bd-445c-bd2c-da61_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_b0510e58-91bd-445c-bd2c-da61_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_b0510e58-91bd-445c-bd2c-da61162138cf" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0_base_key_idx" ON public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_fa93c5ee-d148-407d-bd04-f88e_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_fa93c5ee-d148-407d-bd04-f88e_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_fa93c5ee-d148-407d-bd04-f88e417d22d0" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" ON public."_ddb_2ba73e4b-7be3-4c5d-9c56-2b769ab930c6" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.12/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.12/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.12/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.12/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.12/release-version.txt new file mode 100644 index 000000000..e87940421 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.12/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.12 +catalog 0.0.3 (postgres) +commit bdb7b3df4ace3b80a6e928f144036d056aec0327 +built 2026-09-17T09:33:52Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.12/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.12/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.12/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.13/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.13/catalog.sql new file mode 100644 index 000000000..d3ecb8bad --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.13/catalog.sql @@ -0,0 +1,1074 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBAQW7WC4C', '\x979a9d3083b321b11416996ddef93eafdeb226d8df9a246e46c9eb1c9809f1e648320bbc8b678244f1da53d96f0bc84c7b009fd7b0f5902fcfe66a0b4e7561b5e858d5e0', '123456789012', 'tester', true, '2026-10-29 07:20:28.999098+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('889995684984', 'default', '2026-10-29 07:20:24.879337+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:20:28.049486+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:25.119096+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432412-8d994270', 'seed_backup', '1a29dd38-2885-400f-865d-088a6d1337cf', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:32.41848+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:20:28.364398+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:20:28.682871+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:20:28.364398+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('1a29dd38-2885-400f-865d-088a6d1337cf', '6f9f0986-a749-489c-951f-4480ba7fdac2', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('1a29dd38-2885-400f-865d-088a6d1337cf', 'ee4a0c37-7c89-416e-aa97-4a0113f8bd1c', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 30500, 10, 2052, 4856); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 68307, 9, 518, 42578); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 9896, 1, 9896, 9896); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9504, 1, 9504, 9504); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5689, 3, 1545, 2212); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2888, 3, 828, 1078); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 11522, 1, 11522, 11522); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 44027, 14, 1463, 9367); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 49833, 1, 49833, 49833); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 3328, 3, 664, 1401); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:24.697327+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:20:24.7118+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '889995684984'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:29.301408+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', '1a29dd38-2885-400f-865d-088a6d1337cf', 'exp', false, NULL, '2026-10-29T07:20:29', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:32.423843+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '56c0164f-e84d-4b14-a0fb-326da3680274', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258424488540, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.13/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.13/data.sql new file mode 100644 index 000000000..b62405c7e --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.13/data.sql @@ -0,0 +1,482 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_1a29dd38-2885-400f-865d-088a6d1337cf; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text NOT NULL COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_56c0164f-e84d-4b14-a0fb-326da3680274; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_56c0164f-e84d-4b14-a0fb-326da3680274" ( + pk text NOT NULL COLLATE pg_catalog."C", + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_6f9f0986-a749-489c-951f-4480ba7fdac2; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_1a29dd38-2885-400f-865d-088a6d1337cf; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_56c0164f-e84d-4b14-a0fb-326da3680274; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_56c0164f-e84d-4b14-a0fb-326da3680274" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_56c0164f-e84d-4b14-a0fb-326da3680274" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_56c0164f-e84d-4b14-a0fb-326da3680274" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_6f9f0986-a749-489c-951f-4480ba7fdac2; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:20:24.795649+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:20:24.819409+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:20:24.839746+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:20:24.859668+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:20:24.867762+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743531', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "64bbd65e-d5ce-49d4-aa0f-5cce40c2b802", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258424743531", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.369576+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258424743532', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "3828fc5f-2830-45ab-8f04-acf7cac45fbc", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743532", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.375214+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743533', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "55fad775-23d6-4ca8-bbe3-23c08baf9631", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743533", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.379776+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743534', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "e04dd3d6-e7a6-4bd8-ab15-44626f9e4ab5", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743534", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.383132+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258424743535', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "56750f44-f671-4a7d-95d1-101e41c909fb", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743535", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.386476+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743536', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "426619bd-a66c-4272-a795-25c62aadc8de", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743536", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.390656+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743537', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "22c5a458-0d6b-44b2-9917-1521e8debba3", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743537", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.395823+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258424743538', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "25637627-26df-422f-ba7b-624207e44055", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743538", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.401449+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743539', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "1bdc30a7-0436-4281-b064-d36632ff1aad", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743539", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.404777+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258424743540', '1a29dd38-2885-400f-865d-088a6d1337cf', 'Insert', '{"eventID": "c2ee672f-8a1c-414c-bbb0-903fb7cf3116", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258424743540", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.408031+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', '1a29dd38-2885-400f-865d-088a6d1337cf', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.303969+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', '1a29dd38-2885-400f-865d-088a6d1337cf', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.303969+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', '1a29dd38-2885-400f-865d-088a6d1337cf', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.303969+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', '1a29dd38-2885-400f-865d-088a6d1337cf', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.303969+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258424743540, true); + + +-- +-- Name: _ddb_1a29dd38-2885-400f-865d-088a6d1337cf _ddb_1a29dd38-2885-400f-865d-088a6d1337cf_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" + ADD CONSTRAINT "_ddb_1a29dd38-2885-400f-865d-088a6d1337cf_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_56c0164f-e84d-4b14-a0fb-326da3680274 _ddb_56c0164f-e84d-4b14-a0fb-326da3680274_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_56c0164f-e84d-4b14-a0fb-326da3680274" + ADD CONSTRAINT "_ddb_56c0164f-e84d-4b14-a0fb-326da3680274_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_6f9f0986-a749-489c-951f-4480ba7fdac2 _ddb_6f9f0986-a749-489c-951f-4480ba7fdac2_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" + ADD CONSTRAINT "_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c _ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" + ADD CONSTRAINT "_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_6f9f0986-a749-489c-951f-4480_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_6f9f0986-a749-489c-951f-4480_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_6f9f0986-a749-489c-951f-4480ba7fdac2_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2_base_key_idx" ON public."_ddb_6f9f0986-a749-489c-951f-4480ba7fdac2" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c_base_key_idx" ON public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_ee4a0c37-7c89-416e-aa97-4a01_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_ee4a0c37-7c89-416e-aa97-4a01_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_ee4a0c37-7c89-416e-aa97-4a0113f8bd1c" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_1a29dd38-2885-400f-865d-088a6d1337cf; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_1a29dd38-2885-400f-865d-088a6d1337cf" ON public."_ddb_1a29dd38-2885-400f-865d-088a6d1337cf" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.13/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.13/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.13/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.13/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.13/release-version.txt new file mode 100644 index 000000000..075cdb467 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.13/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.13 +catalog 0.0.3 (postgres) +commit 29b6b7fa183d99d2109e87faa2e394772906f312 +built 2026-09-30T03:47:33Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.13/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.13/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.13/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.14/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.14/catalog.sql new file mode 100644 index 000000000..399aed5ac --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.14/catalog.sql @@ -0,0 +1,1075 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDB0X1AJW1F', '\xeae969badabea908be08037e24f96b31f5b64bef35b77e49bb9f58899444a0465f1dbc72a2b93c884c5e7a47f8b414bc8c125be68518243f0954add5393159e2c11a7360', '123456789012', 'tester', true, '2026-10-29 07:20:26.300376+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('492549333154', 'default', '2026-10-29 07:20:20.521908+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:20:25.345366+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:20.760634+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258429718-91207a8a', 'seed_backup', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:29.724397+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:20:25.662979+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:20:25.981973+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:20:25.662979+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('1b515984-9e84-4b37-a63e-f4a64c5166c2', 'dc13e7ee-6af3-46af-aca3-1227b2965484', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('1b515984-9e84-4b37-a63e-f4a64c5166c2', 'd5e71af3-9bec-4a69-8e8b-4b668a393f5f', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2559, 3, 732, 953); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 14858, 1, 14858, 14858); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 31570, 10, 2080, 6497); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 56482, 1, 56482, 56482); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 69309, 9, 460, 48895); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5169, 3, 1380, 2286); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 1625, 1, 1625, 1625); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 3541, 3, 690, 1487); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 44365, 14, 1337, 9454); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9625, 1, 9625, 9625); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:20.293039+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:20:20.311573+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('resource_tag_migration', 'applied'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '492549333154'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:26.605638+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'exp', false, NULL, '2026-10-29T07:20:26.607', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:29.729698+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '3fe9fd1a-5831-44e7-b725-0b05c87bddcf', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258419933518, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.14/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.14/data.sql new file mode 100644 index 000000000..36a85a581 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.14/data.sql @@ -0,0 +1,482 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text NOT NULL COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf" ( + pk text NOT NULL COLLATE pg_catalog."C", + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_dc13e7ee-6af3-46af-aca3-1227b2965484; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_dc13e7ee-6af3-46af-aca3-1227b2965484; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:20:20.4192+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:20:20.45114+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:20:20.482139+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:20:20.504433+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:20:20.511687+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353575', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "c7cc3bd8-265c-4b6a-9ca4-9de56a0122ef", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258420353575", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.67263+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000003', '000001793258420353576', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "090547fe-a61d-43c1-ac23-999315e98aa4", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353576", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.679727+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353577', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "bf2dfef1-a322-4a2c-8129-2e5c6a3314ef", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353577", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.685198+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353578', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "685d51f7-6272-4263-a8cd-e8b4f6ab4342", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353578", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.689415+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000003', '000001793258420353579', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "f9b272b0-3d9f-41c1-8cac-62ebd5ef9a08", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353579", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.694621+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353580', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "0e984e4a-6935-4220-ae0e-44fb7645943f", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353580", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.698049+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353581', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "45e8db28-9b2f-4dca-bf2f-50d2f67f50f2", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353581", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.701315+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000003', '000001793258420353582', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "ee069e29-86b4-45ae-8550-54cbda8b4618", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353582", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.704766+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353583', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "a878a881-284b-480f-b390-894d4ce1e796", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353583", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.708638+00'); +INSERT INTO public.stream_records VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '000001793258420353584', '1b515984-9e84-4b37-a63e-f4a64c5166c2', 'Insert', '{"eventID": "7dfd0256-0a7a-43f7-847a-b61c22f6e317", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258420353584", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258427}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:27.712256+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000000', '1b515984-9e84-4b37-a63e-f4a64c5166c2', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:26.60781+00'); +INSERT INTO public.stream_shards VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000001', '1b515984-9e84-4b37-a63e-f4a64c5166c2', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:26.60781+00'); +INSERT INTO public.stream_shards VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000002', '1b515984-9e84-4b37-a63e-f4a64c5166c2', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:26.60781+00'); +INSERT INTO public.stream_shards VALUES ('shardId-1b515984-9e84-4b37-a63e-f4a64c5166c2-0000000000000003', '1b515984-9e84-4b37-a63e-f4a64c5166c2', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:26.60781+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258420353584, true); + + +-- +-- Name: _ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2 _ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" + ADD CONSTRAINT "_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf _ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf" + ADD CONSTRAINT "_ddb_3fe9fd1a-5831-44e7-b725-0b05c87bddcf_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f _ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" + ADD CONSTRAINT "_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_dc13e7ee-6af3-46af-aca3-1227b2965484 _ddb_dc13e7ee-6af3-46af-aca3-1227b2965484_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" + ADD CONSTRAINT "_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f_base_key_idx" ON public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_d5e71af3-9bec-4a69-8e8b-4b66_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_d5e71af3-9bec-4a69-8e8b-4b66_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_d5e71af3-9bec-4a69-8e8b-4b668a393f5f" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_dc13e7ee-6af3-46af-aca3-1227_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_dc13e7ee-6af3-46af-aca3-1227_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_dc13e7ee-6af3-46af-aca3-1227b2965484_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484_base_key_idx" ON public."_ddb_dc13e7ee-6af3-46af-aca3-1227b2965484" USING btree (base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" ON public."_ddb_1b515984-9e84-4b37-a63e-f4a64c5166c2" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.14/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.14/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.14/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.14/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.14/release-version.txt new file mode 100644 index 000000000..322e6c115 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.14/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.14 +catalog 0.0.3 (postgres) +commit b5bffa3803758c6028bb40ecdf2bfcae4a051a88 +built 2026-10-23T02:34:50Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.14/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.14/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.14/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.5/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.5/catalog.sql new file mode 100644 index 000000000..91d5b8cb5 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.5/catalog.sql @@ -0,0 +1,1025 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBE2AKXZO2', '\x39e0c58c449987a8cbcc17ae480b20bbbeb04f3a9b53f2bcbeab7a44168496ffab86777c1041a5e35d458114d552337f3f58e58b75871c6b6350408f7bcaf04cc41b5528', '123456789012', 'tester', true, '2026-10-29 07:19:59.019383+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('121821737578', 'default', '2026-10-29 07:19:53.974207+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:58.05069+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:54.215181+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402434-1ec581df', 'seed_backup', '19155e12-e120-4235-98c8-df2633fd2be9', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:02.439384+00'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:58.371248+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:58.697636+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:58.371248+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('19155e12-e120-4235-98c8-df2633fd2be9', '0097b29b-d6b9-481d-b3a4-7bb8ee890208', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('19155e12-e120-4235-98c8-df2633fd2be9', '49292166-e102-4d69-bbe6-9f989ad39292', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5431, 3, 1579, 2245); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 11190, 1, 11190, 11190); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 509, 1, 509, 509); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 4, 4, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 1961, 2, 750, 1211); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'DescribeTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 38805, 3, 569, 37510); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9006, 1, 9006, 9006); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 10436, 1, 10436, 10436); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 23577, 6, 359, 10895); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 33035, 10, 2063, 5286); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2781, 3, 755, 1176); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 45570, 14, 1524, 8870); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 44236, 1, 44236, 44236); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:53.869247+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('gsi_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '121821737578'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:19:59.323834+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', '19155e12-e120-4235-98c8-df2633fd2be9', 'exp', false, NULL, '2026-10-29T07:19:59', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:02.445251+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '54234b6f-a491-4519-84c3-485706b85537', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258393636284, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.5/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.5/data.sql new file mode 100644 index 000000000..61f584575 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.5/data.sql @@ -0,0 +1,434 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_19155e12-e120-4235-98c8-df2633fd2be9; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_49292166-e102-4d69-bbe6-9f989ad39292; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_54234b6f-a491-4519-84c3-485706b85537; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_54234b6f-a491-4519-84c3-485706b85537" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_19155e12-e120-4235-98c8-df2633fd2be9; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_49292166-e102-4d69-bbe6-9f989ad39292; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_54234b6f-a491-4519-84c3-485706b85537; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_54234b6f-a491-4519-84c3-485706b85537" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_54234b6f-a491-4519-84c3-485706b85537" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_54234b6f-a491-4519-84c3-485706b85537" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:53.930575+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:53.944699+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:53.962036+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896096', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "0fedbf61-18d6-4396-9f67-18cb57dbd616", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258393896096", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.387765+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393896097', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "c98f5c08-2bb5-4dc0-bfd8-1fe6d59d31b5", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896097", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.393513+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896098', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "8ae44918-def5-42ee-9a40-3673b992264d", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896098", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.398739+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896099', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "aafd48e2-68ae-4489-af33-156495049504", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896099", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.403862+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393896100', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "6a9b0557-509b-4ddd-96b9-ecd705e31626", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896100", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.407083+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896101', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "7ff2e671-7376-43cb-90f2-d814d6b2caaa", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896101", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.413155+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896102', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "f4f77b66-ee2b-4c65-bf81-ae6fec8b83a0", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896102", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.416402+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258393896103', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "9f9459fc-8668-4386-89bf-5e21d160dd19", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896103", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.421085+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896104', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "60d3ecdf-ce3c-48c8-ac3e-77c39cbd00a2", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896104", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.425638+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258393896105', '19155e12-e120-4235-98c8-df2633fd2be9', 'Insert', '{"eventID": "992f5d3b-985c-48f9-97fe-c85d4bb918c8", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258393896105", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.429296+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', '19155e12-e120-4235-98c8-df2633fd2be9', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.326369+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', '19155e12-e120-4235-98c8-df2633fd2be9', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.326369+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', '19155e12-e120-4235-98c8-df2633fd2be9', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.326369+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', '19155e12-e120-4235-98c8-df2633fd2be9', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:59.326369+00'); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258393896105, true); + + +-- +-- Name: _ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208 _ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" + ADD CONSTRAINT "_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_19155e12-e120-4235-98c8-df2633fd2be9 _ddb_19155e12-e120-4235-98c8-df2633fd2be9_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" + ADD CONSTRAINT "_ddb_19155e12-e120-4235-98c8-df2633fd2be9_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_49292166-e102-4d69-bbe6-9f989ad39292 _ddb_49292166-e102-4d69-bbe6-9f989ad39292_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" + ADD CONSTRAINT "_ddb_49292166-e102-4d69-bbe6-9f989ad39292_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_54234b6f-a491-4519-84c3-485706b85537 _ddb_54234b6f-a491-4519-84c3-485706b85537_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_54234b6f-a491-4519-84c3-485706b85537" + ADD CONSTRAINT "_ddb_54234b6f-a491-4519-84c3-485706b85537_pkey" PRIMARY KEY (pk); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: _ddb_0097b29b-d6b9-481d-b3a4-7bb8_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_0097b29b-d6b9-481d-b3a4-7bb8_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_0097b29b-d6b9-481d-b3a4-7bb8ee890208" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_49292166-e102-4d69-bbe6-9f98_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_49292166-e102-4d69-bbe6-9f98_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_49292166-e102-4d69-bbe6-9f989ad39292" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_19155e12-e120-4235-98c8-df2633fd2be9; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_19155e12-e120-4235-98c8-df2633fd2be9" ON public."_ddb_19155e12-e120-4235-98c8-df2633fd2be9" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.5/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.5/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.5/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.5/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.5/release-version.txt new file mode 100644 index 000000000..df48df62b --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.5/release-version.txt @@ -0,0 +1,4 @@ +extenddb 0.1.5 +catalog 0.0.2 (postgres) +commit bde5f1a0d20d7bcba2ba42e1480a094dd1201d82 +built 2026-08-13T22:13:46Z diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.5/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.5/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.5/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.6/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.6/catalog.sql new file mode 100644 index 000000000..0aaa21087 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.6/catalog.sql @@ -0,0 +1,1020 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBOLOHG5QZ', '\xe977792e406a1bea35c9d617adc7fa0604b86a82bb84df1fba6bfe273afc90b4d75e7651ad2cf6c4aa8139dfb3240499abd8565ea9c1e11ded66cc2bb0a87fb4a36eb956', '123456789012', 'tester', true, '2026-10-29 07:19:59.985448+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('131632539601', 'default', '2026-10-29 07:19:54.774805+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:59.029081+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:55.019979+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258403402-72bb483b', 'seed_backup', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:03.407667+00'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:59.348121+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:59.667711+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:59.348121+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'c440580e-afe8-49b4-bc9d-1b5bfecb229e', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('f47329bd-63a3-4c7a-a028-dca84bcf1b87', '154c901d-438d-4df4-bef3-a09eccbe5e4a', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 47264, 1, 47264, 47264); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 4977, 3, 1353, 2185); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 45399, 14, 1317, 8562); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2290, 3, 623, 944); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 2399, 3, 519, 1104); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 11167, 1, 11167, 11167); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 12716, 1, 12716, 12716); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 33445, 10, 2039, 5597); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 8723, 1, 8723, 8723); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 66848, 9, 373, 40017); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:54.644823+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('gsi_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '131632539601'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:00.289041+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'exp', false, NULL, '2026-10-29T07:20:00', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:03.412738+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '7ce6911f-6383-43b2-8342-bf82c9792631', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394279431, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.6/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.6/data.sql new file mode 100644 index 000000000..b2657b73a --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.6/data.sql @@ -0,0 +1,434 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_7ce6911f-6383-43b2-8342-bf82c9792631; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_7ce6911f-6383-43b2-8342-bf82c9792631" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_7ce6911f-6383-43b2-8342-bf82c9792631; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_7ce6911f-6383-43b2-8342-bf82c9792631" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_7ce6911f-6383-43b2-8342-bf82c9792631" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_7ce6911f-6383-43b2-8342-bf82c9792631" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:54.720468+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:54.738729+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:54.762508+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673916', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "00ffe0b6-dc94-4884-852b-3ecbdfd7ab75", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258394673916", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.355898+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394673917', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "ddb45bf1-061a-4bc5-a725-988303b8bd9d", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673917", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.362149+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673918', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "c2359140-6ac5-4c77-946b-5ce89af42b7b", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673918", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.367886+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673919', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "bc5ab966-1e62-4d7b-baff-04409d342d21", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673919", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.372992+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394673920', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "49029f3a-6ed2-48c5-a3f8-6541815f7e90", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673920", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.377492+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673921', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "a12cdf01-964e-4064-b268-524165ee401b", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673921", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.382156+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673922', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "6b656af9-9c85-4a9a-9ee5-31d520a0971e", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673922", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.385416+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394673923', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "6b9aa409-5e42-4850-95eb-08b3ab3cf5ea", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673923", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.389117+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673924', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "74781556-d775-4a14-9aba-32e3612b544f", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673924", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.393972+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394673925', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', 'Insert', '{"eventID": "b48eeb57-eb91-42d3-b208-e0fd3ac18889", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394673925", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258401}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:01.397207+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.291571+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.291571+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.291571+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', 'f47329bd-63a3-4c7a-a028-dca84bcf1b87', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:00.291571+00'); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394673925, true); + + +-- +-- Name: _ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a _ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" + ADD CONSTRAINT "_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_7ce6911f-6383-43b2-8342-bf82c9792631 _ddb_7ce6911f-6383-43b2-8342-bf82c9792631_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_7ce6911f-6383-43b2-8342-bf82c9792631" + ADD CONSTRAINT "_ddb_7ce6911f-6383-43b2-8342-bf82c9792631_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e _ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" + ADD CONSTRAINT "_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87 _ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" + ADD CONSTRAINT "_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: _ddb_154c901d-438d-4df4-bef3-a09e_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_154c901d-438d-4df4-bef3-a09e_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_154c901d-438d-4df4-bef3-a09eccbe5e4a" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_c440580e-afe8-49b4-bc9d-1b5b_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_c440580e-afe8-49b4-bc9d-1b5b_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_c440580e-afe8-49b4-bc9d-1b5bfecb229e" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" ON public."_ddb_f47329bd-63a3-4c7a-a028-dca84bcf1b87" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.6/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.6/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.6/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.6/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.6/release-version.txt new file mode 100644 index 000000000..b22532ea0 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.6/release-version.txt @@ -0,0 +1,4 @@ +extenddb 0.1.6 +catalog 0.0.2 (postgres) +commit ca89fc42e97054f90b7c3627076fc47c31d6eb2d +built 2026-08-18T09:56:27Z diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.6/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.6/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.6/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.8/catalog.sql b/crates/storage-postgres/tests/fixtures/released/0.1.8/catalog.sql new file mode 100644 index 000000000..56bbfa1f6 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.8/catalog.sql @@ -0,0 +1,1025 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBPSMVS87U', '\x249c018b039b8a5de8a1cadd2da428742f7342e66f0b911ac8733e644df6fd01fde2374b4de3aee77de5ec48fd113cda5d12cb27a2fcf057ed962ffcee35f188ab1200c2', '123456789012', 'tester', true, '2026-10-29 07:19:58.664627+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('347399455801', 'default', '2026-10-29 07:19:54.614876+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:19:57.703194+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:19:54.859134+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258402094-5557dbdb', 'seed_backup', '849e9859-89af-4788-9b97-5e9cb9797534', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:02.099818+00'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:19:58.021142+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:19:58.340902+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:19:58.021142+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('849e9859-89af-4788-9b97-5e9cb9797534', 'aab61207-1400-4488-b9e0-134eb7bb135e', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('849e9859-89af-4788-9b97-5e9cb9797534', 'a273c554-cea6-4cf7-b8b3-b1a7170ecaeb', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 8900, 1, 8900, 8900); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 22762, 6, 408, 10499); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 45534, 14, 1396, 8729); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5035, 3, 1428, 2127); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 33344, 10, 2095, 5405); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 4, 4, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 14, 14, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 1870, 2, 693, 1177); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 46931, 3, 536, 45676); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 10614, 1, 10614, 10614); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2179, 3, 578, 897); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 546, 1, 546, 546); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 6, 6, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 10, 1, 10, 10); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 10540, 1, 10540, 10540); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'RequestCount', '', '', 'DescribeTable', 2, 2, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:19:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 52367, 1, 52367, 52367); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:19:54.467119+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '347399455801'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:19:58.975031+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', '849e9859-89af-4788-9b97-5e9cb9797534', 'exp', false, NULL, '2026-10-29T07:19:58', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:02.104799+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', 'e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394124043, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.8/data.sql b/crates/storage-postgres/tests/fixtures/released/0.1.8/data.sql new file mode 100644 index 000000000..17f8ee760 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.8/data.sql @@ -0,0 +1,449 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_849e9859-89af-4788-9b97-5e9cb9797534; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_aab61207-1400-4488-b9e0-134eb7bb135e; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_849e9859-89af-4788-9b97-5e9cb9797534; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_aab61207-1400-4488-b9e0-134eb7bb135e; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:19:54.555135+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:19:54.577869+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:19:54.595189+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:19:54.602443+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497902', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "d28f6c12-916c-4965-b2cf-a71877ed8b1b", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258394497902", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.047251+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394497903', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "4065f32d-f003-48d4-bdbd-e0f722cdadc4", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497903", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.054041+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497904', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "860d2e86-e1c1-409a-9551-6141c780f7ed", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497904", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.059282+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497905', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "4f59d6ac-acb8-4309-8db4-6fc116d18af1", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497905", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.064113+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394497906', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "7375ed7e-53a9-46f5-94ec-8bcae02dc307", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497906", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.067384+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497907', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "6fa68ca4-4421-4260-97f1-a38633632e4c", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497907", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.070693+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497908', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "617398d6-eea6-45a5-bb5c-a478300dfa02", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497908", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.076325+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258394497909', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "0fc34faf-0d8c-4971-a626-4e9e936a333a", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497909", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.081311+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497910', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "6620f534-e40b-4bc5-9eb6-03e19beb67e7", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497910", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.084743+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258394497911', '849e9859-89af-4788-9b97-5e9cb9797534', 'Insert', '{"eventID": "5c5b8f6d-74c3-4aef-a20a-7bc756695025", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258394497911", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258400}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:00.089374+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', '849e9859-89af-4788-9b97-5e9cb9797534', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:58.977605+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', '849e9859-89af-4788-9b97-5e9cb9797534', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:58.977605+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', '849e9859-89af-4788-9b97-5e9cb9797534', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:58.977605+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', '849e9859-89af-4788-9b97-5e9cb9797534', NULL, '000000000000000000000', NULL, '2026-10-29 07:19:58.977605+00'); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 10, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258394497911, true); + + +-- +-- Name: _ddb_849e9859-89af-4788-9b97-5e9cb9797534 _ddb_849e9859-89af-4788-9b97-5e9cb9797534_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" + ADD CONSTRAINT "_ddb_849e9859-89af-4788-9b97-5e9cb9797534_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb _ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" + ADD CONSTRAINT "_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_aab61207-1400-4488-b9e0-134eb7bb135e _ddb_aab61207-1400-4488-b9e0-134eb7bb135e_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" + ADD CONSTRAINT "_ddb_aab61207-1400-4488-b9e0-134eb7bb135e_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7 _ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7" + ADD CONSTRAINT "_ddb_e4b2cdcf-17a4-4baf-a516-ccd215ec7ca7_pkey" PRIMARY KEY (pk); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: _ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb_base_key_idx" ON public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_a273c554-cea6-4cf7-b8b3-b1a7_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_a273c554-cea6-4cf7-b8b3-b1a7_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_a273c554-cea6-4cf7-b8b3-b1a7170ecaeb" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_aab61207-1400-4488-b9e0-134e_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_aab61207-1400-4488-b9e0-134e_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_aab61207-1400-4488-b9e0-134eb7bb135e_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_aab61207-1400-4488-b9e0-134eb7bb135e_base_key_idx" ON public."_ddb_aab61207-1400-4488-b9e0-134eb7bb135e" USING btree (base_pk, base_sk_s); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_849e9859-89af-4788-9b97-5e9cb9797534; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_849e9859-89af-4788-9b97-5e9cb9797534" ON public."_ddb_849e9859-89af-4788-9b97-5e9cb9797534" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.8/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.8/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.8/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.8/release-version.txt b/crates/storage-postgres/tests/fixtures/released/0.1.8/release-version.txt new file mode 100644 index 000000000..b7cc24752 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.8/release-version.txt @@ -0,0 +1,4 @@ +extenddb 0.1.8 +catalog 0.0.2 (postgres) +commit 60719ab328c48d575e83d633fa022bf1a043abe4 +built 2026-08-21T18:18:00Z diff --git a/crates/storage-postgres/tests/fixtures/released/0.1.8/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/0.1.8/seed-steps.txt new file mode 100644 index 000000000..8e0f90639 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/0.1.8/seed-steps.txt @@ -0,0 +1,6 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 diff --git a/crates/storage-postgres/tests/fixtures/released/README.md b/crates/storage-postgres/tests/fixtures/released/README.md new file mode 100644 index 000000000..a5236c72e --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/README.md @@ -0,0 +1,54 @@ +# Released PostgreSQL catalogs + +Each directory holds one released state as two plain dumps, `catalog.sql` +(the catalog database) and `data.sql` (the data database), made with +`pg_dump --inserts --no-owner --no-privileges`, plus the PostgreSQL and +release versions that made it and, for image states, the seed steps. +`tests/migrations_released.rs` loads each pair into two empty databases, +upgrades them with this binary, and compares the result with a fresh install +seeded the same way. + +| State | Made by | Seeded | +|---|---|---| +| `0.1.5`, `0.1.6`, `0.1.8`, `0.1.10`, `0.1.11`, `0.1.12`, `0.1.13`, `0.1.14` | The published `extenddb/extenddb-postgres:` image on `postgres:16.10-bookworm` | Yes | +| `pv-0.1.11`, `pv-0.1.14` | The same images on `pgvector/pgvector:pg16` | Yes, and a vector table | +| `v0.1.0`, `v0.1.2` | That tag's SQL files, applied as its `init` did; no image of these versions was published | No: catalog-only | + +Tags without a fixture ship the same schema files as one that has one, +according to their blob hashes: v0.1.1 as v0.1.0, v0.1.3 and v0.1.4 as +v0.1.5, v0.1.7 and v0.1.9 as v0.1.8. + +An image state is the release's `init` (as `docker/README.md` documents), +its `serve`, `extenddb manage` for account `123456789012`, user `tester`, a +policy and an access key, then `seed.py`: `seed_tbl` with a sort key, the GSI +`gsi1`, the LSI `lsi1`, a stream and TTL on `exp`, ten items, the backup +`seed_backup`, and `seed_simple` with three items; on pgvector also `vec_tbl` +with the vector index `vidx` and three vectors. + +## Test values + +`scrub.py` replaces, in every dump: + +- the encryption key with bytes 0 to 31 (base64 + `AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=`); +- every password hash with one fixed bcrypt string; +- the data connection string with + `postgresql://extenddb_app:fixture@fixture-host:5432/extenddb`, which the + tests overwrite with their own database. + +The access key's stored secret stays encrypted under the release's own key, +which is gone, so it cannot be decrypted; no test needs it. The dumps hold no +other secret. + +## Regenerating + + TMPDIR= ./regenerate-all.sh + +or one state with `./regenerate.sh `. Each run uses containers +`extenddb-fixture-pg-`, `extenddb-fixture-pg--init` and +`extenddb-fixture-pg--srv` on the network `extenddb-fixture-pg-net-`, +publishes the release server on 127.0.0.1 at port 21550 + n, and removes the +containers, their volumes, the network and the release image at the end +(an image that was present before the run stays). `FIXTURE_PREFIX` and +`FIXTURE_BASE_PORT` change the names and the port base. It needs Docker, `git` (for the tag states) and Python 3 with +`boto3` and `requests`. diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/catalog.sql b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/catalog.sql new file mode 100644 index 000000000..345506219 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/catalog.sql @@ -0,0 +1,1080 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.15 (Debian 16.15-1.pgdg12+2) +-- Dumped by pg_dump version 16.15 (Debian 16.15-1.pgdg12+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDBM3PCUBZ8', '\xfc0af83b556a6429b1c0c62be8c0308e78aed876b67c9e410efc7099ea6b23543ae080a0c14ce616aee0298717b2451cb0895d4bc202fce11a16878969b4a840e67fd2f8', '123456789012', 'tester', true, '2026-10-29 07:20:28.725591+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('855597040331', 'default', '2026-10-29 07:20:25.448472+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:20:27.773362+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:25.687833+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258432140-80b55b5d', 'seed_backup', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:32.145779+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:20:28.089769+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:20:28.407703+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:20:28.089769+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('d1eba15d-d07e-458c-b75c-8ed7f89f03ec', '96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('d1eba15d-d07e-458c-b75c-8ed7f89f03ec', '6e847a36-5672-4e3d-9ec1-d7b859c0b0c9', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 17, 17, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 50911, 1, 50911, 50911); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 3498, 3, 649, 1517); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 9448, 1, 9448, 9448); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 11442, 1, 11442, 11442); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'DescribeTable', 3193, 3, 788, 1490); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'CreateTable', 25149, 1, 25149, 25149); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 95352, 13, 495, 44123); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'PutItem', 6383, 3, 1931, 2503); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 9, 1, 9, 9); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9137, 1, 9137, 9137); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2862, 3, 714, 1357); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 30668, 10, 2026, 4941); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 16, 16, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'vec_tbl', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 48948, 17, 1407, 8990); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5118, 3, 1442, 2025); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:25.290003+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:20:25.305021+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '855597040331'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:29.02673+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'exp', false, NULL, '2026-10-29T07:20:29', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:32.150964+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7', NULL, false, NULL, NULL, false, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'vec_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:33.187463+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/vec_tbl', 'dfd91b2f-8fd4-4426-b581-462393bf43f1', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.vector_indexes VALUES ('dfd91b2f-8fd4-4426-b581-462393bf43f1', '3b45415f-d636-4198-b6a6-5c7b515abdb2', 'vidx', 4, 'COSINE', '{"AttributeName": "emb"}', NULL, '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL, NULL, NULL); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258425023709, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/data.sql b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/data.sql new file mode 100644 index 000000000..e3768d199 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/data.sql @@ -0,0 +1,560 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.15 (Debian 16.15-1.pgdg12+2) +-- Dumped by pg_dump version 16.15 (Debian 16.15-1.pgdg12+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: vector; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public; + + +-- +-- Name: EXTENSION vector; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION vector IS 'vector data type and ivfflat and hnsw access methods'; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" ( + pk text NOT NULL, + sk_s text, + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL, + base_sk_s text NOT NULL, + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" ( + pk text NOT NULL, + sk_s text NOT NULL, + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1" ( + pk text NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" ( + part bytea NOT NULL, + base_pk text NOT NULL, + embedding public.vector(4) NOT NULL, + nrm double precision NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1" VALUES ('v0', '{"pk": {"S": "v0"}, "emb": {"L": [{"N": "0.1"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); +INSERT INTO public."_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1" VALUES ('v1', '{"pk": {"S": "v1"}, "emb": {"L": [{"N": "0.2"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); +INSERT INTO public."_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1" VALUES ('v2', '{"pk": {"S": "v2"}, "emb": {"L": [{"N": "0.30000000000000004"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); + + +-- +-- Data for Name: _ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" VALUES ('\x00616c6c', 'v0', '[0.1,0.2,0.3,0.4]', 0.547722578048706, '{"pk": {"S": "v0"}}'); +INSERT INTO public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" VALUES ('\x00616c6c', 'v1', '[0.2,0.2,0.3,0.4]', 0.5744562745094299, '{"pk": {"S": "v1"}}'); +INSERT INTO public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" VALUES ('\x00616c6c', 'v2', '[0.3,0.2,0.3,0.4]', 0.6164414286613464, '{"pk": {"S": "v2"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:20:25.387669+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:20:25.402276+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:20:25.41619+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:20:25.429355+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:20:25.435994+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352460', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "91354c46-92be-4abe-a6b9-efd8e3262902", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258425352460", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.096419+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258425352461', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "7a822aa7-8ace-419c-bad5-477a15af379b", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352461", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.10215+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352462', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "200d0e1b-a5f9-442b-b771-b7095a1f1555", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352462", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.107212+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352463', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "e03cd727-c547-4151-8e48-510acd7b0c3d", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352463", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.11179+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258425352464', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "85b9004f-53f2-4b0b-8c5c-2d36c13b2410", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352464", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.115193+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352465', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "4e4afeee-e2d0-47b9-a304-4633fa0bdbf7", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352465", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.11965+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352466', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "95702c70-c165-4abc-9db9-cc05d6298ca9", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352466", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.12289+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000003', '000001793258425352467', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "dc31d785-8f24-4538-b438-95f19044e958", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352467", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.12608+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352468', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "af032a8e-aa8b-4f00-ba46-285d0eeac323", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352468", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.130519+00'); +INSERT INTO public.stream_records VALUES ('shardId-seed_tbl-0000000000000001', '000001793258425352469', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', 'Insert', '{"eventID": "91ecfba7-e1b9-43e2-9c0c-c174bf72af3f", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258425352469", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258430}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:30.135253+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000000', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.029253+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000001', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.029253+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000002', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.029253+00'); +INSERT INTO public.stream_shards VALUES ('shardId-seed_tbl-0000000000000003', 'd1eba15d-d07e-458c-b75c-8ed7f89f03ec', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:29.029253+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 13, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258425352469, true); + + +-- +-- Name: _ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7 _ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7" + ADD CONSTRAINT "_ddb_1fdc2b2f-ad9a-4e71-9634-8346f4dfe3f7_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9 _ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" + ADD CONSTRAINT "_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a _ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" + ADD CONSTRAINT "_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec _ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" + ADD CONSTRAINT "_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1 _ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1" + ADD CONSTRAINT "_ddb_dfd91b2f-8fd4-4426-b581-462393bf43f1_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2 _ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" + ADD CONSTRAINT "_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2_pkey" PRIMARY KEY (base_pk); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9_base_key_idx" ON public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_6e847a36-5672-4e3d-9ec1-d7b8_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_6e847a36-5672-4e3d-9ec1-d7b8_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_6e847a36-5672-4e3d-9ec1-d7b859c0b0c9" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_96b74cd9-0ad7-4b5f-ad43-8f1c_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_96b74cd9-0ad7-4b5f-ad43-8f1c_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a_base_key_idx" ON public."_ddb_96b74cd9-0ad7-4b5f-ad43-8f1cbd93ab2a" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2_part_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2_part_idx" ON public."_ddb_vec_3b45415f-d636-4198-b6a6-5c7b515abdb2" USING btree (part); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" ON public."_ddb_d1eba15d-d07e-458c-b75c-8ed7f89f03ec" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/postgres-version.txt new file mode 100644 index 000000000..784886ba7 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/release-version.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/release-version.txt new file mode 100644 index 000000000..295a236c8 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.11 +catalog 0.0.3 (postgres) +commit df37a77e3c96633e10c361956c43e07756756134 +built 2026-09-09T19:04:24Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/seed-steps.txt new file mode 100644 index 000000000..de4213103 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.11/seed-steps.txt @@ -0,0 +1,7 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 +ok create_table vec_tbl with vector index vidx diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/catalog.sql b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/catalog.sql new file mode 100644 index 000000000..37b916a04 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/catalog.sql @@ -0,0 +1,1081 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.15 (Debian 16.15-1.pgdg12+2) +-- Dumped by pg_dump version 16.15 (Debian 16.15-1.pgdg12+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + vector_indexes jsonb +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: vector_indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_indexes ( + table_id text NOT NULL, + index_id text NOT NULL, + index_name text NOT NULL, + dimensions integer NOT NULL, + distance_function text NOT NULL, + vector_attribute jsonb NOT NULL, + search_schema jsonb, + projection jsonb NOT NULL, + index_status text DEFAULT 'CREATING'::text NOT NULL, + backfilling boolean, + skipped_item_count bigint, + build_owner text, + build_heartbeat_at timestamp with time zone, + CONSTRAINT chk_vector_active_has_no_backfilling CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))), + CONSTRAINT chk_vector_dimensions_positive CHECK ((dimensions > 0)) +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.access_keys VALUES ('AKIAEXTENDDB5BE4C9P5', '\x0fe68ca886b06266f931e9d8d8fdac77a8294f1347765bb0169bdc64718e2c8eda68389c61ac713d83e44840e10fb153d8f91c6e0b67af0443444b06a366cb0939271d30', '123456789012', 'tester', true, '2026-10-29 07:20:28.353908+00'); + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('696634761358', 'default', '2026-10-29 07:20:23.898687+00'); +INSERT INTO public.accounts VALUES ('123456789012', 'seed', '2026-10-29 07:20:27.399776+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:24.138403+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p1', NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p2', NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public.backup_items VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed-p0', NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.backups VALUES ('arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl/backup/1793258431765-b90cb1ba', 'seed_backup', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'seed_tbl', '123456789012', 'AVAILABLE', 'USER', 0, 10, '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, '2026-10-29 07:20:31.77131+00', '{"Version": 1, "VectorIndexes": []}'); + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'SelfServicePolicy', '{"Version": "2012-10-17", "Statement": [{"Action": ["iam:CreateAccessKey", "iam:DeleteAccessKey", "iam:ListAccessKeys", "iam:ChangePassword"], "Effect": "Allow", "Resource": "arn:aws:iam::123456789012:user/tester"}]}', '2026-10-29 07:20:27.717961+00'); +INSERT INTO public.iam_policies VALUES ('123456789012', 'user', 'tester', 'ddb', '{"Version": "2012-10-17", "Statement": [{"Action": "dynamodb:*", "Effect": "Allow", "Resource": "*"}]}', '2026-10-29 07:20:28.035797+00'); + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.iam_users VALUES ('123456789012', 'tester', 'arn:aws:iam::123456789012:user/tester', NULL, '2026-10-29 07:20:27.717961+00'); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.indexes VALUES ('e5614f75-20b4-42ec-8cee-38c5a47607dc', 'fc7c55b3-dc1f-444c-bbeb-8a781208152a', 'gsi1', 'GSI', '[{"KeyType": "HASH", "AttributeName": "gpk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); +INSERT INTO public.indexes VALUES ('e5614f75-20b4-42ec-8cee-38c5a47607dc', '265ecb86-6caa-4ed0-8cb8-2657d21adf1a', 'lsi1', 'LSI', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "lsk"}]', '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL); + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateBackup', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Catalog', 88139, 13, 469, 44306); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'DescribeTable', 2622, 3, 771, 999); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'CreateTable', 25053, 1, 25053, 25053); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryLatency', '', 'Request', 'Data', 52658, 17, 1373, 9285); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'DescribeTable', 3509, 3, 682, 1459); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'DescribeTable', 9, 9, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Catalog', 13, 13, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'CreateTable', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_simple', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'PutItem', 5122, 3, 1419, 2174); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_simple', '', 'CreateTable', 11978, 1, 11978, 11978); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'PutItem', 16, 16, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'RequestCount', '', '', 'UpdateTimeToLive', 1, 1, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'DescribeTable', 3731, 3, 835, 1654); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateTable', 51832, 1, 51832, 51832); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'vec_tbl', '', 'PutItem', 7081, 3, 2174, 2578); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolActiveConnections', '', '', '', 0, 1, 0, 0); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'vec_tbl', '', 'PutItem', 3, 3, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'ConsumedWriteCapacityUnits', 'seed_tbl', '', 'PutItem', 10, 10, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'CreateBackup', 9491, 1, 9491, 9491); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'UpdateTimeToLive', 1647, 1, 1647, 1647); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'StorageQueryCount', '', 'Request', 'Data', 17, 17, 1, 1); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'SuccessfulRequestLatency', 'seed_tbl', '', 'PutItem', 33487, 10, 1906, 5552); +INSERT INTO public.metrics VALUES ('2026-10-29 07:20:00+00', 'PoolIdleConnections', '', '', '', 9, 1, 9, 9); + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:23.735611+00'); +INSERT INTO public.schema_history VALUES ('002_vector_indexes.sql', '2026-10-29 07:20:23.750627+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('index_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('resource_tag_migration', 'applied'); +INSERT INTO public.settings VALUES ('catalog_version', '0.0.3'); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('default_account_id', '696634761358'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.tables VALUES ('123456789012', 'seed_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}, {"KeyType": "RANGE", "AttributeName": "sk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, {"AttributeName": "lsk", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, '{"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}', 'ACTIVE', '2026-10-29 07:20:28.656978+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_tbl', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'exp', false, NULL, '2026-10-29T07:20:28.658', true, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'seed_simple', '[{"KeyType": "HASH", "AttributeName": "id"}]', '[{"AttributeName": "id", "AttributeType": "N"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:31.77666+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/seed_simple', '3c6c5804-7d8c-4350-b813-a19e74032d8a', NULL, false, NULL, NULL, false, NULL, NULL, NULL); +INSERT INTO public.tables VALUES ('123456789012', 'vec_tbl', '[{"KeyType": "HASH", "AttributeName": "pk"}]', '[{"AttributeName": "pk", "AttributeType": "S"}]', 'PAY_PER_REQUEST', NULL, NULL, 'ACTIVE', '2026-10-29 07:20:32.81424+00', 0, 0, 'arn:aws:dynamodb:us-east-1:123456789012:table/vec_tbl', '402b7bc8-1c0d-4ea8-90d0-1969c91ea382', NULL, false, NULL, NULL, false, NULL, NULL, NULL); + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: vector_indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.vector_indexes VALUES ('402b7bc8-1c0d-4ea8-90d0-1969c91ea382', 'f534d64e-bb45-452c-9578-d21b4ea5fb6e', 'vidx', 4, 'COSINE', '{"AttributeName": "emb"}', NULL, '{"ProjectionType": "ALL"}', 'ACTIVE', NULL, NULL, NULL, NULL); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258423505065, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: vector_indexes vector_indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: idx_vector_indexes_index_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: vector_indexes vector_indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_indexes + ADD CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/data.sql b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/data.sql new file mode 100644 index 000000000..512201d45 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/data.sql @@ -0,0 +1,560 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.15 (Debian 16.15-1.pgdg12+2) +-- Dumped by pg_dump version 16.15 (Debian 16.15-1.pgdg12+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: vector; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public; + + +-- +-- Name: EXTENSION vector; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION vector IS 'vector data type and ivfflat and hnsw access methods'; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: _ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a" ( + pk text NOT NULL COLLATE pg_catalog."C", + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382" ( + pk text NOT NULL COLLATE pg_catalog."C", + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text NOT NULL COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" ( + pk text NOT NULL COLLATE pg_catalog."C", + sk_s text COLLATE pg_catalog."C", + sk_n numeric, + sk_b bytea, + base_pk text NOT NULL COLLATE pg_catalog."C", + base_sk_s text NOT NULL COLLATE pg_catalog."C", + base_sk_n numeric, + base_sk_b bytea, + item_data jsonb NOT NULL +); + + +-- +-- Name: _ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" ( + part bytea NOT NULL, + base_pk text NOT NULL COLLATE pg_catalog."C", + embedding public.vector(4) NOT NULL, + nrm double precision NOT NULL, + item_data jsonb NOT NULL +); + + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: vector_index_holds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.vector_index_holds ( + table_id text NOT NULL, + index_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: _ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p0', NULL, 100, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p1', NULL, 101, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p2', NULL, 102, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p0', NULL, 103, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p1', NULL, 104, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p2', NULL, 105, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p0', NULL, 106, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p1', NULL, 107, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p2', NULL, 108, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" VALUES ('seed-p0', NULL, 109, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a" VALUES ('0', '{"v": {"S": "x"}, "id": {"N": "0"}}'); +INSERT INTO public."_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a" VALUES ('1', '{"v": {"S": "x"}, "id": {"N": "1"}}'); +INSERT INTO public."_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a" VALUES ('2', '{"v": {"S": "x"}, "id": {"N": "2"}}'); + + +-- +-- Data for Name: _ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382" VALUES ('v0', '{"pk": {"S": "v0"}, "emb": {"L": [{"N": "0.1"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); +INSERT INTO public."_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382" VALUES ('v1', '{"pk": {"S": "v1"}, "emb": {"L": [{"N": "0.2"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); +INSERT INTO public."_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382" VALUES ('v2', '{"pk": {"S": "v2"}, "emb": {"L": [{"N": "0.30000000000000004"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}'); + + +-- +-- Data for Name: _ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" VALUES ('seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g0', 's000', NULL, NULL, 'seed-p0', 's000', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g1', 's001', NULL, NULL, 'seed-p1', 's001', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g0', 's002', NULL, NULL, 'seed-p2', 's002', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g1', 's003', NULL, NULL, 'seed-p0', 's003', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g0', 's004', NULL, NULL, 'seed-p1', 's004', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g1', 's005', NULL, NULL, 'seed-p2', 's005', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g0', 's006', NULL, NULL, 'seed-p0', 's006', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g1', 's007', NULL, NULL, 'seed-p1', 's007', NULL, NULL, '{"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g0', 's008', NULL, NULL, 'seed-p2', 's008', NULL, NULL, '{"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}'); +INSERT INTO public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" VALUES ('g1', 's009', NULL, NULL, 'seed-p0', 's009', NULL, NULL, '{"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}'); + + +-- +-- Data for Name: _ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" VALUES ('\x00616c6c', 'v0', '[0.1,0.2,0.3,0.4]', 0.547722578048706, '{"pk": {"S": "v0"}}'); +INSERT INTO public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" VALUES ('\x00616c6c', 'v1', '[0.2,0.2,0.3,0.4]', 0.5744562745094299, '{"pk": {"S": "v1"}}'); +INSERT INTO public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" VALUES ('\x00616c6c', 'v2', '[0.3,0.2,0.3,0.4]', 0.6164414286613464, '{"pk": {"S": "v2"}}'); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:20:23.837247+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:20:23.851674+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:20:23.865411+00'); +INSERT INTO public.schema_history VALUES ('004_vector_index_state.sql', '2026-10-29 07:20:23.878657+00'); +INSERT INTO public.schema_history VALUES ('003_gsi_base_key_index', '2026-10-29 07:20:23.885859+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801341', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "819c76f0-b73c-4f83-95ef-62554381f2f5", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s000"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "100"}, "payload": {"S": "item 0"}}, "SizeBytes": 46, "SequenceNumber": "000001793258423801341", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.719082+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000003', '000001793258423801342', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "4e68c958-0731-4c25-90f7-719df56afa29", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s001"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "101"}, "payload": {"S": "item 1"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801342", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.72468+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801343', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "03e391a9-0720-405b-bbe0-49d8a414ccff", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s002"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "102"}, "payload": {"S": "item 2"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801343", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.731027+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801344', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "545607aa-27db-452b-a40b-4685cfc04741", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s003"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "103"}, "payload": {"S": "item 3"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801344", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.734604+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000003', '000001793258423801345', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "4044929f-0f68-47c4-bf6f-c651ab1e6f54", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s004"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "104"}, "payload": {"S": "item 4"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801345", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.741369+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801346', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "fb4c475d-85e2-414c-8f56-8c3bbab36476", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s005"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "105"}, "payload": {"S": "item 5"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801346", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.745888+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801347', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "e4a77fd8-0099-4c94-be08-682ba2f912d2", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s006"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "106"}, "payload": {"S": "item 6"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801347", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.749235+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000003', '000001793258423801348', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "abd94ae3-1866-4f72-b980-d0f9f592440e", "dynamodb": {"Keys": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}}, "NewImage": {"pk": {"S": "seed-p1"}, "sk": {"S": "s007"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "107"}, "payload": {"S": "item 7"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801348", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.753673+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801349', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "31d0cb69-b9a5-42cb-ad2f-afc4db7763a3", "dynamodb": {"Keys": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}}, "NewImage": {"pk": {"S": "seed-p2"}, "sk": {"S": "s008"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g0"}, "lsk": {"N": "108"}, "payload": {"S": "item 8"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801349", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.757548+00'); +INSERT INTO public.stream_records VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', '000001793258423801350', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', 'Insert', '{"eventID": "d95f5ace-26c2-428e-a602-a3cb433de2f1", "dynamodb": {"Keys": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}}, "NewImage": {"pk": {"S": "seed-p0"}, "sk": {"S": "s009"}, "exp": {"N": "4102444800"}, "gpk": {"S": "g1"}, "lsk": {"N": "109"}, "payload": {"S": "item 9"}}, "SizeBytes": 47, "SequenceNumber": "000001793258423801350", "StreamViewType": "NEW_AND_OLD_IMAGES", "ApproximateCreationDateTime": 1793258429}, "awsRegion": "us-east-1", "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.1"}', '2026-10-29 07:20:29.760587+00'); + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.stream_shards VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000000', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:28.659262+00'); +INSERT INTO public.stream_shards VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000001', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:28.659262+00'); +INSERT INTO public.stream_shards VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000002', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:28.659262+00'); +INSERT INTO public.stream_shards VALUES ('shardId-e5614f75-20b4-42ec-8cee-38c5a47607dc-0000000000000003', 'e5614f75-20b4-42ec-8cee-38c5a47607dc', NULL, '000000000000000000000', NULL, '2026-10-29 07:20:28.659262+00'); + + +-- +-- Data for Name: vector_index_holds; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 13, true); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258423801350, true); + + +-- +-- Name: _ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a _ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" + ADD CONSTRAINT "_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a _ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a" + ADD CONSTRAINT "_ddb_3c6c5804-7d8c-4350-b813-a19e74032d8a_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382 _ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382" + ADD CONSTRAINT "_ddb_402b7bc8-1c0d-4ea8-90d0-1969c91ea382_pkey" PRIMARY KEY (pk); + + +-- +-- Name: _ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc _ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" + ADD CONSTRAINT "_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc_pkey" PRIMARY KEY (pk, sk_s); + + +-- +-- Name: _ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a _ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" + ADD CONSTRAINT "_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a_pkey" PRIMARY KEY (pk, base_pk, base_sk_s); + + +-- +-- Name: _ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e _ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" + ADD CONSTRAINT "_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e_pkey" PRIMARY KEY (base_pk); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: vector_index_holds vector_index_holds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.vector_index_holds + ADD CONSTRAINT vector_index_holds_pkey PRIMARY KEY (table_id, index_id); + + +-- +-- Name: _ddb_265ecb86-6caa-4ed0-8cb8-2657_pk_sk_n_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_265ecb86-6caa-4ed0-8cb8-2657_pk_sk_n_base_pk_base_sk_s_idx" ON public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" USING btree (pk, sk_n, base_pk, base_sk_s); + + +-- +-- Name: _ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a_base_key_idx" ON public."_ddb_265ecb86-6caa-4ed0-8cb8-2657d21adf1a" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a_base_key_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a_base_key_idx" ON public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" USING btree (base_pk, base_sk_s); + + +-- +-- Name: _ddb_fc7c55b3-dc1f-444c-bbeb-8a78_pk_sk_s_base_pk_base_sk_s_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_fc7c55b3-dc1f-444c-bbeb-8a78_pk_sk_s_base_pk_base_sk_s_idx" ON public."_ddb_fc7c55b3-dc1f-444c-bbeb-8a781208152a" USING btree (pk, sk_s, base_pk, base_sk_s); + + +-- +-- Name: _ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e_part_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e_part_idx" ON public."_ddb_vec_f534d64e-bb45-452c-9578-d21b4ea5fb6e" USING btree (part); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_ttl__ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "idx_ttl__ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" ON public."_ddb_e5614f75-20b4-42ec-8cee-38c5a47607dc" USING btree (((((item_data -> 'exp'::text) ->> 'N'::text))::bigint)) WHERE (((item_data -> 'exp'::text) ->> 'N'::text) IS NOT NULL); + + +-- +-- Name: idx_vector_index_holds_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/postgres-version.txt new file mode 100644 index 000000000..784886ba7 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/release-version.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/release-version.txt new file mode 100644 index 000000000..322e6c115 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/release-version.txt @@ -0,0 +1,5 @@ +extenddb 0.1.14 +catalog 0.0.3 (postgres) +commit b5bffa3803758c6028bb40ecdf2bfcae4a051a88 +built 2026-10-23T02:34:50Z +test-hooks disabled diff --git a/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/seed-steps.txt b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/seed-steps.txt new file mode 100644 index 000000000..de4213103 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/pv-0.1.14/seed-steps.txt @@ -0,0 +1,7 @@ +ok create_table seed_tbl +ok update_time_to_live seed_tbl +ok put_item seed_tbl x10 +ok create_backup seed_backup +ok create_table seed_simple +ok put_item seed_simple x3 +ok create_table vec_tbl with vector index vidx diff --git a/crates/storage-postgres/tests/fixtures/released/regenerate-all.sh b/crates/storage-postgres/tests/fixtures/released/regenerate-all.sh new file mode 100755 index 000000000..dcf5ccb7f --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/regenerate-all.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Regenerate every released PostgreSQL fixture, a few at a time, with +# regenerate.sh. Logs go to .log beside each fixture directory's +# parent unless LOGDIR is set. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +LOGDIR="${LOGDIR:-$HERE}" +STATES=(0.1.5 0.1.6 0.1.8 0.1.10 0.1.11 0.1.12 0.1.13 0.1.14 pv-0.1.11 pv-0.1.14 v0.1.0 v0.1.2) +PARALLEL="${PARALLEL:-4}" +n=0 +pids=() +for state in "${STATES[@]}"; do + n=$((n + 1)) + bash "$HERE/regenerate.sh" "$state" "$n" > "$LOGDIR/$state.log" 2>&1 & + pids+=("$!") + if [ "${#pids[@]}" -ge "$PARALLEL" ]; then + wait "${pids[0]}"; echo "$(tail -n 1 "$LOGDIR/${STATES[$((n - ${#pids[@]}))]}.log")" + pids=("${pids[@]:1}") + fi +done +for pid in "${pids[@]}"; do wait "$pid"; done +for state in "${STATES[@]}"; do echo "$state: $(tail -n 1 "$LOGDIR/$state.log")"; done diff --git a/crates/storage-postgres/tests/fixtures/released/regenerate.sh b/crates/storage-postgres/tests/fixtures/released/regenerate.sh new file mode 100755 index 000000000..9a19a928b --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/regenerate.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Regenerate one released PostgreSQL fixture: /catalog.sql and +# /data.sql in this directory. +# +# Usage: regenerate.sh +# +# 0.1.5 ... 0.1.14 (the published extenddb/extenddb-postgres image +# of that version), pv-0.1.11 or pv-0.1.14 (the same image on +# pgvector), or v0.1.0 and v0.1.2 (built from that tag's SQL files, +# catalog-only, because no image of those versions was published). +# A number that tells concurrent runs apart. The containers are +# $FIXTURE_PREFIX- (PostgreSQL) and $FIXTURE_PREFIX--init / +# $FIXTURE_PREFIX--srv (the release), the network is +# $FIXTURE_PREFIX-net-, and the release server listens on +# 127.0.0.1:$((FIXTURE_BASE_PORT + n)). +# +# FIXTURE_PREFIX defaults to extenddb-fixture-pg and FIXTURE_BASE_PORT to +# 21550; set them when those names or ports are taken. +# +# An image state runs the release's `init` as docker/README.md documents, its +# `serve`, `extenddb manage` for an account, a user, a policy and a key, then +# seed.py: a table with a sort key, a GSI, an LSI, TTL and a stream, items, and +# a backup (and on pgvector a table with a vector index). The server is +# stopped and both databases are dumped with `pg_dump --inserts --no-owner +# --no-privileges`; scrub.py replaces the encryption key, every password hash +# and the connection string's password with fixed test values. Everything the +# script creates is removed at the end, and the release image too when this +# run pulled it (KEEP_IMAGE=1 keeps it). +set -euo pipefail + +STATE="$1"; N="$2" +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../../.." && pwd)" +OUT="$HERE/$STATE" +PREFIX="${FIXTURE_PREFIX:-extenddb-fixture-pg}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/$PREFIX-$STATE.XXXXXX")" +NET="$PREFIX-net-$N" +PGC="$PREFIX-$N" +PORT=$((${FIXTURE_BASE_PORT:-21550} + N)) +PGPW=PgAdminPw123 +APPPW=ExtendDBApp123 +ADMINPW=ExtendDBLocalAdmin123 +CFG=/var/lib/extenddb/extenddb.toml +UIDGID="$(id -u):$(id -g)" +case "$STATE" in + pv-*) VERSION="${STATE#pv-}"; PGIMG=pgvector/pgvector:pg16; VECTOR=1 ;; + v*) VERSION=""; PGIMG=public.ecr.aws/docker/library/postgres:16.10-bookworm; VECTOR=0 ;; + *) VERSION="$STATE"; PGIMG=public.ecr.aws/docker/library/postgres:16.10-bookworm; VECTOR=0 ;; +esac +IMG="extenddb/extenddb-postgres:$VERSION" +# An image that was here before this run stays. +HAD_IMAGE=0 +if [ -n "$VERSION" ] && docker image inspect "$IMG" >/dev/null 2>&1; then + HAD_IMAGE=1 +fi + +note() { echo "[$(date -u +%H:%M:%S)] $STATE: $*"; } +cleanup() { + docker rm -f -v "$PGC-srv" "$PGC-init" "$PGC" >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true + if [ -n "$VERSION" ] && [ "$HAD_IMAGE" != 1 ] && [ "${KEEP_IMAGE:-0}" != 1 ]; then + docker rmi "$IMG" >/dev/null 2>&1 || true + fi + rm -rf "$WORK" +} +trap cleanup EXIT + +psql_in() { docker exec -i "$PGC" psql -X -q -v ON_ERROR_STOP=1 -U "$@"; } + +docker network create "$NET" >/dev/null +note "start $PGIMG as $PGC" +docker run -d --name "$PGC" --network "$NET" -e POSTGRES_PASSWORD="$PGPW" "$PGIMG" >/dev/null +for _ in $(seq 1 120); do + docker exec "$PGC" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null && break + sleep 0.5 +done + +if [ -z "$VERSION" ]; then + # A tag fixture: what that tag's `init` ran, from its own files. + TAG="$STATE" + note "build from the $TAG files" + psql_in postgres -d postgres <<<"CREATE USER extenddb_app WITH PASSWORD '$APPPW'; GRANT extenddb_app TO postgres;" + psql_in postgres -d postgres <<<"CREATE DATABASE extenddb_catalog OWNER extenddb_app; CREATE DATABASE extenddb OWNER extenddb_app;" + git -C "$REPO" show "$TAG:crates/storage-postgres/migrations/001_schema.sql" | psql_in extenddb_app -d extenddb_catalog + psql_in extenddb_app -d extenddb_catalog <<<"INSERT INTO schema_history (filename) VALUES ('001_schema.sql');" + python3 "$HERE/scrub.py" --bootstrap "$TAG" | psql_in extenddb_app -d extenddb_catalog + if [ "$TAG" = v0.1.0 ]; then + # v0.1.0 ran data 001 and kept no data ledger. + git -C "$REPO" show "$TAG:crates/storage-postgres/data_migrations/001_data_schema.sql" | psql_in extenddb_app -d extenddb + else + psql_in extenddb_app -d extenddb <<<"CREATE TABLE schema_history (filename TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW());" + for f in $(git -C "$REPO" ls-tree --name-only "$TAG" crates/storage-postgres/data_migrations/ | sort); do + git -C "$REPO" show "$TAG:$f" | psql_in extenddb_app -d extenddb + psql_in extenddb_app -d extenddb <<<"INSERT INTO schema_history (filename) VALUES ('$(basename "$f")');" + done + fi +else + note "init with $IMG" + docker pull -q "$IMG" >/dev/null + mkdir -p "$WORK/state" + docker run --rm --name "$PGC-init" --network "$NET" --user "$UIDGID" \ + -v "$WORK/state":/var/lib/extenddb -e HOME=/var/lib/extenddb \ + -e EXTENDDB_PG_PASSWORD="$PGPW" -e EXTENDDB_APP_PASSWORD="$APPPW" \ + -e EXTENDDB_ADMIN_USER=admin -e EXTENDDB_ADMIN_PASSWORD="$ADMINPW" \ + "$IMG" init --config "$CFG" --backend postgres --pg-host "$PGC" --pg-port 5432 \ + --pg-user postgres --extenddb-user extenddb_app --bind-addr 0.0.0.0 --tls-san localhost \ + > "$WORK/init.log" 2>&1 || { cat "$WORK/init.log"; exit 1; } + note "serve on 127.0.0.1:$PORT" + docker run -d --name "$PGC-srv" --network "$NET" --user "$UIDGID" \ + -v "$WORK/state":/var/lib/extenddb -e HOME=/var/lib/extenddb \ + -p "127.0.0.1:$PORT:18443" "$IMG" >/dev/null + up=0 + for _ in $(seq 1 120); do + if docker exec "$PGC-srv" extenddb healthcheck --config "$CFG" >/dev/null 2>&1; then up=1; break; fi + sleep 0.5 + done + [ "$up" = 1 ] || { docker logs "$PGC-srv"; exit 1; } + M=(docker exec -e EXTENDDB_PASSWORD="$ADMINPW" "$PGC-srv" extenddb manage --user admin --config "$CFG") + "${M[@]}" create-account --account-id 123456789012 --account-name seed >/dev/null + "${M[@]}" create-user --account-id 123456789012 --user-name tester >/dev/null + "${M[@]}" put-user-policy --account-id 123456789012 --user-name tester --policy-name ddb \ + --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' >/dev/null + "${M[@]}" create-access-key --account-id 123456789012 --user-name tester > "$WORK/key.json" + python3 "$HERE/seed.py" "https://127.0.0.1:$PORT" "$WORK/key.json" "$VECTOR" "$WORK/seed.json" + docker stop -t 20 "$PGC-srv" >/dev/null +fi + +note "dump" +mkdir -p "$OUT" +for pair in catalog:extenddb_catalog data:extenddb; do + docker exec "$PGC" pg_dump -U postgres --inserts --no-owner --no-privileges "${pair#*:}" \ + | python3 "$HERE/scrub.py" --dump > "$OUT/${pair%%:*}.sql" +done +docker exec "$PGC" psql -X -At -U postgres -c "SELECT version()" > "$OUT/postgres-version.txt" +if [ -n "$VERSION" ]; then + docker run --rm --entrypoint extenddb "$IMG" --version > "$OUT/release-version.txt" 2>&1 || true + python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); print("\n".join(("ok " if x["ok"] else "FAILED ")+x["step"] for x in r))' \ + "$WORK/seed.json" > "$OUT/seed-steps.txt" +else + echo "built from the $STATE files, catalog-only" > "$OUT/release-version.txt" +fi +note "wrote $OUT" diff --git a/crates/storage-postgres/tests/fixtures/released/scrub.py b/crates/storage-postgres/tests/fixtures/released/scrub.py new file mode 100755 index 000000000..ad7c2df07 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/scrub.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Fixed test values for the released PostgreSQL fixtures. + +scrub.py --dump < pg_dump-output > fixture.sql + Replace the encryption key, every password hash and the password inside + the data connection string with fixed test values, and drop the + `SET search_path`/`set_config` lines' effect on later statements by + leaving them out: the fixtures are loaded on one connection each, and the + tests read the objects through the public schema. + +scrub.py --bootstrap + Print the rows `init` of that tag wrote after its schema files: the + encryption key, the default account (with `default_account_id` from + v0.1.3 on), the admin user and the data connection rows, with the same + fixed values. + +The access key's stored secret stays encrypted under the release's own key, +which these values replace, so it cannot be decrypted; no test needs it. +""" +import re +import sys + +TEST_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" # bytes 0..31, base64 +# bcrypt of "fixture-password", cost 4. +TEST_HASH = "$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka" +TEST_CONNECTION = "postgresql://extenddb_app:fixture@fixture-host:5432/extenddb" + +INSERT = re.compile(r"^INSERT INTO public\.(\w+) VALUES \((.*)\);$") + + +def quote(value): + return "'" + value.replace("'", "''") + "'" + + +def scrub_line(line): + m = INSERT.match(line) + if not m: + return line + table, values = m.group(1), m.group(2) + if table == "settings": + key = values.split(",", 1)[0].strip() + if key == "'encryption_key'": + return f"INSERT INTO public.settings VALUES ('encryption_key', {quote(TEST_KEY)});" + if key == "'data_database_connection_string'": + return ( + "INSERT INTO public.settings VALUES " + f"('data_database_connection_string', {quote(TEST_CONNECTION)});" + ) + if table in ("admin_users", "iam_users"): + # Replace every bcrypt hash in the row; NULL hashes stay NULL. + return re.sub(r"'\$2[aby]\$[^']*'", quote(TEST_HASH), line) + return line + + +def dump(): + for raw in sys.stdin: + line = raw.rstrip("\n") + # psql meta-commands (`\restrict`) are not SQL. + if line.startswith("SELECT pg_catalog.set_config('search_path'") or line.startswith("\\"): + continue + print(scrub_line(line)) + + +def bootstrap(tag): + account = "111122223333" + print(f"INSERT INTO settings (key, value) VALUES ('encryption_key', {quote(TEST_KEY)});") + print(f"INSERT INTO accounts (account_id, account_name) VALUES ('{account}', 'default');") + minor_patch = tuple(int(x) for x in tag.lstrip("v").split(".")[1:]) + if minor_patch >= (1, 3): + print(f"INSERT INTO settings (key, value) VALUES ('default_account_id', '{account}');") + print( + "INSERT INTO admin_users (admin_name, password_hash) VALUES " + f"('admin', {quote(TEST_HASH)});" + ) + print( + "INSERT INTO settings (key, value) VALUES " + f"('data_database_connection_string', {quote(TEST_CONNECTION)});" + ) + print("INSERT INTO settings (key, value) VALUES ('data_database_name', 'extenddb');") + + +if __name__ == "__main__": + if sys.argv[1] == "--dump": + dump() + elif sys.argv[1] == "--bootstrap": + bootstrap(sys.argv[2]) + else: + raise SystemExit("usage: scrub.py --dump | --bootstrap ") diff --git a/crates/storage-postgres/tests/fixtures/released/seed.py b/crates/storage-postgres/tests/fixtures/released/seed.py new file mode 100755 index 000000000..e9145fca6 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/seed.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Seed a released ExtendDB server through the DynamoDB API. + +Usage: seed.py + +Creates seed_tbl (sort key, GSI gsi1, LSI lsi1, a stream, TTL on `exp`), +writes ten items, takes the backup seed_backup, and creates seed_simple with +three items. With vector=1 also vec_tbl with the vector index vidx and three +vectors, through signed requests, because boto3 does not know VectorIndexes. +Every call's outcome is recorded; a failed step is recorded and the script +goes on, so a release that lacks a feature still gets a fixture. +""" +import json +import sys +import time +import warnings + +import boto3 +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.config import Config +from botocore.credentials import Credentials + +warnings.filterwarnings("ignore") +endpoint, keyfile, vector, outfile = sys.argv[1:5] +key = json.load(open(keyfile)) +ddb = boto3.client( + "dynamodb", endpoint_url=endpoint, region_name="us-east-1", + aws_access_key_id=key["access_key_id"], aws_secret_access_key=key["secret_access_key"], + verify=False, config=Config(retries={"max_attempts": 2}, read_timeout=30, connect_timeout=5), +) +results = [] + + +def step(name, fn): + try: + results.append({"step": name, "ok": True, "result": fn()}) + except Exception as exc: + results.append({"step": name, "ok": False, "error": f"{type(exc).__name__}: {exc}"}) + + +def wait_active(table, timeout=90): + deadline = time.time() + timeout + while time.time() < deadline: + d = ddb.describe_table(TableName=table)["Table"] + if d["TableStatus"] == "ACTIVE" and all( + g.get("IndexStatus") == "ACTIVE" for g in d.get("GlobalSecondaryIndexes", []) + ): + return "ACTIVE" + time.sleep(0.5) + raise TimeoutError(f"{table} not ACTIVE") + + +def create_seed_tbl(): + ddb.create_table( + TableName="seed_tbl", + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "lsk", "AttributeType": "N"}, + ], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + }], + LocalSecondaryIndexes=[{ + "IndexName": "lsi1", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "lsk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + }], + BillingMode="PAY_PER_REQUEST", + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + ) + return wait_active("seed_tbl") + + +def put_items(): + for i in range(10): + ddb.put_item(TableName="seed_tbl", Item={ + "pk": {"S": f"seed-p{i % 3}"}, "sk": {"S": f"s{i:03d}"}, "gpk": {"S": f"g{i % 2}"}, + "lsk": {"N": str(100 + i)}, "exp": {"N": str(4102444800)}, "payload": {"S": f"item {i}"}, + }) + return 10 + + +def signed(op, body): + data = json.dumps(body).encode() + req = AWSRequest(method="POST", url=endpoint, data=data, headers={ + "X-Amz-Target": f"DynamoDB_20120810.{op}", "Content-Type": "application/x-amz-json-1.0"}) + SigV4Auth(Credentials(key["access_key_id"], key["secret_access_key"]), "dynamodb", "us-east-1").add_auth(req) + r = requests.post(endpoint, data=data, headers=dict(req.headers), verify=False, timeout=30) + if r.status_code != 200: + raise RuntimeError(f"{op} {r.status_code} {r.text[:200]}") + return r.json() if r.content else {} + + +def vector_table(): + signed("CreateTable", { + "TableName": "vec_tbl", + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{"IndexName": "vidx", "Dimensions": 4, "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, "Projection": {"ProjectionType": "ALL"}}], + }) + for _ in range(240): + t = signed("DescribeTable", {"TableName": "vec_tbl"})["Table"] + if t["TableStatus"] == "ACTIVE" and all(v.get("IndexStatus") == "ACTIVE" for v in t.get("VectorIndexes", [])): + break + time.sleep(0.5) + for i in range(3): + signed("PutItem", {"TableName": "vec_tbl", "Item": { + "pk": {"S": f"v{i}"}, "emb": {"L": [{"N": str(0.1 * (i + 1))}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}]}}}) + return "ACTIVE" + + +step("create_table seed_tbl", create_seed_tbl) +step("update_time_to_live seed_tbl", lambda: ddb.update_time_to_live( + TableName="seed_tbl", TimeToLiveSpecification={"Enabled": True, "AttributeName": "exp"})["TimeToLiveSpecification"]) +step("put_item seed_tbl x10", put_items) +time.sleep(2) # asynchronous index propagation +step("create_backup seed_backup", lambda: ddb.create_backup( + TableName="seed_tbl", BackupName="seed_backup")["BackupDetails"]["BackupStatus"]) +step("create_table seed_simple", lambda: (ddb.create_table( + TableName="seed_simple", AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "N"}], + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], BillingMode="PAY_PER_REQUEST"), + wait_active("seed_simple"))[1]) +step("put_item seed_simple x3", lambda: [ + ddb.put_item(TableName="seed_simple", Item={"id": {"N": str(i)}, "v": {"S": "x"}}) and i for i in range(3)]) +if vector == "1": + step("create_table vec_tbl with vector index vidx", vector_table) +time.sleep(1) +json.dump(results, open(outfile, "w"), indent=1, default=str) +failed = [r["step"] for r in results if not r["ok"]] +print(f"seed: {len(results) - len(failed)} ok, {len(failed)} failed {failed}") diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.0/catalog.sql b/crates/storage-postgres/tests/fixtures/released/v0.1.0/catalog.sql new file mode 100644 index 000000000..0618541fe --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.0/catalog.sql @@ -0,0 +1,975 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('111122223333', 'default', '2026-10-29 07:20:22.642427+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:22.643792+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:22.565067+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('gsi_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258421032670, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.0/data.sql b/crates/storage-postgres/tests/fixtures/released/v0.1.0/data.sql new file mode 100644 index 000000000..bd0cfdea2 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.0/data.sql @@ -0,0 +1,156 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258422723778, true); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.0/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/v0.1.0/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.0/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.0/release-version.txt b/crates/storage-postgres/tests/fixtures/released/v0.1.0/release-version.txt new file mode 100644 index 000000000..d27ca2a7e --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.0/release-version.txt @@ -0,0 +1 @@ +built from the v0.1.0 files, catalog-only diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.2/catalog.sql b/crates/storage-postgres/tests/fixtures/released/v0.1.2/catalog.sql new file mode 100644 index 000000000..f4ca01293 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.2/catalog.sql @@ -0,0 +1,978 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: access_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.access_keys ( + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + user_name text NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: accounts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.accounts ( + account_id text NOT NULL, + account_name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: admin_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.admin_users ( + admin_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: backup_items; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backup_items ( + backup_arn text NOT NULL, + pk text NOT NULL, + sk text, + item_data jsonb NOT NULL +); + + +-- +-- Name: backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.backups ( + backup_arn text NOT NULL, + backup_name text NOT NULL, + table_id text NOT NULL, + table_name text NOT NULL, + account_id text NOT NULL, + backup_status text DEFAULT 'AVAILABLE'::text NOT NULL, + backup_type text DEFAULT 'USER'::text NOT NULL, + backup_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: continuous_backups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.continuous_backups ( + account_id text NOT NULL, + table_name text NOT NULL, + pitr_enabled boolean DEFAULT false NOT NULL, + earliest_restorable timestamp with time zone, + latest_restorable timestamp with time zone +); + + +-- +-- Name: iam_group_members; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_group_members ( + account_id text NOT NULL, + group_name text NOT NULL, + user_name text NOT NULL +); + + +-- +-- Name: iam_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_groups ( + account_id text NOT NULL, + group_name text NOT NULL, + group_arn text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_permissions_boundaries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_permissions_boundaries ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_document jsonb NOT NULL, + CONSTRAINT iam_permissions_boundaries_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +); + + +-- +-- Name: iam_policies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_policies ( + account_id text NOT NULL, + principal_type text NOT NULL, + principal_name text NOT NULL, + policy_name text NOT NULL, + policy_document jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT iam_policies_principal_type_check CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +); + + +-- +-- Name: iam_role_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_role_tags ( + account_id text NOT NULL, + role_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_roles ( + account_id text NOT NULL, + role_name text NOT NULL, + role_arn text NOT NULL, + trust_policy jsonb NOT NULL, + permissions_boundary_arn text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_sessions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_sessions ( + session_token text NOT NULL, + access_key_id text NOT NULL, + secret_key_encrypted bytea NOT NULL, + account_id text NOT NULL, + role_name text NOT NULL, + session_name text NOT NULL, + session_tags jsonb, + session_policy jsonb, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: iam_user_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_user_tags ( + account_id text NOT NULL, + user_name text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Name: iam_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.iam_users ( + account_id text NOT NULL, + user_name text NOT NULL, + user_arn text NOT NULL, + password_hash text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: indexes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.indexes ( + table_id text NOT NULL, + index_id text DEFAULT (gen_random_uuid())::text NOT NULL, + index_name text NOT NULL, + index_type text NOT NULL, + key_schema jsonb NOT NULL, + projection jsonb NOT NULL, + index_status text DEFAULT 'ACTIVE'::text NOT NULL, + provisioned_throughput jsonb, + propagation_delay_ms integer, + CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +); + + +-- +-- Name: login_attempts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.login_attempts ( + principal text NOT NULL, + attempted_at timestamp with time zone DEFAULT now() NOT NULL, + success boolean NOT NULL, + source_ip text +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + bucket timestamp with time zone NOT NULL, + metric text NOT NULL, + table_name text DEFAULT ''::text NOT NULL, + index_name text DEFAULT ''::text NOT NULL, + operation text DEFAULT ''::text NOT NULL, + sum double precision DEFAULT 0 NOT NULL, + count bigint DEFAULT 0 NOT NULL, + min double precision DEFAULT 'Infinity'::double precision NOT NULL, + max double precision DEFAULT '-Infinity'::double precision NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + key text NOT NULL, + value text NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tables ( + account_id text NOT NULL, + table_name text NOT NULL, + key_schema jsonb NOT NULL, + attribute_definitions jsonb NOT NULL, + billing_mode text DEFAULT 'PAY_PER_REQUEST'::text NOT NULL, + provisioned_throughput jsonb, + stream_specification jsonb, + table_status text DEFAULT 'CREATING'::text NOT NULL, + creation_date_time timestamp with time zone DEFAULT now() NOT NULL, + table_size_bytes bigint DEFAULT 0 NOT NULL, + item_count bigint DEFAULT 0 NOT NULL, + table_arn text NOT NULL, + table_id text NOT NULL, + ttl_attribute text, + deletion_protection_enabled boolean DEFAULT false NOT NULL, + status_transition_at timestamp with time zone, + stream_label text, + ttl_index_ready boolean DEFAULT false NOT NULL, + table_class text, + sse_specification jsonb, + on_demand_throughput jsonb +); + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tags ( + resource_arn text NOT NULL, + tag_key text NOT NULL, + tag_value text NOT NULL +); + + +-- +-- Data for Name: access_keys; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: accounts; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.accounts VALUES ('111122223333', 'default', '2026-10-29 07:20:20.982682+00'); + + +-- +-- Data for Name: admin_users; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.admin_users VALUES ('admin', '$2b$04$KQg8SAyQ3g2v0nmyz2wPXOWb2pGZtfJ0yLe6E8V1OAQmvQjQlZ9ka', '2026-10-29 07:20:20.988115+00'); + + +-- +-- Data for Name: backup_items; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: continuous_backups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_group_members; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_groups; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_permissions_boundaries; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_policies; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_role_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_roles; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_sessions; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_user_tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: iam_users; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: indexes; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: login_attempts; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: metrics; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_schema.sql', '2026-10-29 07:20:20.88955+00'); + + +-- +-- Data for Name: settings; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.settings VALUES ('catalog_version', '0.0.2'); +INSERT INTO public.settings VALUES ('control_plane_delay_seconds', '0.25'); +INSERT INTO public.settings VALUES ('gsi_propagation_delay_ms', '10'); +INSERT INTO public.settings VALUES ('encryption_key', 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8='); +INSERT INTO public.settings VALUES ('data_database_connection_string', 'postgresql://extenddb_app:fixture@fixture-host:5432/extenddb'); +INSERT INTO public.settings VALUES ('data_database_name', 'extenddb'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tables; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: tags; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258420340185, true); + + +-- +-- Name: access_keys access_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_pkey PRIMARY KEY (access_key_id); + + +-- +-- Name: accounts accounts_account_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_account_name_key UNIQUE (account_name); + + +-- +-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (account_id); + + +-- +-- Name: admin_users admin_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.admin_users + ADD CONSTRAINT admin_users_pkey PRIMARY KEY (admin_name); + + +-- +-- Name: backups backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backups + ADD CONSTRAINT backups_pkey PRIMARY KEY (backup_arn); + + +-- +-- Name: continuous_backups continuous_backups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.continuous_backups + ADD CONSTRAINT continuous_backups_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: iam_group_members iam_group_members_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_pkey PRIMARY KEY (account_id, group_name, user_name); + + +-- +-- Name: iam_groups iam_groups_group_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_group_arn_key UNIQUE (group_arn); + + +-- +-- Name: iam_groups iam_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_pkey PRIMARY KEY (account_id, group_name); + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_pkey PRIMARY KEY (account_id, principal_type, principal_name); + + +-- +-- Name: iam_policies iam_policies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_pkey PRIMARY KEY (account_id, principal_type, principal_name, policy_name); + + +-- +-- Name: iam_role_tags iam_role_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_pkey PRIMARY KEY (account_id, role_name, tag_key); + + +-- +-- Name: iam_roles iam_roles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_pkey PRIMARY KEY (account_id, role_name); + + +-- +-- Name: iam_roles iam_roles_role_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_role_arn_key UNIQUE (role_arn); + + +-- +-- Name: iam_sessions iam_sessions_access_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_access_key_id_key UNIQUE (access_key_id); + + +-- +-- Name: iam_sessions iam_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_pkey PRIMARY KEY (session_token); + + +-- +-- Name: iam_user_tags iam_user_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_pkey PRIMARY KEY (account_id, user_name, tag_key); + + +-- +-- Name: iam_users iam_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_pkey PRIMARY KEY (account_id, user_name); + + +-- +-- Name: iam_users iam_users_user_arn_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_user_arn_key UNIQUE (user_arn); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: indexes indexes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_pkey PRIMARY KEY (table_id, index_name); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (bucket, metric, table_name, index_name, operation); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (key); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: tables tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_pkey PRIMARY KEY (account_id, table_name); + + +-- +-- Name: tables tables_table_id_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_table_id_unique UNIQUE (table_id); + + +-- +-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (resource_arn, tag_key); + + +-- +-- Name: idx_backup_items_arn; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn); + + +-- +-- Name: idx_backups_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_login_attempts_principal_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC); + + +-- +-- Name: idx_login_attempts_source_ip_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL); + + +-- +-- Name: idx_metrics_bucket; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: idx_tables_pending_transition; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL); + + +-- +-- Name: access_keys access_keys_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.access_keys + ADD CONSTRAINT access_keys_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: backup_items backup_items_backup_arn_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.backup_items + ADD CONSTRAINT backup_items_backup_arn_fkey FOREIGN KEY (backup_arn) REFERENCES public.backups(backup_arn) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_group_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_group_name_fkey FOREIGN KEY (account_id, group_name) REFERENCES public.iam_groups(account_id, group_name) ON DELETE CASCADE; + + +-- +-- Name: iam_group_members iam_group_members_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_group_members + ADD CONSTRAINT iam_group_members_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_groups iam_groups_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_groups + ADD CONSTRAINT iam_groups_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_permissions_boundaries iam_permissions_boundaries_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_permissions_boundaries + ADD CONSTRAINT iam_permissions_boundaries_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_policies iam_policies_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_policies + ADD CONSTRAINT iam_policies_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_role_tags iam_role_tags_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_role_tags + ADD CONSTRAINT iam_role_tags_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_roles iam_roles_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_roles + ADD CONSTRAINT iam_roles_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: iam_sessions iam_sessions_account_id_role_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_sessions + ADD CONSTRAINT iam_sessions_account_id_role_name_fkey FOREIGN KEY (account_id, role_name) REFERENCES public.iam_roles(account_id, role_name) ON DELETE CASCADE; + + +-- +-- Name: iam_user_tags iam_user_tags_account_id_user_name_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_user_tags + ADD CONSTRAINT iam_user_tags_account_id_user_name_fkey FOREIGN KEY (account_id, user_name) REFERENCES public.iam_users(account_id, user_name) ON DELETE CASCADE; + + +-- +-- Name: iam_users iam_users_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.iam_users + ADD CONSTRAINT iam_users_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- Name: indexes indexes_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.indexes + ADD CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- Name: stream_shards stream_shards_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_table_id_fkey FOREIGN KEY (table_id) REFERENCES public.tables(table_id) ON DELETE CASCADE; + + +-- +-- Name: tables tables_account_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tables + ADD CONSTRAINT tables_account_id_fkey FOREIGN KEY (account_id) REFERENCES public.accounts(account_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.2/data.sql b/crates/storage-postgres/tests/fixtures/released/v0.1.2/data.sql new file mode 100644 index 000000000..6c17b4ee0 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.2/data.sql @@ -0,0 +1,253 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.10 (Debian 16.10-1.pgdg12+1) +-- Dumped by pg_dump version 16.10 (Debian 16.10-1.pgdg12+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: gsi_pending; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.gsi_pending ( + id bigint NOT NULL, + table_id text NOT NULL, + worker_partition integer NOT NULL, + old_item jsonb, + new_item jsonb, + index_context jsonb NOT NULL, + ready_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.gsi_pending_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.gsi_pending_id_seq OWNED BY public.gsi_pending.id; + + +-- +-- Name: idempotency_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.idempotency_tokens ( + account_id text NOT NULL, + token text NOT NULL, + fingerprint text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: schema_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_history ( + filename text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_records; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_records ( + shard_id text NOT NULL, + sequence_number text NOT NULL, + table_id text NOT NULL, + event_name text NOT NULL, + record_data jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: stream_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.stream_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: stream_shards; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stream_shards ( + shard_id text NOT NULL, + table_id text NOT NULL, + parent_shard_id text, + starting_sequence_number text NOT NULL, + ending_sequence_number text, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: gsi_pending id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending ALTER COLUMN id SET DEFAULT nextval('public.gsi_pending_id_seq'::regclass); + + +-- +-- Data for Name: gsi_pending; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: idempotency_tokens; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: schema_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +INSERT INTO public.schema_history VALUES ('001_data_schema.sql', '2026-10-29 07:20:21.561768+00'); +INSERT INTO public.schema_history VALUES ('002_gsi_pending.sql', '2026-10-29 07:20:21.911599+00'); +INSERT INTO public.schema_history VALUES ('003_idempotency_account_scope.sql', '2026-10-29 07:20:22.161794+00'); + + +-- +-- Data for Name: stream_records; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Data for Name: stream_shards; Type: TABLE DATA; Schema: public; Owner: - +-- + + + +-- +-- Name: gsi_pending_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.gsi_pending_id_seq', 1, false); + + +-- +-- Name: stream_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('public.stream_seq', 1793258421229070, true); + + +-- +-- Name: gsi_pending gsi_pending_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.gsi_pending + ADD CONSTRAINT gsi_pending_pkey PRIMARY KEY (id); + + +-- +-- Name: idempotency_tokens idempotency_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.idempotency_tokens + ADD CONSTRAINT idempotency_tokens_pkey PRIMARY KEY (account_id, token); + + +-- +-- Name: schema_history schema_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_history + ADD CONSTRAINT schema_history_pkey PRIMARY KEY (filename); + + +-- +-- Name: stream_records stream_records_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_pkey PRIMARY KEY (shard_id, sequence_number); + + +-- +-- Name: stream_shards stream_shards_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_shards + ADD CONSTRAINT stream_shards_pkey PRIMARY KEY (shard_id); + + +-- +-- Name: idx_gsi_pending_claim; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id); + + +-- +-- Name: idx_idempotency_tokens_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at); + + +-- +-- Name: idx_stream_records_created; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at); + + +-- +-- Name: idx_stream_shards_table; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id); + + +-- +-- Name: stream_records stream_records_shard_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.stream_records + ADD CONSTRAINT stream_records_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.stream_shards(shard_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.2/postgres-version.txt b/crates/storage-postgres/tests/fixtures/released/v0.1.2/postgres-version.txt new file mode 100644 index 000000000..9a7b20962 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.2/postgres-version.txt @@ -0,0 +1 @@ +PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit diff --git a/crates/storage-postgres/tests/fixtures/released/v0.1.2/release-version.txt b/crates/storage-postgres/tests/fixtures/released/v0.1.2/release-version.txt new file mode 100644 index 000000000..51d137d09 --- /dev/null +++ b/crates/storage-postgres/tests/fixtures/released/v0.1.2/release-version.txt @@ -0,0 +1 @@ +built from the v0.1.2 files, catalog-only diff --git a/crates/storage-postgres/tests/gsi_online_build.rs b/crates/storage-postgres/tests/gsi_online_build.rs index 7fdb52b81..52dd48e67 100644 --- a/crates/storage-postgres/tests/gsi_online_build.rs +++ b/crates/storage-postgres/tests/gsi_online_build.rs @@ -64,9 +64,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } fn shipped_migrations() -> Vec { @@ -118,10 +131,13 @@ async fn scratch() -> Scratch { .await .expect("apply a shipped migration"); } - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/gsi_stale_writer.rs b/crates/storage-postgres/tests/gsi_stale_writer.rs index c79fe8cab..2791fbdfb 100644 --- a/crates/storage-postgres/tests/gsi_stale_writer.rs +++ b/crates/storage-postgres/tests/gsi_stale_writer.rs @@ -65,9 +65,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } fn shipped_migrations() -> Vec { @@ -119,10 +132,13 @@ async fn scratch() -> Scratch { .await .expect("apply a shipped migration"); } - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/key_collation.rs b/crates/storage-postgres/tests/key_collation.rs index 4988d17eb..4f363eaa7 100644 --- a/crates/storage-postgres/tests/key_collation.rs +++ b/crates/storage-postgres/tests/key_collation.rs @@ -86,6 +86,13 @@ fn shipped_migrations() -> Vec { } fn skip(test: &str) { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "{test}: EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED \ + says this job must run the PostgreSQL tests" + ); eprintln!( "SKIP {test}: EXTENDDB_TEST_PG_CONNECTION_STRING is not set, so there is no PostgreSQL \ to build a scratch catalog in." @@ -119,14 +126,20 @@ async fn scratch() -> Scratch { .await .expect("apply a shipped migration"); } - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") - .execute(&db) - .await - .expect("pin the propagation delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the propagation delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/migrate_data_db_name.rs b/crates/storage-postgres/tests/migrate_data_db_name.rs index 5e6a67123..826dbcf22 100644 --- a/crates/storage-postgres/tests/migrate_data_db_name.rs +++ b/crates/storage-postgres/tests/migrate_data_db_name.rs @@ -1,25 +1,24 @@ +// Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 //! `extenddb migrate` on a deployment whose data database is not named //! `` minus `_catalog`. //! //! `init` records the data database name in the catalog's settings -//! (`data_database_name`). The bootstrapper built from a config file and the -//! `migrate` argument vector (no `--data-db`) must use that name, so the data -//! migrations are checked and applied on the real data database; the -//! derivation is only the fallback for a catalog with no recorded name, and -//! an explicit `--data-db` wins over both. +//! (`data_database_name`). The target the commands resolve from the catalog's +//! connection string must use that name, so the status and the runner read +//! and write the real data database; the derivation is only the fallback for +//! a catalog with no recorded name, and an explicit `--data-db` wins over +//! both. //! //! Requires `EXTENDDB_TEST_PG_CONNECTION_STRING` (a base URL //! `postgres://user:pass@host:port`, no database); without it every test -//! reports a skip and passes. Each test creates its own databases and config -//! file and removes them when its deployment is dropped, on a panic too. The -//! config file carries the application password, as the one `init` writes -//! does: the bootstrapper reads the catalog connection string from a file -//! only and connects to the catalog with that password, so the file is -//! written owner-readable and removed with the deployment. +//! reports a skip and passes. Each test creates its own databases and removes +//! them when its deployment is dropped, on a panic too. use extenddb_storage::bootstrapper::{BootstrapConfig, Bootstrapper}; +use extenddb_storage::migrations::{CatalogMigrations, CatalogStatus, Target}; use extenddb_storage_postgres::PostgresBootstrapper; +use extenddb_storage_postgres::migrations::{PostgresCatalogMigrations, resolve}; use sqlx::PgPool; use sqlx::postgres::PgPoolOptions; @@ -28,6 +27,19 @@ fn base_conn() -> Option { (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) } +/// Report a skip, or fail when `EXTENDDB_TEST_PG_REQUIRED` says this job must +/// run the PostgreSQL tests. +fn skip_unless_required() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says this \ + job must run the PostgreSQL tests" + ); + eprintln!("skipped: EXTENDDB_TEST_PG_CONNECTION_STRING not set"); +} + struct Parts { host: String, port: u16, @@ -51,22 +63,6 @@ fn parts(base: &str) -> Parts { } } -/// Write `contents` to a new file only its owner can read. -fn write_private(path: &std::path::Path, contents: &str) { - use std::io::Write; - let mut options = std::fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - options - .open(path) - .and_then(|mut f| f.write_all(contents.as_bytes())) - .expect("config file"); -} - /// A deployment initialised the way `init --catalog-db --data-db ` /// does it, with a data database name unrelated to the catalog's. Dropping /// it drops its three databases and its config file. @@ -76,12 +72,10 @@ struct Deployment { catalog_db: String, data_db: String, derived_db: String, - config_path: std::path::PathBuf, } impl Drop for Deployment { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.config_path); // Runs on a panic too, so it cannot await on the test's runtime: a // thread of its own with its own runtime and connection. let base = self.base.clone(); @@ -145,99 +139,65 @@ impl Deployment { .expect("connect"); init.create_catalog_db().await.expect("catalog db"); init.create_data_db().await.expect("data db"); - init.run_catalog_migrations() - .await - .expect("catalog migrations"); + let catalog: sqlx::postgres::PgConnectOptions = + format!("{base}/{catalog_db}").parse().expect("catalog URL"); + extenddb_storage_postgres::migrations::migrate( + &extenddb_storage_postgres::migrations::PostgresTarget::new(catalog, data_db.clone()), + "test", + std::time::Duration::ZERO, + ) + .await + .expect("catalog migrations"); init.record_data_connection().await.expect("record"); - init.run_data_migrations().await.expect("data migrations"); - // The config file `migrate` reads: the catalog connection string only, - // as `init` writes it. Owner-readable, removed when `self` drops. - let target = std::env::var_os("CARGO_TARGET_DIR") - .map(std::path::PathBuf::from) - .unwrap_or_else(|| { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target") - }); - let config_path = target.join(format!("{stem}.toml")); - write_private( - &config_path, - &format!( - "[storage]\nbackend = \"postgres\"\n\n[storage.postgres]\nconnection_string = \ - \"postgres://{}:{}@{}:{}/{catalog_db}\"\n", - p.user, p.pass, p.host, p.port - ), - ); Self { base, admin, catalog_db, data_db, derived_db: stem, - config_path, } } - /// The argument vector `extenddb migrate` hands the bootstrapper. - fn migrate_args(&self, extra: &[&str]) -> Vec { - let p = parts(&self.base); - let mut args = vec![ - "extenddb".to_owned(), - "migrate".to_owned(), - "--config".to_owned(), - self.config_path.display().to_string(), - "--pg-user".to_owned(), - p.user, - "--pg-pass".to_owned(), - p.pass, - ]; - args.extend(extra.iter().map(|s| (*s).to_owned())); - args + /// The catalog's connection string, as the config file `init` writes + /// gives it. + fn catalog_url(&self) -> String { + format!("{}/{}", self.base, self.catalog_db) } - async fn bootstrapper(&self, extra: &[&str]) -> PostgresBootstrapper { - PostgresBootstrapper::from_config( - self.config_path.to_str().expect("UTF-8 path"), - &self.migrate_args(extra), - ) - .await - .expect("from_config") + /// The target `extenddb migrate` builds, with `data_db` as its + /// `--data-db`. + fn target(&self, data_db: Option<&str>) -> Target { + Target { + data_store: data_db.map(str::to_owned), + ..Target::at(self.catalog_url()) + } } } -/// Without `--data-db`, the bootstrapper `migrate` builds reaches the data -/// database `init` recorded: `pending_data_migrations` reads that database's -/// ledger (and finds it current) instead of failing on the derived name, and -/// `migrate`'s one write to the data database, `run_data_migrations`, lands -/// there. The derived database is never created. +/// Without `--data-db`, the commands' target is the data database `init` +/// recorded: the status reads that database's ledger (and finds it current) +/// instead of failing on the derived name, which is never created. An +/// explicit `--data-db` wins over the recorded name. #[tokio::test] async fn migrate_uses_the_data_database_name_init_recorded() { let Some(_) = base_conn() else { - eprintln!("skipped: EXTENDDB_TEST_PG_CONNECTION_STRING not set"); + skip_unless_required(); return; }; let d = Deployment::new().await; - let stored: String = - sqlx::query_scalar("SELECT value FROM settings WHERE key = 'data_database_name'") - .fetch_one( - &PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{}/{}", d.base, d.catalog_db)) - .await - .expect("catalog"), - ) - .await - .expect("recorded name"); - assert_eq!(stored, d.data_db, "init records the data database name"); - - let migrate = d.bootstrapper(&[]).await; - let pending = migrate - .pending_data_migrations() + let resolved = resolve(&d.target(None)) .await - .expect("pending data migrations on the recorded data database"); - assert!(pending.is_empty(), "{pending:?}"); - migrate - .run_data_migrations() + .expect("resolve") + .expect("the catalog database exists"); + assert_eq!(resolved.data_database, d.data_db); + let status = PostgresCatalogMigrations + .status(&d.target(None)) .await - .expect("data migrations"); + .expect("status on the recorded data database"); + assert!( + matches!(status, CatalogStatus::Current { .. }), + "{status:?}" + ); let derived_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)") .bind(&d.derived_db) @@ -250,15 +210,32 @@ async fn migrate_uses_the_data_database_name_init_recorded() { d.derived_db ); - // An explicit --data-db wins over the recorded name. - let explicit = d.bootstrapper(&["--data-db", &d.derived_db]).await; - let err = explicit - .pending_data_migrations() + let explicit = d.target(Some(&d.derived_db)); + let err = PostgresCatalogMigrations + .status(&explicit) .await .expect_err("the derived database does not exist"); assert!( - format!("{err:?}").contains(&d.derived_db), - "the error names the explicit database: {err:?}" + err.to_string().contains(&d.derived_db), + "the error names the explicit database: {err}" + ); + // Once it exists, the explicit name wins over the database the catalog's + // connection string row names, on that row's server: the status reads + // its empty ledger instead of refusing the two names. + sqlx::query(&format!("CREATE DATABASE \"{}\"", d.derived_db)) + .execute(&d.admin) + .await + .expect("create the derived database"); + let status = PostgresCatalogMigrations.status(&explicit).await; + let _ = sqlx::query(&format!( + "DROP DATABASE IF EXISTS \"{}\" WITH (FORCE)", + d.derived_db + )) + .execute(&d.admin) + .await; + assert!( + matches!(status, Ok(CatalogStatus::Behind { .. })), + "{status:?}" ); drop(d); } @@ -267,27 +244,63 @@ async fn migrate_uses_the_data_database_name_init_recorded() { #[tokio::test] async fn a_catalog_without_a_recorded_name_falls_back_to_the_derivation() { let Some(_) = base_conn() else { - eprintln!("skipped: EXTENDDB_TEST_PG_CONNECTION_STRING not set"); + skip_unless_required(); return; }; let d = Deployment::new().await; let catalog = PgPoolOptions::new() .max_connections(1) - .connect(&format!("{}/{}", d.base, d.catalog_db)) + .connect(&d.catalog_url()) .await .expect("catalog"); sqlx::query("DELETE FROM settings WHERE key = 'data_database_name'") .execute(&catalog) .await .expect("forget the name"); - let migrate = d.bootstrapper(&[]).await; - let err = migrate - .pending_data_migrations() + catalog.close().await; + let resolved = resolve(&d.target(None)) + .await + .expect("resolve") + .expect("the catalog database exists"); + assert_eq!(resolved.data_database, d.derived_db); + let err = PostgresCatalogMigrations + .status(&d.target(None)) .await .expect_err("the derived database does not exist"); assert!( - format!("{err:?}").contains(&d.derived_db), - "the fallback is the derived name: {err:?}" + err.to_string().contains(&d.derived_db), + "the fallback is the derived name: {err}" + ); + drop(d); +} + +/// The commands give the backup clause the data database the catalog +/// records, through `data_store_name`, when no `--data-db` is given; a given +/// name wins. +#[tokio::test] +async fn the_backup_advice_names_the_recorded_data_database() { + let Some(_) = base_conn() else { + skip_unless_required(); + return; + }; + let d = Deployment::new().await; + let name = PostgresCatalogMigrations + .data_store_name(&d.target(None)) + .await; + assert_eq!(name.as_deref(), Some(d.data_db.as_str())); + let advice = PostgresCatalogMigrations.backup_advice(&Target { + data_store: name, + ..d.target(None) + }); + assert!( + advice.ends_with(&format!("and the data database {}", d.data_db)), + "{advice}" + ); + assert_eq!( + PostgresCatalogMigrations + .data_store_name(&d.target(Some(&d.derived_db))) + .await, + Some(d.derived_db.clone()) ); drop(d); } diff --git a/crates/storage-postgres/tests/migrations_crash.rs b/crates/storage-postgres/tests/migrations_crash.rs new file mode 100644 index 000000000..cd0aadd8f --- /dev/null +++ b/crates/storage-postgres/tests/migrations_crash.rs @@ -0,0 +1,465 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! A migrator that dies inside each kind of body, a concurrent index build +//! cancelled part way, two migrators, the key transactions against a role's +//! timeouts, and the serving registration against `migrate`. A migrator is killed by ending +//! its sessions with `pg_terminate_backend`, which is what the database sees +//! when the process dies. See `migrations_support` for the database these +//! tests need. + +mod migrations_support; + +use std::time::Duration; + +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId, RunReport, Runner}; +use extenddb_storage_postgres::migrations::{ + MigrationLock, PostgresMigrations, PostgresTarget, ServingRegistration, catalog_status, migrate, +}; +use migrations_support::{LATEST, Scratch, base_conn, count, ledger, pool, skip, step_count}; +use sqlx::postgres::PgConnectOptions; +use sqlx::{Connection, PgPool}; + +const RELEASE: &str = "1.0.0-test"; + +/// Run every pending migration in a task, with no lock. +fn spawn_run(target: PostgresTarget) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + let m = PostgresMigrations::open_unlocked(&target).await?; + let result = Runner::new(&m, RELEASE).migrate().await; + m.close().await; + result + }) +} + +/// The pid of the runner's session in `database` once it waits on a lock +/// while running a statement that contains `statement`. +async fn waiting_runner(admin: &PgPool, database: &str, statement: &str) -> i32 { + for _ in 0..200 { + let pid: Option = sqlx::query_scalar( + "SELECT pid FROM pg_stat_activity WHERE datname = $1 \ + AND starts_with(application_name, 'extenddb migrate run ') AND wait_event_type = 'Lock' \ + AND query LIKE '%' || $2 || '%'", + ) + .bind(database) + .bind(statement) + .fetch_optional(admin) + .await + .unwrap(); + if let Some(pid) = pid { + return pid; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + panic!("the runner never waited on {statement:?} in {database}"); +} + +async fn index_valid(data: &PgPool, index: &str) -> Option { + sqlx::query_scalar("SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass($1)") + .bind(format!("public.\"{index}\"")) + .fetch_optional(data) + .await + .unwrap() +} + +/// The GSI index table of seed_tbl and its base key index. +async fn gsi_index(catalog: &PgPool) -> (String, String) { + let id: String = sqlx::query_scalar("SELECT index_id FROM indexes WHERE index_name = 'gsi1'") + .fetch_one(catalog) + .await + .unwrap(); + (format!("_ddb_{id}"), format!("_ddb_{id}_base_key_idx")) +} + +#[tokio::test] +async fn a_cancelled_concurrent_build_is_not_recorded_and_the_rerun_rebuilds_it() { + let Some(base) = base_conn() else { + return skip("cancelled build"); + }; + let s = Scratch::new(&base, "pgm_cic").await; + // Index tables created by 0.1.5 have no base key index. + s.load("0.1.5").await; + s.migrate_through(MigrationId::SortKeyDefinitions).await; + let (catalog, data, admin) = ( + s.catalog().await, + s.data().await, + pool(&format!("{base}/postgres"), 1).await, + ); + let (table, index) = gsi_index(&catalog).await; + assert_eq!(index_valid(&data, &index).await, None); + // An open transaction that wrote the index table: the concurrent build + // waits for it to end. + let mut blocker = data.begin().await.unwrap(); + sqlx::query(&format!("LOCK TABLE \"{table}\" IN ROW EXCLUSIVE MODE")) + .execute(&mut *blocker) + .await + .unwrap(); + let run = spawn_run(s.target()); + let pid = waiting_runner(&admin, &s.data_name, "CREATE INDEX CONCURRENTLY").await; + sqlx::query("SELECT pg_cancel_backend($1)") + .bind(pid) + .execute(&admin) + .await + .unwrap(); + let err = run + .await + .unwrap() + .expect_err("the cancelled build fails the run"); + assert!(err.to_string().contains("canceling statement"), "{err}"); + blocker.rollback().await.unwrap(); + // The build left the index invalid, and nothing recorded migration 4 in + // the data database, so the catalog is behind. + assert_eq!(index_valid(&data, &index).await, Some(false)); + assert_eq!( + ledger(&data).await.iter().map(|r| r.0).collect::>(), + [1] + ); + assert!(matches!( + s.status().await, + CatalogStatus::Behind { + version: Some(3), + .. + } + )); + let report = s.migrate_through(MigrationId::LATEST).await; + assert_eq!(report.to, Some(LATEST)); + assert_eq!(index_valid(&data, &index).await, Some(true)); + let notes: String = sqlx::query_scalar("SELECT notes FROM extenddb_migrations WHERE id = 4") + .fetch_one(&data) + .await + .unwrap(); + assert!( + notes.contains("gsi1: rebuilt") && notes.contains("lsi1: built"), + "{notes}" + ); + for p in [catalog, data, admin] { + p.close().await; + } +} + +/// The migrator's session ends while it waits inside a body; the database +/// rolls the body's transaction back, and a rerun completes. +async fn killed_inside( + state: &str, + through: Option, + database_is_data: bool, + lock_sql: &str, + waits_on: &str, +) { + let Some(base) = base_conn() else { + return skip("killed migrator"); + }; + let s = Scratch::new(&base, "pgm_kill").await; + s.load(state).await; + if let Some(id) = through { + s.migrate_through(id).await; + } + let (catalog, data, admin) = ( + s.catalog().await, + s.data().await, + pool(&format!("{base}/postgres"), 1).await, + ); + let (table, _) = gsi_index(&catalog).await; + let target_pool = if database_is_data { &data } else { &catalog }; + let database = if database_is_data { + s.data_name.clone() + } else { + s.catalog_name.clone() + }; + let rows_before = ledger(target_pool).await; + let mut blocker = target_pool.begin().await.unwrap(); + sqlx::query(&lock_sql.replace("{table}", &table)) + .execute(&mut *blocker) + .await + .unwrap(); + let run = spawn_run(s.target()); + let pid = waiting_runner(&admin, &database, waits_on).await; + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(pid) + .execute(&admin) + .await + .unwrap(); + run.await.unwrap().expect_err("the killed migrator fails"); + blocker.rollback().await.unwrap(); + // The killed body's store has no new row. (Migration 4's catalog row is + // written first, by design, before its data body runs.) + assert_eq!( + ledger(target_pool).await, + rows_before, + "the killed body committed rows" + ); + let report = s.migrate_through(MigrationId::LATEST).await; + assert_eq!(report.to, Some(LATEST)); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + for p in [catalog, data, admin] { + p.close().await; + } +} + +#[tokio::test] +async fn a_migrator_killed_inside_a_file_body_leaves_nothing_and_the_rerun_completes() { + // pg_stat_activity shows the first 1024 bytes of the file's text. + killed_inside( + "0.1.14", + None, + false, + "LOCK TABLE tables IN ACCESS EXCLUSIVE MODE", + "Migration 1, baseline", + ) + .await; +} + +#[tokio::test] +async fn a_migrator_killed_inside_a_transactional_code_body_leaves_nothing() { + killed_inside( + "0.1.14", + Some(MigrationId::ResourceTagConditionKeys), + false, + "LOCK TABLE tables IN ACCESS EXCLUSIVE MODE", + "FROM tables", + ) + .await; +} + +#[tokio::test] +async fn a_migrator_killed_inside_an_index_build_is_rebuilt_by_the_rerun() { + killed_inside( + "0.1.5", + Some(MigrationId::SortKeyDefinitions), + true, + "LOCK TABLE \"{table}\" IN ROW EXCLUSIVE MODE", + "CREATE INDEX CONCURRENTLY", + ) + .await; +} + +#[tokio::test] +async fn two_migrators_apply_once() { + let Some(base) = base_conn() else { + return skip("two migrators"); + }; + let s = Scratch::new(&base, "pgm_two").await; + s.load("0.1.14").await; + let target = s.target(); + let (a, b) = tokio::join!( + migrate(&target, RELEASE, Duration::from_secs(60)), + migrate(&target, RELEASE, Duration::from_secs(60)) + ); + let (a, b) = (a.expect("first"), b.expect("second")); + // One applies every step; the other finds them recorded. + let applied = [a.applied.len(), b.applied.len()]; + assert!( + applied.contains(&0) && applied.contains(&step_count()), + "{applied:?}" + ); + let catalog = s.catalog().await; + assert_eq!(ledger(&catalog).await.len(), MigrationId::ALL.len()); + catalog.close().await; +} + +/// A role whose `idle_in_transaction_session_timeout` (5 s) is shorter than +/// the renewal interval (15 s), and on PostgreSQL 17 and later whose +/// `transaction_timeout` (20 s) is shorter than the hold: the serving +/// registration and the migration lock, renewed at the production interval, +/// both hold their keys over two renewals, while a plain transaction of the +/// same role left idle is ended by the setting. +#[tokio::test] +async fn the_key_transactions_outlast_the_roles_timeouts_at_the_production_interval() { + let Some(base) = base_conn() else { + return skip("timeouts at 15 s"); + }; + let s = Scratch::with_names(&base, "pgm_keep", "_c", "_d", &["_c2"]).await; + let lock_database = format!("{}2", s.catalog_name); + let role = format!("{}_short", s.catalog_name); + let admin = pool(&format!("{base}/postgres"), 1).await; + sqlx::query(&format!( + "CREATE ROLE \"{role}\" LOGIN PASSWORD 'short-timeouts'" + )) + .execute(&admin) + .await + .unwrap(); + sqlx::query(&format!( + "ALTER ROLE \"{role}\" SET idle_in_transaction_session_timeout = '5s'" + )) + .execute(&admin) + .await + .unwrap(); + let version: i32 = sqlx::query_scalar("SELECT current_setting('server_version_num')::int") + .fetch_one(&admin) + .await + .unwrap(); + if version >= 170_000 { + sqlx::query(&format!( + "ALTER ROLE \"{role}\" SET transaction_timeout = '20s'" + )) + .execute(&admin) + .await + .unwrap(); + } + let as_role = |database: &str| -> PgConnectOptions { + format!("{base}/{database}") + .parse::() + .expect("a postgres URL") + .username(&role) + .password("short-timeouts") + }; + let serving = ServingRegistration::register(&as_role(&s.catalog_name)) + .await + .expect("register as the role"); + let lock = MigrationLock::acquire(&as_role(&lock_database), Duration::from_secs(5)) + .await + .expect("the migration lock as the role"); + // The role's setting ends a transaction it leaves idle. + let mut plain = sqlx::PgConnection::connect_with(&as_role(&s.catalog_name)) + .await + .unwrap(); + sqlx::raw_sql("BEGIN; SELECT 1") + .execute(&mut plain) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(32)).await; + assert!( + sqlx::query("SELECT 1").execute(&mut plain).await.is_err(), + "the role's idle_in_transaction_session_timeout did not end an idle transaction" + ); + drop(plain); + assert_eq!( + serving.lost().borrow().clone(), + None, + "the registration was lost" + ); + serving + .check_alive() + .await + .expect("the registration's session after two renewals"); + lock.check_alive() + .await + .expect("the migration lock's session after two renewals"); + serving.release().await; + lock.release().await; + drop(s); + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{role}\"")) + .execute(&admin) + .await; + admin.close().await; +} + +#[tokio::test] +async fn a_server_refuses_while_a_migrator_holds_only_the_migration_key() { + let Some(base) = base_conn() else { + return skip("legacy migrator"); + }; + let s = Scratch::new(&base, "pgm_legacy").await; + // A migrator of a release before 1.0, or one between taking its own key + // and the servers' key, holds the migration key alone. + let catalog = s.catalog().await; + let mut legacy = catalog.begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(4539458, 1)") + .execute(&mut *legacy) + .await + .unwrap(); + let refused = ServingRegistration::register(&s.options()).await; + assert!( + matches!(refused, Err(MigrationError::Locked(_))), + "registered beside a migrator" + ); + legacy.rollback().await.unwrap(); + ServingRegistration::register(&s.options()) + .await + .expect("registers once it is gone") + .release() + .await; + catalog.close().await; +} + +/// A table lock held a little longer than one lock timeout is waited out by +/// running the body again. +#[tokio::test] +async fn a_lock_held_past_one_timeout_is_waited_out() { + let Some(base) = base_conn() else { + return skip("lock retries"); + }; + let s = Scratch::new(&base, "pgm_retry").await; + s.load("0.1.14").await; + let catalog = s.catalog().await; + let mut blocker = catalog.begin().await.unwrap(); + sqlx::query("LOCK TABLE tables IN ACCESS EXCLUSIVE MODE") + .execute(&mut *blocker) + .await + .unwrap(); + let run = spawn_run(s.target()); + tokio::time::sleep(Duration::from_secs(7)).await; + blocker.rollback().await.unwrap(); + let report = run + .await + .unwrap() + .expect("the retry ran after the lock was released"); + assert_eq!(report.to, Some(LATEST)); + catalog.close().await; +} + +#[tokio::test] +async fn migrate_waits_for_running_servers_and_names_them() { + let Some(base) = base_conn() else { + return skip("serve and migrate"); + }; + let s = Scratch::new(&base, "pgm_serve").await; + s.load("0.1.14").await; + let serving = ServingRegistration::register(&s.options()).await.unwrap(); + let admin = pool(&format!("{base}/postgres"), 1).await; + let app: String = sqlx::query_scalar( + "SELECT a.application_name FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid \ + WHERE l.locktype = 'advisory' AND l.objid = 2 AND l.mode = 'ShareLock' AND l.granted", + ) + .fetch_one(&admin) + .await + .unwrap(); + assert_eq!(app, "extenddb serve"); + let err = migrate(&s.target(), RELEASE, Duration::from_secs(1)) + .await + .expect_err("refused"); + assert_eq!(err.exit_code(), 6, "{err}"); + assert!(err.to_string().contains("\"extenddb serve\""), "{err}"); + // The status and the preview read beside a running server. + assert!(matches!( + catalog_status(&s.target()).await.unwrap(), + CatalogStatus::Behind { .. } + )); + // A migrator waiting while the server stops goes on. + let waiting = { + let target = s.target(); + tokio::spawn(async move { migrate(&target, RELEASE, Duration::from_secs(30)).await }) + }; + // Once the waiting migrator holds the migration key, a second server + // cannot register. + let mut held = false; + for _ in 0..400 { + held = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype = 'advisory' \ + AND classid = 4539458 AND objid = 1 AND objsubid = 2 AND mode = 'ExclusiveLock' \ + AND granted AND database = (SELECT oid FROM pg_database WHERE datname = $1))", + ) + .bind(&s.catalog_name) + .fetch_one(&admin) + .await + .unwrap(); + if held { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert!(held, "the waiting migrator never took the migration key"); + assert!(ServingRegistration::register(&s.options()).await.is_err()); + serving.release().await; + let report = waiting + .await + .unwrap() + .expect("migrate after the server stopped"); + assert_eq!(report.to, Some(LATEST)); + let catalog = s.catalog().await; + assert_eq!( + count(&catalog, "SELECT count(*) FROM extenddb_migrations").await, + i64::try_from(MigrationId::ALL.len()).unwrap() + ); + catalog.close().await; + admin.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_fresh.rs b/crates/storage-postgres/tests/migrations_fresh.rs new file mode 100644 index 000000000..2542bb88b --- /dev/null +++ b/crates/storage-postgres/tests/migrations_fresh.rs @@ -0,0 +1,382 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! Migration 1 on empty databases, the expected definitions, the status of +//! every state, and the post-check. See `migrations_support` for the +//! database these tests need. + +mod migrations_support; + +use extenddb_storage::migrations::{CatalogStatus, MigrationId, StoreKind}; +use extenddb_storage_postgres::migrations::{ + Definitions, catalog_status, compare, describe, migrate, +}; +use migrations_support::{ + LATEST, Scratch, base_conn, count, ids_in, ledger, legacy_rows, pool, settings, skip, + step_count, +}; + +const CATALOG_BASELINE: &str = include_str!("../migrations/0001_baseline.sql"); +const DATA_BASELINE: &str = include_str!("../data_migrations/0001_baseline.sql"); +const RELEASE: &str = "1.0.0-test"; + +fn expected_path(dir: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join(dir) + .join("0001_baseline.expected.json") +} + +/// The committed expected definitions are what migration 1's files create in +/// empty databases. `EXTENDDB_BLESS_EXPECTED=1` writes them instead. +#[tokio::test] +async fn the_expected_definitions_are_what_the_baseline_files_create() { + let Some(base) = base_conn() else { + return skip("expected definitions"); + }; + let s = Scratch::new(&base, "pgm_expect").await; + for (url, sql, dir) in [ + (s.catalog_url(), CATALOG_BASELINE, "migrations"), + (s.data_url(), DATA_BASELINE, "data_migrations"), + ] { + let p = pool(&url, 1).await; + sqlx::raw_sql(sql) + .execute(&p) + .await + .expect("apply the baseline file"); + let defs = describe(&p).await.expect("describe"); + p.close().await; + let path = expected_path(dir); + if migrations_support::bless() { + let text = serde_json::to_string_pretty(&defs).expect("serialize") + "\n"; + std::fs::write(&path, text).expect("write the expected definitions"); + continue; + } + let committed: Definitions = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("parse"); + let (differences, notes) = compare(&committed, &defs); + assert!( + differences.is_empty() && notes.is_empty(), + "{dir}: {differences:?} {notes:?}" + ); + assert_eq!(committed, defs, "{dir}"); + } +} + +#[tokio::test] +async fn a_fresh_install_is_current_and_a_second_run_changes_nothing() { + let Some(base) = base_conn() else { + return skip("fresh install"); + }; + let s = Scratch::new(&base, "pgm_fresh").await; + assert_eq!(s.status().await, CatalogStatus::Uninitialized); + let report = migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect("migrate"); + assert_eq!(report.from, None); + assert_eq!(report.to, Some(LATEST)); + // Every id in order, the catalog first; the data database for the ids + // with a data body. + let order: Vec<(u32, String)> = report + .applied + .iter() + .map(|a| (a.id, a.store_name.clone())) + .collect(); + assert_eq!(order, s.all_steps()); + let (catalog, data) = (s.catalog().await, s.data().await); + let ids = |rows: Vec<(i32, String, String)>| rows.into_iter().map(|r| r.0).collect::>(); + assert_eq!(ids(ledger(&catalog).await), ids_in(StoreKind::Catalog)); + let data_ids = ids(ledger(&data).await); + assert_eq!(data_ids, ids_in(StoreKind::Data)); + // Migrations 1 and 4 have data bodies. + assert!(data_ids.starts_with(&[1, 4]), "{data_ids:?}"); + let settings_rows = settings(&catalog).await; + assert_eq!( + settings_rows.get("catalog_version"), + Some(&LATEST.to_string()) + ); + assert!(!settings_rows.contains_key("resource_tag_migration")); + assert_eq!(settings_rows.get("data_database_name"), Some(&s.data_name)); + assert_eq!( + settings_rows.get("data_database_connection_string"), + Some(&s.data_url()) + ); + // No baseline seeds a setting: the version and the connection rows only. + assert_eq!(settings_rows.len(), 3, "{settings_rows:?}"); + assert_eq!( + legacy_rows(&catalog).await, + [ + "001_schema.sql", + "002_vector_indexes.sql", + "003_backup_definitions.sql" + ] + ); + assert_eq!( + legacy_rows(&data).await, + [ + "001_data_schema.sql", + "002_gsi_pending.sql", + "003_gsi_base_key_index", + "003_idempotency_account_scope.sql", + "004_vector_index_state.sql" + ] + ); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + let before = ( + describe(&catalog).await.unwrap(), + describe(&data).await.unwrap(), + ledger(&catalog).await, + ); + let again = migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect("rerun"); + assert!(again.applied.is_empty(), "{again:?}"); + let after = ( + describe(&catalog).await.unwrap(), + describe(&data).await.unwrap(), + ledger(&catalog).await, + ); + assert_eq!(before, after); +} + +#[tokio::test] +async fn the_status_names_each_state_and_writes_nothing() { + let Some(base) = base_conn() else { + return skip("status"); + }; + // A catalog database that does not exist. + let missing = Scratch::new(&base, "pgm_missing").await; + let target = extenddb_storage_postgres::migrations::PostgresTarget::new( + format!("{base}/pgm_no_such_database").parse().unwrap(), + &missing.data_name, + ); + assert_eq!( + catalog_status(&target).await.unwrap(), + CatalogStatus::Uninitialized + ); + + let s = Scratch::new(&base, "pgm_status").await; + s.load("0.1.14").await; + let catalog = s.catalog().await; + let snapshot = || async { + let c = s.catalog().await; + let d = s.data().await; + let out = ( + describe(&c).await.unwrap(), + describe(&d).await.unwrap(), + settings(&c).await, + ); + c.close().await; + d.close().await; + out + }; + let before = snapshot().await; + let CatalogStatus::Behind { + version: None, + pending, + } = s.status().await + else { + panic!("a legacy catalog is behind at migration 1"); + }; + assert_eq!(pending.len(), step_count()); + assert_eq!( + before, + snapshot().await, + "reading the status wrote to the catalog" + ); + + s.migrate_through(MigrationId::LATEST).await; + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + // A release before 1.0 ran its `migrate`, which inserts the marker again. + sqlx::query("INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied')") + .execute(&catalog) + .await + .unwrap(); + assert!(matches!( + s.status().await, + CatalogStatus::WrittenByOlderRelease { .. } + )); + sqlx::query("DELETE FROM settings WHERE key = 'resource_tag_migration'") + .execute(&catalog) + .await + .unwrap(); + // A recorded checksum that is not this binary's. + let right: String = sqlx::query_scalar("SELECT checksum FROM extenddb_migrations WHERE id = 1") + .fetch_one(&catalog) + .await + .unwrap(); + sqlx::query("UPDATE extenddb_migrations SET checksum = 'x' WHERE id = 1") + .execute(&catalog) + .await + .unwrap(); + assert!(matches!( + s.status().await, + CatalogStatus::Changed { id: 1, .. } + )); + sqlx::query("UPDATE extenddb_migrations SET checksum = $1 WHERE id = 1") + .bind(&right) + .execute(&catalog) + .await + .unwrap(); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + // Ids this binary does not carry. + let next = i32::try_from(LATEST + 1).unwrap(); + sqlx::query( + "INSERT INTO extenddb_migrations (id, name, kind, checksum, outcome, applied_by) \ + VALUES ($1, 'later', 'additive', 'c', 'applied', 'later')", + ) + .bind(next) + .execute(&catalog) + .await + .unwrap(); + let status = s.status().await; + assert!( + matches!(status, CatalogStatus::AheadCompatible { .. }), + "{status:?}" + ); + sqlx::query("UPDATE extenddb_migrations SET kind = 'breaking' WHERE id = $1") + .bind(next) + .execute(&catalog) + .await + .unwrap(); + assert!(matches!( + s.status().await, + CatalogStatus::AheadIncompatible { .. } + )); + catalog.close().await; +} + +/// Migration 1 never moves a stream sequence back and leaves the tokens of an +/// idempotency table that is already keyed per account. +#[tokio::test] +async fn migration_1_keeps_the_stream_sequences_and_the_idempotency_tokens() { + let Some(base) = base_conn() else { + return skip("sequences and tokens"); + }; + let s = Scratch::new(&base, "pgm_seq").await; + s.load("0.1.14").await; + let (catalog, data) = (s.catalog().await, s.data().await); + // Ahead of the clock by about eleven days, as after a clock correction. + let ahead: i64 = + sqlx::query_scalar("SELECT (EXTRACT(EPOCH FROM now()) * 1000000)::BIGINT + 1000000000000") + .fetch_one(&data) + .await + .unwrap(); + for p in [&catalog, &data] { + sqlx::query("SELECT setval('stream_seq', $1)") + .bind(ahead) + .execute(p) + .await + .unwrap(); + } + sqlx::query("INSERT INTO idempotency_tokens (account_id, token, fingerprint) VALUES ('123456789012', 't1', 'f')") + .execute(&data) + .await + .unwrap(); + migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect("migrate"); + for p in [&catalog, &data] { + let last: i64 = sqlx::query_scalar("SELECT last_value FROM stream_seq") + .fetch_one(p) + .await + .unwrap(); + assert!(last >= ahead, "stream_seq went back from {ahead} to {last}"); + } + assert_eq!( + count(&data, "SELECT count(*) FROM idempotency_tokens").await, + 1 + ); + catalog.close().await; + data.close().await; +} + +/// A difference the post-check finds stops migration 1 in that store and +/// commits nothing there; a pre-release `table_restores` is accepted with a +/// note. +#[tokio::test] +async fn the_post_check_stops_a_different_shape_and_accepts_the_pre_release_one() { + let Some(base) = base_conn() else { + return skip("post-check"); + }; + let s = Scratch::new(&base, "pgm_post").await; + s.load("0.1.14").await; + let catalog = s.catalog().await; + sqlx::raw_sql( + "ALTER TABLE accounts ADD COLUMN extra TEXT NOT NULL DEFAULT 'x'; \ + ALTER TABLE accounts ALTER COLUMN created_at SET DEFAULT '2020-01-01 00:00:00+00'", + ) + .execute(&catalog) + .await + .unwrap(); + let err = migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect_err("refused"); + let message = err.to_string(); + assert!( + message.contains("differs from the v1.0 schema in 2 place(s)") + && message.contains("column accounts.extra is not part of the v1.0 schema; drop it") + && message.contains("column accounts.created_at has default '2020-01-01") + && message.contains( + "where v1.0 has now(); change it to timestamp with time zone NOT NULL DEFAULT \ + now()" + ) + && message.contains("then run migrate --yes again") + && !message.contains("Column {"), + "{message}" + ); + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM pg_class WHERE relname = 'extenddb_migrations'" + ) + .await, + 0 + ); + let rows = settings(&catalog).await; + assert_eq!( + rows.get("catalog_version").map(String::as_str), + Some("0.0.3") + ); + assert!(rows.contains_key("resource_tag_migration")); + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM pg_class WHERE relname = 'backup_definitions'" + ) + .await, + 0 + ); + sqlx::raw_sql( + "ALTER TABLE accounts DROP COLUMN extra; \ + ALTER TABLE accounts ALTER COLUMN created_at SET DEFAULT now()", + ) + .execute(&catalog) + .await + .unwrap(); + + // The pre-release revision of legacy 003, then migrate. + sqlx::raw_sql(include_str!("fixtures/003_backup_definitions_earlier.sql")) + .execute(&catalog) + .await + .unwrap(); + migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect("migrate"); + let note: String = sqlx::query_scalar("SELECT notes FROM extenddb_migrations WHERE id = 1") + .fetch_one(&catalog) + .await + .unwrap(); + assert!( + note.contains("table_restores.source_table_arn is nullable"), + "{note}" + ); + let default: Option = sqlx::query_scalar( + "SELECT column_default FROM information_schema.columns \ + WHERE table_name = 'table_restores' AND column_name = 'restore_date_time'", + ) + .fetch_one(&catalog) + .await + .unwrap(); + assert_eq!(default, None); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + catalog.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_golden.rs b/crates/storage-postgres/tests/migrations_golden.rs new file mode 100644 index 000000000..47d36bb25 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_golden.rs @@ -0,0 +1,424 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! Golden tests of the PostgreSQL code bodies: each runs on a released +//! fixture prepared with the damage it repairs, and the rows it changed and +//! its ledger notes are compared with `tests/fixtures/golden/000N_.txt`. +//! `EXTENDDB_BLESS_EXPECTED=1` writes the files instead. See +//! `migrations_support` for the database these tests need. + +mod migrations_support; + +use std::fmt::Write as _; +use std::time::Duration; + +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId, Runner}; +use extenddb_storage_postgres::migrations::{PostgresMigrations, migrate}; +use migrations_support::{ACCOUNT, LATEST, Scratch, base_conn, id_names, skip}; +use sqlx::PgPool; + +fn golden(name: &str, actual: &str) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/golden") + .join(name); + if migrations_support::bless() { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, actual).unwrap(); + return; + } + let expected = + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + assert_eq!(actual, expected, "{name} differs from its golden file"); +} + +async fn notes(catalog: &PgPool, id: i32) -> String { + sqlx::query_scalar("SELECT notes FROM extenddb_migrations WHERE id = $1") + .bind(id) + .fetch_one(catalog) + .await + .unwrap() +} + +const TAGGED: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}"#; +/// Both forms on one key with different values, which migration 2 refuses. +const COLLIDING: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod","aws:ResourceTag/env":"dev"}}}]}"#; + +/// A `dynamodb:ResourceTag` policy in every place a policy is stored. +async fn tag_every_policy(catalog: &PgPool) { + for sql in [ + "UPDATE iam_policies SET policy_document = $1::jsonb WHERE principal_name = 'tester'", + "INSERT INTO iam_groups (account_id, group_name, group_arn) VALUES ('123456789012', 'g1', 'arn:aws:iam::123456789012:group/g1')", + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, policy_document) VALUES ('123456789012', 'group', 'g1', 'gp', $1::jsonb)", + "INSERT INTO iam_roles (account_id, role_name, role_arn, trust_policy) VALUES ('123456789012', 'r1', 'arn:aws:iam::123456789012:role/r1', '{}')", + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, policy_document) VALUES ('123456789012', 'role', 'r1', 'rp', $1::jsonb)", + "INSERT INTO iam_permissions_boundaries (account_id, principal_type, principal_name, policy_document) VALUES ('123456789012', 'user', 'tester', $1::jsonb)", + "INSERT INTO iam_permissions_boundaries (account_id, principal_type, principal_name, policy_document) VALUES ('123456789012', 'role', 'r1', $1::jsonb)", + "INSERT INTO iam_sessions (session_token, access_key_id, secret_key_encrypted, account_id, role_name, session_name, session_policy, expires_at) VALUES ('tok', 'ASIAEXAMPLE', '\\x00', '123456789012', 'r1', 's', $1::jsonb, now() + interval '1 day')", + ] { + let q = sqlx::query(sql); + let q = if sql.contains("$1") { + q.bind(TAGGED) + } else { + q + }; + q.execute(catalog) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + } +} + +#[tokio::test] +async fn migration_2_runs_again_after_a_lock_timeout() { + let Some(base) = base_conn() else { + return skip("migration 2 lock timeout"); + }; + let s = Scratch::new(&base, "pgm_g2lock").await; + s.load("0.1.14").await; + s.migrate_through(MigrationId::Baseline).await; + let catalog = s.catalog().await; + tag_every_policy(&catalog).await; + // Another session holds the policies table in SHARE mode for 7 s: the + // body's reads go on, and its first write waits past the runner's 5 s + // lock timeout. The management store's error keeps SQLSTATE 55P03, so + // the runner runs the body again, and the second attempt writes once the + // lock is released. + let holder = migrations_support::pool(&s.catalog_url(), 1).await; + let mut held = holder.begin().await.unwrap(); + sqlx::query("LOCK TABLE iam_policies IN SHARE MODE") + .execute(&mut *held) + .await + .unwrap(); + let release = tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(7)).await; + held.commit().await.unwrap(); + holder.close().await; + }); + let started = std::time::Instant::now(); + s.migrate_through(MigrationId::ResourceTagConditionKeys) + .await; + release.await.unwrap(); + assert!( + started.elapsed() >= Duration::from_secs(5), + "the body did not wait for the lock: {:?}", + started.elapsed() + ); + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM iam_policies WHERE policy_document::text LIKE '%dynamodb:ResourceTag%'", + ) + .fetch_one(&catalog) + .await + .unwrap(); + assert_eq!(left, 0, "a policy was not rewritten"); + catalog.close().await; +} + +#[tokio::test] +async fn migration_2_rewrites_resource_tag_condition_keys_in_every_place() { + let Some(base) = base_conn() else { + return skip("golden 2"); + }; + let s = Scratch::new(&base, "pgm_g2").await; + s.load("0.1.14").await; + s.migrate_through(MigrationId::Baseline).await; + let catalog = s.catalog().await; + tag_every_policy(&catalog).await; + s.migrate_through(MigrationId::ResourceTagConditionKeys) + .await; + let mut out = String::new(); + // Sorted by byte order (`COLLATE "C"`), so the file is the same whatever + // the database's collation and C library. + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT * FROM (SELECT 'policy' AS kind, principal_type || ' ' || principal_name || ' ' || policy_name AS name, policy_document::text, '' \ + FROM iam_policies UNION ALL \ + SELECT 'boundary', principal_type || ' ' || principal_name, policy_document::text, '' FROM iam_permissions_boundaries UNION ALL \ + SELECT 'session', session_token, session_policy::text, '' FROM iam_sessions) AS r \ + ORDER BY kind COLLATE \"C\", name COLLATE \"C\"", + ) + .fetch_all(&catalog) + .await + .unwrap(); + for (kind, name, document, _) in rows { + writeln!(out, "{kind} {name}: {document}").unwrap(); + } + // The body visits policies in the order the management store lists + // them, which follows the database's collation, so the notes are sorted. + let mut rewrote: Vec = notes(&catalog, 2) + .await + .lines() + .map(str::to_owned) + .collect(); + rewrote.sort(); + writeln!(out, "notes:\n{}", rewrote.join("\n")).unwrap(); + golden("0002_resource_tag_condition_keys.txt", &out); + catalog.close().await; +} + +#[tokio::test] +async fn migration_2_collisions_stop_the_run_before_any_write() { + let Some(base) = base_conn() else { + return skip("golden 2 collision"); + }; + let s = Scratch::new(&base, "pgm_g2c").await; + s.load("0.1.14").await; + let catalog = s.catalog().await; + sqlx::query( + "UPDATE iam_policies SET policy_document = $1::jsonb WHERE principal_name = 'tester'", + ) + .bind(COLLIDING) + .execute(&catalog) + .await + .unwrap(); + assert!(matches!( + s.status().await, + CatalogStatus::NeedsAttention { .. } + )); + let err = migrate(&s.target(), "test", Duration::from_secs(5)) + .await + .expect_err("refused"); + assert!( + matches!( + err, + MigrationError::Status(CatalogStatus::NeedsAttention { .. }) + ), + "{err}" + ); + assert_eq!(err.exit_code(), 5); + let ledger: bool = + sqlx::query_scalar("SELECT to_regclass('public.extenddb_migrations') IS NOT NULL") + .fetch_one(&catalog) + .await + .unwrap(); + assert!(!ledger, "a refused run wrote the ledger"); + catalog.close().await; +} + +/// The text of every stored policy, sorted by byte order. +async fn every_policy_document(catalog: &PgPool) -> Vec { + sqlx::query_scalar( + "SELECT kind || ' ' || name || ': ' || document FROM (SELECT 'policy' AS kind, \ + principal_type || ' ' || principal_name || ' ' || policy_name AS name, \ + policy_document::text AS document FROM iam_policies UNION ALL \ + SELECT 'boundary', principal_type || ' ' || principal_name, policy_document::text \ + FROM iam_permissions_boundaries UNION ALL \ + SELECT 'session', session_token, session_policy::text FROM iam_sessions) AS r \ + ORDER BY kind COLLATE \"C\", name COLLATE \"C\"", + ) + .fetch_all(catalog) + .await + .unwrap() +} + +/// Record what a release before 1.0 leaves when it writes to a migrated +/// catalog, then reconverge: migration 2 is named as skipped, and the +/// catalog is current again. +async fn reconverge_after_an_older_release_wrote(s: &Scratch, catalog: &PgPool) { + sqlx::query("INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied')") + .execute(catalog) + .await + .unwrap(); + assert_eq!(s.status().await.label(), "WrittenByOlderRelease"); + let m = PostgresMigrations::open_unlocked(&s.target()) + .await + .unwrap(); + let report = Runner::new(&m, "test").reconverge().await; + m.close().await; + let report = report.expect("reconverge"); + assert!(report.applied.iter().all(|step| step.id != 2), "{report:?}"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), + "{report:?}" + ); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); +} + +/// From 1.0 on a `dynamodb:ResourceTag` key written through the API has no +/// effect, as on the service. Migration 2 is not convergent, so a reconverge +/// leaves a policy written after it, in each place a policy is stored, with +/// the text it was written with, and so with the decision it had. +#[tokio::test] +async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { + let Some(base) = base_conn() else { + return skip("reconverge after migration 2"); + }; + let s = Scratch::new(&base, "pgm_g2rc").await; + s.load("0.1.14").await; + s.migrate_through(MigrationId::LATEST).await; + let catalog = s.catalog().await; + tag_every_policy(&catalog).await; + let before = every_policy_document(&catalog).await; + assert!( + before + .iter() + .any(|d| d.contains("dynamodb:ResourceTag/env")), + "{before:?}" + ); + reconverge_after_an_older_release_wrote(&s, &catalog).await; + assert_eq!(every_policy_document(&catalog).await, before); + catalog.close().await; +} + +/// A policy written after migration 2 with both forms on one key, which +/// the rewrite would refuse, does not stop a reconverge. +#[tokio::test] +async fn a_reconverge_is_not_stopped_by_a_policy_migration_2_would_refuse() { + let Some(base) = base_conn() else { + return skip("reconverge after migration 2, both forms"); + }; + let s = Scratch::new(&base, "pgm_g2rcc").await; + s.load("0.1.14").await; + s.migrate_through(MigrationId::LATEST).await; + let catalog = s.catalog().await; + sqlx::query( + "UPDATE iam_policies SET policy_document = $1::jsonb WHERE principal_name = 'tester'", + ) + .bind(COLLIDING) + .execute(&catalog) + .await + .unwrap(); + let before = every_policy_document(&catalog).await; + reconverge_after_an_older_release_wrote(&s, &catalog).await; + assert_eq!(every_policy_document(&catalog).await, before); + catalog.close().await; +} + +#[tokio::test] +async fn migration_3_restores_sort_key_definitions_in_tables_and_backups() { + let Some(base) = base_conn() else { + return skip("golden 3"); + }; + let s = Scratch::new(&base, "pgm_g3").await; + s.load("0.1.14").await; + s.migrate_through(MigrationId::ResourceTagConditionKeys) + .await; + let catalog = s.catalog().await; + // The pre-fix UpdateTable kept only the GSI's definitions: the table's + // sort key `sk` and partition key `pk` are gone, from the table and from + // the backup taken of it. + let damaged = r#"[{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"lsk","AttributeType":"N"}]"#; + for sql in [ + "UPDATE tables SET attribute_definitions = $1::jsonb WHERE table_name = 'seed_tbl'", + "UPDATE backups SET attribute_definitions = $1::jsonb WHERE table_name = 'seed_tbl'", + ] { + sqlx::query(sql) + .bind(damaged) + .execute(&catalog) + .await + .unwrap(); + } + // A backup with no items whose source table is gone gives no type. + sqlx::query( + "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, key_schema, attribute_definitions) \ + VALUES ('arn:orphan', 'orphan', 'gone', 'gone_tbl', $1, \ + '[{\"AttributeName\":\"pk\",\"KeyType\":\"HASH\"},{\"AttributeName\":\"sk\",\"KeyType\":\"RANGE\"}]', '[]')", + ) + .bind(ACCOUNT) + .execute(&catalog) + .await + .unwrap(); + s.migrate_through(MigrationId::SortKeyDefinitions).await; + let mut out = String::new(); + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT * FROM (SELECT 'table ' || table_name AS name, attribute_definitions::text FROM tables UNION ALL \ + SELECT 'backup ' || backup_name, attribute_definitions::text FROM backups) AS r \ + ORDER BY name COLLATE \"C\"", + ) + .fetch_all(&catalog) + .await + .unwrap(); + for (name, defs) in rows { + writeln!(out, "{name}: {defs}").unwrap(); + } + let arn: String = + sqlx::query_scalar("SELECT backup_arn FROM backups WHERE backup_name = 'seed_backup'") + .fetch_one(&catalog) + .await + .unwrap(); + writeln!( + out, + "notes:\n{}", + notes(&catalog, 3).await.replace(&arn, "") + ) + .unwrap(); + golden("0003_sort_key_definitions.txt", &out); + catalog.close().await; +} + +#[tokio::test] +async fn migration_4_adds_the_base_key_index_and_skips_what_it_cannot_reach() { + let Some(base) = base_conn() else { + return skip("golden 4"); + }; + let s = Scratch::new(&base, "pgm_g4").await; + // Index tables created by 0.1.5 have no base key index. + s.load("0.1.5").await; + s.migrate_through(MigrationId::SortKeyDefinitions).await; + let catalog = s.catalog().await; + // A table being deleted, and a table whose index table is gone. + for (name, status, table_id, index_id) in [ + ( + "going", + "DELETING", + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-0000000000a1", + ), + ( + "lost", + "ACTIVE", + "00000000-0000-4000-8000-000000000002", + "00000000-0000-4000-8000-0000000000a2", + ), + ] { + sqlx::query( + "INSERT INTO tables (account_id, table_name, key_schema, attribute_definitions, table_status, table_arn, table_id) \ + VALUES ($1, $2, '[]', '[]', $3, 'arn:' || $2, $4)", + ) + .bind(ACCOUNT) + .bind(name) + .bind(status) + .bind(table_id) + .execute(&catalog) + .await + .unwrap(); + sqlx::query( + "INSERT INTO indexes (table_id, index_id, index_name, index_type, key_schema, projection) \ + VALUES ($1, $2, $3 || '_gsi', 'GSI', '[]', '{}')", + ) + .bind(table_id) + .bind(index_id) + .bind(name) + .execute(&catalog) + .await + .unwrap(); + } + s.migrate_through(MigrationId::IndexTableBaseKey).await; + let data = s.data().await; + let names = id_names(&catalog).await; + let mut lines: Vec = sqlx::query_as::<_, (String, bool)>( + "SELECT pg_get_indexdef(i.indexrelid), i.indisvalid FROM pg_index i \ + JOIN pg_class c ON c.oid = i.indexrelid WHERE c.relname LIKE '%\\_base\\_key\\_idx'", + ) + .fetch_all(&data) + .await + .unwrap() + .into_iter() + .map(|(def, valid)| { + let mut def = def; + for (id, name) in &names { + def = def.replace(id.as_str(), name); + } + format!("{def} valid={valid}") + }) + .collect(); + lines.sort(); + let mut out = lines.join("\n"); + let mut note = notes(&data, 4).await; + for (id, name) in &names { + note = note.replace(id.as_str(), name); + } + write!(out, "\nnotes:\n{note}\n").unwrap(); + golden("0004_index_table_base_key.txt", &out); + catalog.close().await; + data.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_guards.rs b/crates/storage-postgres/tests/migrations_guards.rs new file mode 100644 index 000000000..e13886aed --- /dev/null +++ b/crates/storage-postgres/tests/migrations_guards.rs @@ -0,0 +1,566 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! The backend's guards that no released body reaches: a code body that +//! commits the runner's transaction itself is stopped before its ledger +//! row, the serving session carries its keepalive settings, a registration +//! whose database stops answering is reported lost, a runner whose client +//! goes away mid-body is ended by the database, an index build that waits +//! on a lock is bounded, and a role that does not own the tables is told +//! what to run. See `migrations_support` for the database these tests need. + +mod migrations_support; + +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + CodeBody, Done, HasManagementStore, Kind, LedgerRow, MigrationBackend, MigrationError, + MigrationId, Outcome, RunKind, Runner, Step, StoreKind, StoreRef, Target, before_lock, exit, +}; +use extenddb_storage_postgres::migrations::{ + MigrationLock, PostgresCatalogMigrations, PostgresMigrations, PostgresStores, PostgresTarget, + ServingRegistration, migrate, status_notes, +}; +use migrations_support::proxy::Proxy; +use migrations_support::{ACCOUNT, Scratch, base_conn, count, note_steps, pool, skip}; + +/// Creates a user through the management store, which commits a +/// transaction of its own on the runner's connection. +struct CommitsItself; + +#[async_trait] +impl CodeBody for CommitsItself { + fn fingerprint(&self) -> &'static str { + "0002_resource_tag_condition_keys/test/1" + } + + async fn detect(&self, _stores: &PostgresStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &PostgresStores) -> Result { + stores + .management_store() + .create_user(ACCOUNT, "made_by_a_body", None) + .await + .map_err(|e| MigrationError::Store(format!("{e:?}")))?; + Ok(Done::default()) + } +} + +#[tokio::test] +async fn a_body_that_commits_the_runners_transaction_is_stopped_before_its_row() { + let Some(base) = base_conn() else { + return skip("commit guard"); + }; + let s = Scratch::new(&base, "pgm_guard").await; + s.migrate_through(MigrationId::Baseline).await; + let catalog = s.catalog().await; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, 'a')") + .bind(ACCOUNT) + .execute(&catalog) + .await + .unwrap(); + let m = PostgresMigrations::open_unlocked(&s.target()) + .await + .unwrap(); + let store = StoreRef { + kind: StoreKind::Catalog, + name: s.catalog_name.clone(), + }; + let row = LedgerRow { + id: 2, + name: "resource_tag_condition_keys".to_owned(), + kind: Kind::Breaking, + checksum: "c".to_owned(), + outcome: Outcome::Applied, + notes: String::new(), + applied_at: String::new(), + applied_by: "test".to_owned(), + }; + let err = m + .apply( + &store, + MigrationId::ResourceTagConditionKeys, + Step::Code(&CommitsItself), + Some(row), + Some(2), + ) + .await + .expect_err("stopped"); + assert!( + err.to_string() + .contains("committed the runner's transaction itself"), + "{err}" + ); + m.close().await; + // The ledger has no row for the body, whatever the body committed. + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM extenddb_migrations WHERE id = 2" + ) + .await, + 0 + ); + catalog.close().await; +} + +#[tokio::test] +async fn the_serving_session_sets_its_keepalives_and_name() { + let Some(base) = base_conn() else { + return skip("serving session"); + }; + let s = Scratch::new(&base, "pgm_ka").await; + let serving = ServingRegistration::register(&s.options()).await.unwrap(); + let settings = serving.session_settings().await.unwrap(); + assert_eq!( + settings, + [ + ("application_name".to_owned(), "extenddb serve".to_owned()), + ("tcp_keepalives_count".to_owned(), "4".to_owned()), + ("tcp_keepalives_idle".to_owned(), "10".to_owned()), + ("tcp_keepalives_interval".to_owned(), "5".to_owned()), + ] + ); + serving.release().await; +} + +/// A database that stops answering without closing the connection: the +/// registration counts as lost within twice its keepalive interval, and the +/// migration lock's check fails instead of waiting. +#[tokio::test] +async fn a_registration_whose_database_stops_answering_is_lost() { + let Some(base) = base_conn() else { + return skip("renewal deadline"); + }; + let s = Scratch::new(&base, "pgm_pause").await; + let options = s.options(); + let proxy = Proxy::start((options.get_host().to_owned(), options.get_port())).await; + let serving = ServingRegistration::register_with(&proxy.options(&s), Duration::from_secs(1)) + .await + .unwrap(); + let mut lost = serving.lost(); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!(lost.borrow().is_none(), "lost while the database answers"); + proxy.paused.store(true, Ordering::SeqCst); + let paused_at = Instant::now(); + tokio::time::timeout(Duration::from_secs(10), lost.changed()) + .await + .expect("the registration is reported lost within 10 s of the pause") + .unwrap(); + let reason = lost.borrow().clone().unwrap(); + assert!(reason.contains("could not be renewed"), "{reason}"); + assert!(paused_at.elapsed() >= Duration::from_secs(1), "{reason}"); + proxy.paused.store(false, Ordering::SeqCst); + serving.release().await; + + let lock = MigrationLock::acquire_with( + &proxy.options(&s), + Duration::from_secs(5), + Duration::from_secs(3), + ) + .await + .unwrap(); + proxy.paused.store(true, Ordering::SeqCst); + let checked = tokio::time::timeout(Duration::from_secs(10), lock.check_alive()) + .await + .expect("the check returns instead of waiting for the database"); + let err = checked.expect_err("an unanswered check fails"); + assert!(err.to_string().contains("did not answer"), "{err}"); + proxy.paused.store(false, Ordering::SeqCst); + lock.release().await; +} + +/// At a keepalive interval above the floors (3 s: a renewal waits 2 s, and +/// the deadline is 6 s), a registration whose database stops answering +/// right after it registered counts as lost 6 s after its last renewal, the +/// registration: the renewal at 3 s gets no answer by 5 s, and the next one +/// waits only for the 1 s left. +#[tokio::test] +async fn a_registration_is_lost_at_twice_its_interval_and_no_later() { + let Some(base) = base_conn() else { + return skip("renewal deadline at 3 s"); + }; + let s = Scratch::new(&base, "pgm_pause3").await; + let options = s.options(); + let proxy = Proxy::start((options.get_host().to_owned(), options.get_port())).await; + let serving = ServingRegistration::register_with(&proxy.options(&s), Duration::from_secs(3)) + .await + .unwrap(); + let mut lost = serving.lost(); + proxy.paused.store(true, Ordering::SeqCst); + let paused_at = Instant::now(); + tokio::time::timeout(Duration::from_secs(15), lost.changed()) + .await + .expect("the registration is reported lost within 15 s of the pause") + .unwrap(); + let elapsed = paused_at.elapsed(); + let reason = lost.borrow().clone().unwrap(); + assert!(reason.contains("could not be renewed"), "{reason}"); + assert!( + elapsed >= Duration::from_millis(5500) && elapsed < Duration::from_millis(6800), + "lost {elapsed:?} after the pause: {reason}" + ); + proxy.paused.store(false, Ordering::SeqCst); + serving.release().await; +} + +/// Writes a settings row and then sleeps for 60 s, inside the runner's +/// transaction on the catalog. +struct SleepsAfterAWrite; + +#[async_trait] +impl CodeBody for SleepsAfterAWrite { + fn fingerprint(&self) -> &'static str { + "0002_resource_tag_condition_keys/test/2" + } + + async fn detect(&self, _stores: &PostgresStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &PostgresStores) -> Result { + let catalog = stores.catalog_connection(); + for sql in [ + "INSERT INTO settings (key, value) VALUES ('written_by_a_body', 'x')", + "SELECT pg_sleep(60)", + ] { + sqlx::query(sql) + .execute(catalog) + .await + .map_err(|e| MigrationError::store("the sleeping body", e))?; + } + Ok(Done::default()) + } +} + +/// A runner whose client side goes away in the middle of a body, as when +/// the migrator's process dies: the database ends the body's statement and +/// its session within seconds instead of running the statement to its end, +/// and nothing of the body or its ledger row is committed. +#[tokio::test] +async fn a_runner_whose_client_goes_away_mid_body_is_ended_by_the_database() { + let Some(base) = base_conn() else { + return skip("client gone"); + }; + let s = Scratch::new(&base, "pgm_gone").await; + s.migrate_through(MigrationId::Baseline).await; + let options = s.options(); + let proxy = Proxy::start((options.get_host().to_owned(), options.get_port())).await; + let target = PostgresTarget::new(proxy.options(&s), &s.data_name); + let store = StoreRef { + kind: StoreKind::Catalog, + name: s.catalog_name.clone(), + }; + let run = tokio::spawn(async move { + let m = PostgresMigrations::open_unlocked(&target).await?; + let row = LedgerRow { + id: 2, + name: "resource_tag_condition_keys".to_owned(), + kind: Kind::Breaking, + checksum: "c".to_owned(), + outcome: Outcome::Applied, + notes: String::new(), + applied_at: String::new(), + applied_by: "test".to_owned(), + }; + let applied = m + .apply( + &store, + MigrationId::ResourceTagConditionKeys, + Step::Code(&SleepsAfterAWrite), + Some(row), + Some(2), + ) + .await; + m.close().await; + applied.map(|_| ()) + }); + let admin = pool(&format!("{base}/postgres"), 1).await; + let mut sleeping = None; + for _ in 0..400 { + sleeping = sqlx::query_scalar( + "SELECT pid FROM pg_stat_activity WHERE datname = $1 \ + AND starts_with(application_name, 'extenddb migrate run ') \ + AND query LIKE 'SELECT pg_sleep%' AND state = 'active'", + ) + .bind(&s.catalog_name) + .fetch_optional(&admin) + .await + .unwrap(); + if sleeping.is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let pid: i32 = sleeping.expect("the body's statement never ran"); + proxy.cut.store(true, Ordering::SeqCst); + let cut_at = Instant::now(); + let mut ended = false; + while cut_at.elapsed() < Duration::from_secs(10) { + let present: bool = + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)") + .bind(pid) + .fetch_one(&admin) + .await + .unwrap(); + if !present { + ended = true; + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!( + ended, + "the session of pid {pid} still ran its statement 10 s after its client went away" + ); + let err = run + .await + .unwrap() + .expect_err("a body whose client went away fails"); + assert!(!err.to_string().is_empty()); + let catalog = s.catalog().await; + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM settings WHERE key = 'written_by_a_body'" + ) + .await, + 0, + "the body's row was committed" + ); + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM extenddb_migrations WHERE id = 2" + ) + .await, + 0, + "the body's ledger row was committed" + ); + catalog.close().await; + admin.close().await; +} + +/// Migration 4's concurrent build that waits on a lock is cancelled after the +/// lock timeout and built again, and the run stops naming the holder once +/// every attempt waited; nothing is recorded, and the rerun builds it. +#[tokio::test] +async fn an_index_build_behind_a_held_lock_stops_naming_the_holder() { + let Some(base) = base_conn() else { + return skip("index lock wait"); + }; + let s = Scratch::new(&base, "pgm_cicwait").await; + s.load("0.1.5").await; + s.migrate_through(MigrationId::SortKeyDefinitions).await; + let (catalog, data) = (s.catalog().await, s.data().await); + let index_id: String = + sqlx::query_scalar("SELECT index_id FROM indexes WHERE index_name = 'gsi1'") + .fetch_one(&catalog) + .await + .unwrap(); + let mut blocker = data.begin().await.unwrap(); + sqlx::query(&format!( + "LOCK TABLE \"_ddb_{index_id}\" IN ROW EXCLUSIVE MODE" + )) + .execute(&mut *blocker) + .await + .unwrap(); + let blocker_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *blocker) + .await + .unwrap(); + let started = Instant::now(); + let m = PostgresMigrations::open_unlocked(&s.target()) + .await + .unwrap(); + let result = tokio::time::timeout(Duration::from_secs(60), Runner::new(&m, "test").migrate()) + .await + .expect("the build's lock wait is bounded"); + m.close().await; + let err = result.expect_err("every attempt waited on the held lock"); + let message = err.to_string(); + assert!( + message.contains("could not be built") && message.contains(&format!("pid {blocker_pid}")), + "{message}" + ); + assert!(started.elapsed() >= Duration::from_secs(15), "{message}"); + blocker.rollback().await.unwrap(); + let ids: Vec = sqlx::query_scalar("SELECT id FROM extenddb_migrations ORDER BY id") + .fetch_all(&data) + .await + .unwrap(); + assert!( + !ids.contains(&4), + "migration 4 recorded in the data database: {ids:?}" + ); + s.migrate_through(MigrationId::LATEST).await; + for p in [catalog, data] { + p.close().await; + } +} + +/// A released catalog whose tables a superuser owns: the application role +/// may read and write them but not alter them, and migration 1 stops naming +/// the owner and the statements that give the tables to the role. +#[tokio::test] +async fn a_role_that_does_not_own_the_tables_is_told_what_to_run() { + let Some(base) = base_conn() else { + return skip("table owners"); + }; + let s = Scratch::new(&base, "pgm_owner").await; + s.load("0.1.14").await; + let role = format!("{}_app", s.catalog_name); + let admin = pool(&format!("{base}/postgres"), 1).await; + sqlx::query(&format!( + "CREATE ROLE \"{role}\" LOGIN PASSWORD 'owner-test'" + )) + .execute(&admin) + .await + .unwrap(); + for database in [&s.catalog_name, &s.data_name] { + sqlx::query(&format!( + "GRANT ALL ON DATABASE \"{database}\" TO \"{role}\"" + )) + .execute(&admin) + .await + .unwrap(); + let p = pool(&format!("{base}/{database}"), 1).await; + for grant in [ + "GRANT ALL ON SCHEMA public TO \"{role}\"", + "GRANT ALL ON ALL TABLES IN SCHEMA public TO \"{role}\"", + "GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO \"{role}\"", + ] { + sqlx::query(&grant.replace("{role}", &role)) + .execute(&p) + .await + .unwrap(); + } + p.close().await; + } + // A view the superuser owns: the note names it and the statement it + // prints gives it to the role with the tables. + let catalog_admin = pool(&format!("{base}/{}", s.catalog_name), 1).await; + sqlx::query("CREATE VIEW owner_test_view AS SELECT 1 AS x") + .execute(&catalog_admin) + .await + .unwrap(); + catalog_admin.close().await; + let mut target = s.target(); + target.catalog = target.catalog.username(&role).password("owner-test"); + let host = base + .rsplit('@') + .next() + .expect("a host in the connection string"); + // The server, and so the runner, reaches the data database as the role + // too, through the catalog's data connection string row. + let catalog_admin = pool(&format!("{base}/{}", s.catalog_name), 1).await; + sqlx::query("UPDATE settings SET value = $1 WHERE key = 'data_database_connection_string'") + .bind(format!( + "postgresql://{role}:owner-test@{host}/{}", + s.data_name + )) + .execute(&catalog_admin) + .await + .unwrap(); + catalog_admin.close().await; + // Before any run: a note per database names the owner and the + // statement that gives that database's objects to the role, and + // migrate --yes refuses on them before it takes the lock. + let notes = status_notes(&target).await.expect("status notes"); + assert_eq!(notes.len(), 2, "{notes:?}"); + assert!( + notes[0] + .text + .contains(&format!("catalog database {}:", s.catalog_name)), + "{notes:?}" + ); + assert!( + notes[1] + .text + .contains(&format!("data database {}:", s.data_name)), + "{notes:?}" + ); + for note in ¬es { + assert!(note.refuses_runs, "{note:?}"); + assert!( + note.text + .contains(&format!("the application role {role} does not own")) + && note.text.contains("owned by postgres") + && note.text.contains("OWNER TO") + && !note.text.contains("REASSIGN"), + "{}", + note.text + ); + } + let location = format!("postgresql://{role}:owner-test@{host}/{}", s.catalog_name); + let refused = before_lock( + &PostgresCatalogMigrations, + &Target::at(location.as_str()), + RunKind::Migrate, + ) + .await + .expect_err("migrate --yes refuses before the lock"); + assert_eq!(refused.exit_code(), exit::ERROR); + assert!( + refused.to_string().contains("no lock was taken"), + "{refused}" + ); + let err = migrate(&target, "test", Duration::from_secs(5)) + .await + .expect_err("the role cannot alter tables it does not own"); + let message = err.to_string(); + assert!( + message.contains("must own every table") + && message.contains("owned by postgres") + && message.contains("DO $$") + && message.contains("OWNER TO") + && !message.contains("REASSIGN"), + "{message}" + ); + let catalog = s.catalog().await; + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM pg_tables WHERE tablename = 'extenddb_migrations'" + ) + .await, + 0, + "migration 1 committed something" + ); + catalog.close().await; + // The statements the note prints give the objects to the role; the note + // is then gone and the run applies every migration. The objects are a + // superuser's, so a superuser runs each of them. + for (note, database) in notes.iter().zip([&s.catalog_name, &s.data_name]) { + let steps = note_steps(¬e.text); + assert!(!steps.is_empty(), "{}", note.text); + let p = pool(&format!("{base}/{database}"), 1).await; + for (who, statement) in steps { + assert_eq!(who, "a superuser", "{}", note.text); + sqlx::raw_sql(&statement) + .execute(&p) + .await + .unwrap_or_else(|e| panic!("{statement}: {e}")); + } + p.close().await; + } + assert_eq!(status_notes(&target).await.expect("status notes"), []); + migrate(&target, "test", Duration::from_secs(5)) + .await + .expect("the role owns every object now"); + for database in [&s.catalog_name, &s.data_name] { + let p = pool(&format!("{base}/{database}"), 1).await; + let _ = sqlx::query(&format!("DROP OWNED BY \"{role}\"")) + .execute(&p) + .await; + p.close().await; + } + drop(s); + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{role}\"")) + .execute(&admin) + .await; + admin.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_owners.rs b/crates/storage-postgres/tests/migrations_owners.rs new file mode 100644 index 000000000..eb9836fea --- /dev/null +++ b/crates/storage-postgres/tests/migrations_owners.rs @@ -0,0 +1,297 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The ownership note where no superuser runs its statements, as on a +//! managed service; the notes after a restore that left the application +//! role unable to read the catalog's settings, which never name a data +//! database the role did not give; and the lock wait's description of a +//! session whose details the role may not see. See `migrations_support` +//! for the database these tests need. + +mod migrations_support; + +use std::time::Duration; + +use extenddb_storage::migrations::{CatalogMigrations, MigrationId, Target}; +use extenddb_storage_postgres::migrations::{ + MigrationLock, PostgresCatalogMigrations, migrate, status_notes, +}; +use migrations_support::{Scratch, base_conn, note_steps, pool, skip}; +use sqlx::PgPool; + +/// Run `sql` on `url`, panicking with it on an error. +async fn run(url: &str, sql: &str) { + let p = pool(url, 1).await; + sqlx::raw_sql(sql) + .execute(&p) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + p.close().await; +} + +/// `postgresql://:@/`. +fn as_role(base: &str, role: &str, password: &str, database: &str) -> String { + let host = base.rsplit('@').next().expect("a host in the base URL"); + format!("postgresql://{role}:{password}@{host}/{database}") +} + +async fn drop_roles(admin: &PgPool, roles: &[&str]) { + for role in roles { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{role}\"")) + .execute(admin) + .await; + } +} + +/// The objects belong to an owner that is not a superuser, and the +/// application role was created by an administrator that is not one +/// either: each statement the notes print, run as printed by the role the +/// note names for it, gives the objects to the application role, and the +/// run then applies every migration. +#[tokio::test] +async fn an_owner_that_is_not_a_superuser_runs_the_printed_statements() { + let Some(base) = base_conn() else { + return skip("owner without a superuser"); + }; + let s = Scratch::new(&base, "pgm_nosuper").await; + let (adm, app, dba) = ( + format!("{}_adm", s.catalog_name), + format!("{}_app", s.catalog_name), + format!("{}_dba", s.catalog_name), + ); + let admin = pool(&format!("{base}/postgres"), 1).await; + // An administrator gives only the attributes it has (PostgreSQL 16), so + // it has CREATEDB to create an owner that has it. + run( + &format!("{base}/postgres"), + &format!("CREATE ROLE \"{adm}\" LOGIN CREATEROLE CREATEDB PASSWORD 'adm-test'"), + ) + .await; + let adm_url = as_role(&base, &adm, "adm-test", "postgres"); + run( + &adm_url, + &format!( + "CREATE ROLE \"{app}\" LOGIN PASSWORD 'owner-test'; \ + CREATE ROLE \"{dba}\" LOGIN CREATEDB PASSWORD 'dba-test'" + ), + ) + .await; + // The owner owns both databases, as the administrator that restored + // them does, and loads the released catalog itself. + for database in [&s.catalog_name, &s.data_name] { + run( + &format!("{base}/postgres"), + &format!("ALTER DATABASE \"{database}\" OWNER TO \"{dba}\""), + ) + .await; + } + let dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/released/0.1.14"); + for (file, database) in [("catalog.sql", &s.catalog_name), ("data.sql", &s.data_name)] { + let sql = std::fs::read_to_string(dir.join(file)).expect("fixture"); + run(&as_role(&base, &dba, "dba-test", database), &sql).await; + run( + &as_role(&base, &dba, "dba-test", database), + &format!( + "GRANT ALL ON ALL TABLES IN SCHEMA public TO \"{app}\"; \ + GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO \"{app}\"" + ), + ) + .await; + } + let data_url = as_role(&base, &app, "owner-test", &s.data_name); + run( + &as_role(&base, &dba, "dba-test", &s.catalog_name), + &format!( + "UPDATE settings SET value = '{data_url}' WHERE key = \ + 'data_database_connection_string'; UPDATE settings SET value = '{}' WHERE key = \ + 'data_database_name'", + s.data_name + ), + ) + .await; + let mut target = s.target().with_data_connection_row(data_url); + target.catalog = target.catalog.username(&app).password("owner-test"); + let notes = status_notes(&target).await.expect("status notes"); + assert_eq!(notes.len(), 2, "{notes:?}"); + for (note, database) in notes.iter().zip([&s.catalog_name, &s.data_name]) { + let steps = note_steps(¬e.text); + let order: Vec<&str> = steps + .iter() + .map(|(_, statement)| statement.split(' ').next().unwrap_or("")) + .collect(); + assert_eq!(order, ["GRANT", "GRANT", "DO", "REVOKE"], "{}", note.text); + assert_eq!(steps[3].1, format!("REVOKE \"{app}\" FROM \"{dba}\";")); + assert_eq!(steps[0].1, format!("GRANT \"{app}\" TO \"{dba}\";")); + assert_eq!( + steps[1].1, + format!("GRANT CREATE ON SCHEMA public TO \"{app}\";") + ); + for (who, statement) in &steps { + let url = if who.starts_with("an administrator that may grant") { + as_role(&base, &adm, "adm-test", database) + } else if who.starts_with(&format!("{dba},")) { + as_role(&base, &dba, "dba-test", database) + } else { + panic!( + "a statement for {who}, where no superuser runs them: {}", + note.text + ); + }; + run(&url, statement).await; + } + } + assert_eq!(status_notes(&target).await.expect("status notes"), []); + migrate(&target, "test", Duration::from_secs(5)) + .await + .expect("the role owns every object now"); + let owner: String = sqlx::query_scalar( + "SELECT pg_get_userbyid(datdba)::text FROM pg_database WHERE datname = $1", + ) + .bind(&s.catalog_name) + .fetch_one(&admin) + .await + .unwrap(); + assert_eq!(owner, dba, "the databases keep their owner"); + let member: bool = sqlx::query_scalar("SELECT pg_has_role($1::name, $2::name, 'MEMBER')") + .bind(&dba) + .bind(&app) + .fetch_one(&admin) + .await + .unwrap(); + assert!(!member, "the last step takes back the owner's membership"); + for database in [&s.catalog_name, &s.data_name] { + run( + &format!("{base}/{database}"), + &format!("DROP OWNED BY \"{app}\""), + ) + .await; + } + drop(s); + drop_roles(&admin, &[&app, &dba, &adm]).await; + admin.close().await; +} + +/// A restore that left the application role unable to read the catalog's +/// settings, where the data database is not named after the catalog +/// database and a database of the name derived from the catalog's belongs +/// to another application: the notes name the catalog database and say how +/// to see the data database's note, and never name the other application's +/// database; with the data database given, both notes are printed. +#[tokio::test] +async fn a_restore_without_grants_names_no_guessed_data_database() { + let Some(base) = base_conn() else { + return skip("restore without grants"); + }; + let s = Scratch::with_names(&base, "pgm_nogrant", "_catalog", "_items", &[""]).await; + s.load("0.1.14").await; + let other = s.catalog_name.trim_end_matches("_catalog").to_owned(); + run( + &format!("{base}/{other}"), + "CREATE TABLE orders (id int PRIMARY KEY)", + ) + .await; + let app = format!("{other}_app"); + let admin = pool(&format!("{base}/postgres"), 1).await; + run( + &format!("{base}/postgres"), + &format!("CREATE ROLE \"{app}\" LOGIN PASSWORD 'owner-test'"), + ) + .await; + let mut target = Target::at(as_role(&base, &app, "owner-test", &s.catalog_name).as_str()); + let denied = PostgresCatalogMigrations + .status(&target) + .await + .expect_err("the role cannot read the settings"); + let text = denied.to_string(); + let notes: Vec<&str> = text.lines().filter(|l| l.starts_with("Note: ")).collect(); + assert!( + text.contains("permission denied for table settings"), + "{text}" + ); + assert_eq!(notes.len(), 2, "{text}"); + assert!( + notes[0].contains(&format!("catalog database {}:", s.catalog_name)), + "{text}" + ); + assert!( + notes[1].contains("which the application role cannot read") + && notes[1].contains("--data-db "), + "{text}" + ); + assert!( + !text.contains(&format!("database {other}:")) && !text.contains("orders"), + "a note names the database of the derived name: {text}" + ); + target.data_store = Some(s.data_name.clone()); + let denied = PostgresCatalogMigrations + .status(&target) + .await + .expect_err("the role still cannot read the settings"); + let text = denied.to_string(); + let notes: Vec<&str> = text.lines().filter(|l| l.starts_with("Note: ")).collect(); + assert_eq!(notes.len(), 2, "{text}"); + assert!( + notes[0].contains(&format!("catalog database {}:", s.catalog_name)) + && notes[1].contains(&format!("data database {}:", s.data_name)), + "{text}" + ); + assert!( + !text.contains("--data-db ") && !text.contains("orders"), + "{text}" + ); + drop(s); + drop_roles(&admin, &[&app]).await; + admin.close().await; +} + +/// A session of another role holds the migration lock: the wait names its +/// pid and role, and says its address and connection time are hidden from +/// this role, instead of printing them empty. +#[tokio::test] +async fn a_lock_held_by_another_role_names_that_role() { + let Some(base) = base_conn() else { + return skip("hidden lock holder"); + }; + let s = Scratch::new(&base, "pgm_hidden").await; + s.migrate_through(MigrationId::LATEST).await; + let (holder, app) = ( + format!("{}_holder", s.catalog_name), + format!("{}_app", s.catalog_name), + ); + let admin = pool(&format!("{base}/postgres"), 1).await; + run( + &format!("{base}/postgres"), + &format!( + "CREATE ROLE \"{holder}\" LOGIN PASSWORD 'holder-test'; \ + CREATE ROLE \"{app}\" LOGIN PASSWORD 'owner-test'" + ), + ) + .await; + let held = pool(&as_role(&base, &holder, "holder-test", &s.catalog_name), 1).await; + let mut tx = held.begin().await.unwrap(); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *tx) + .await + .unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(4539458, 1)") + .execute(&mut *tx) + .await + .unwrap(); + let catalog = s.options().username(&app).password("owner-test"); + let err = match MigrationLock::acquire(&catalog, Duration::from_secs(1)).await { + Ok(_) => panic!("the lock is held by another session"), + Err(e) => e.to_string(), + }; + assert!( + err.contains(&format!("pid {pid}, a session of the role {holder}")) + && err.contains("cannot see") + && !err.contains("connected )"), + "{err}" + ); + tx.rollback().await.unwrap(); + held.close().await; + drop(s); + drop_roles(&admin, &[&holder, &app]).await; + admin.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_released.rs b/crates/storage-postgres/tests/migrations_released.rs new file mode 100644 index 000000000..297d47b60 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_released.rs @@ -0,0 +1,319 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! Every released PostgreSQL state, loaded from its committed fixture, +//! upgraded by this binary, and compared with a fresh install seeded the same +//! way: the catalog and data objects, the per-table objects with ids mapped +//! to names, and the settings rows, against the accepted differences below. +//! The rows every table held are still there. +//! +//! Accepted differences, per state: +//! - key columns of tables created before v0.1.13 keep the database +//! collation, where CreateTable now declares `COLLATE "C"` (filed); +//! - the `control_plane_delay_seconds` row and the propagation delay row the +//! release seeded stay, under the legacy name `gsi_propagation_delay_ms` +//! on catalogs created before v0.1.7; +//! - no `default_account_id` on catalogs created before v0.1.3. +//! +//! The vector states need a server with pgvector: +//! `EXTENDDB_TEST_PGVECTOR_CONNECTION_STRING`, or the main server when it has +//! the package. See `migrations_support` for the rest. + +mod migrations_support; + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use extenddb_storage::migrations::{CatalogStatus, MigrationId}; +use extenddb_storage_postgres::migrations::{compare, migrate}; +use migrations_support::{ + LATEST, Scratch, base_conn, describe_pool, has_pgvector, id_names, ledger, legacy_rows, + per_table, seed_fresh, settings, shared, skip, skip_without_pgvector, +}; +use sqlx::PgPool; + +const RELEASE: &str = "1.0.0-test"; + +/// What a released state's fixture holds. +struct State { + name: &'static str, + /// Built from a tag's SQL files, with no table a binary created. + catalog_only: bool, + vector: bool, + /// Created by a release before v0.1.13: key columns on the database + /// collation. + database_collation: bool, + /// The release seeded `gsi_propagation_delay_ms` (before v0.1.7). + legacy_delay_key: bool, + /// `init` wrote `default_account_id` (v0.1.3 on). + default_account: bool, +} + +const fn image(name: &'static str, before_13: bool, before_7: bool, vector: bool) -> State { + State { + name, + catalog_only: false, + vector, + database_collation: before_13, + legacy_delay_key: before_7, + default_account: true, + } +} + +async fn row_counts(s: &Scratch) -> BTreeMap { + let (catalog, data) = (s.catalog().await, s.data().await); + let mut out = BTreeMap::new(); + for table in [ + "accounts", + "tables", + "indexes", + "iam_users", + "iam_policies", + "access_keys", + "admin_users", + "backups", + "backup_items", + "tags", + ] { + out.insert( + format!("catalog {table}"), + count_rows(&catalog, table).await, + ); + } + for table in ["stream_shards", "stream_records", "gsi_pending"] { + out.insert(format!("data {table}"), count_rows(&data, table).await); + } + let names = id_names(&catalog).await; + let per_table: Vec = sqlx::query_scalar( + "SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND c.relkind = 'r' AND c.relname LIKE '\\_ddb\\_%'", + ) + .fetch_all(&data) + .await + .unwrap(); + for table in per_table { + let mut label = table.clone(); + for (id, name) in &names { + label = label.replace(id.as_str(), name); + } + out.insert(format!("items {label}"), count_rows(&data, &table).await); + } + catalog.close().await; + data.close().await; + out +} + +async fn count_rows(p: &PgPool, table: &str) -> i64 { + sqlx::query_scalar(&format!("SELECT count(*) FROM \"{table}\"")) + .fetch_one(p) + .await + .unwrap_or(-1) +} + +async fn upgrade_and_compare(state: &State) { + let Some(mut base) = base_conn() else { + return skip(state.name); + }; + if state.vector && !has_pgvector(&base).await { + match std::env::var("EXTENDDB_TEST_PGVECTOR_CONNECTION_STRING") { + Ok(url) if !url.trim().is_empty() => base = url.trim_end_matches('/').to_owned(), + _ => return skip_without_pgvector(state.name), + } + } + let s = Scratch::new(&base, "pgm_rel").await; + s.load(state.name).await; + let rows_before = row_counts(&s).await; + let CatalogStatus::Behind { version: None, .. } = s.status().await else { + panic!( + "{}: a released catalog is behind at migration 1", + state.name + ); + }; + let report = migrate(&s.target(), RELEASE, Duration::from_secs(5)) + .await + .unwrap_or_else(|e| panic!("{}: {e}", state.name)); + assert_eq!(report.to, Some(LATEST), "{}", state.name); + assert_eq!( + s.status().await, + CatalogStatus::Current { version: LATEST }, + "{}", + state.name + ); + // A table the release did not have (-1) is created empty. + let rows_after: BTreeMap = row_counts(&s) + .await + .into_iter() + .filter(|(k, _)| rows_before.get(k).is_some_and(|n| *n >= 0)) + .collect(); + let rows_before: BTreeMap = + rows_before.into_iter().filter(|(_, n)| *n >= 0).collect(); + assert_eq!(rows_after, rows_before, "{}: rows changed", state.name); + + let fresh = Scratch::new(&base, "pgm_relf").await; + migrate(&fresh.target(), RELEASE, Duration::from_secs(5)) + .await + .expect("fresh install"); + if !state.catalog_only { + seed_fresh(&fresh, state.vector).await; + } + let (up_c, up_d, fr_c, fr_d) = ( + s.catalog().await, + s.data().await, + fresh.catalog().await, + fresh.data().await, + ); + let (up_cat, up_data) = (describe_pool(&up_c).await, describe_pool(&up_d).await); + let (fr_cat, fr_data) = (describe_pool(&fr_c).await, describe_pool(&fr_d).await); + for (what, up, fr) in [("catalog", &up_cat, &fr_cat), ("data", &up_data, &fr_data)] { + let (a, notes) = compare(&shared(fr), &shared(up)); + let (b, _) = compare(&shared(up), &shared(fr)); + assert!( + a.is_empty() && b.is_empty() && notes.is_empty(), + "{} {what}: {a:?} {b:?} {notes:?}", + state.name + ); + } + if !state.catalog_only { + let up_tables = per_table(&up_data, &id_names(&up_c).await); + let mut fr_tables = per_table(&fr_data, &id_names(&fr_c).await); + if state.database_collation { + fr_tables = fr_tables + .into_iter() + .map(|o| o.replace("collation: Some(\"C\")", "collation: None")) + .collect(); + } + let only_up: BTreeSet<_> = up_tables.difference(&fr_tables).collect(); + let only_fresh: BTreeSet<_> = fr_tables.difference(&up_tables).collect(); + assert!( + only_up.is_empty() && only_fresh.is_empty(), + "{}: per-table objects differ\nonly upgraded: {only_up:#?}\nonly fresh: {only_fresh:#?}", + state.name + ); + } + + let up_settings = settings(&up_c).await; + let mut expected: BTreeSet = settings(&fr_c).await.into_keys().collect(); + // Rows the test wrote to seed the fresh install, not the install. + expected.remove("control_plane_delay_seconds"); + expected.remove("index_propagation_delay_ms"); + // Rows `init` writes after the schema, with values of its own. + expected.insert("encryption_key".to_owned()); + if state.default_account { + expected.insert("default_account_id".to_owned()); + } + // The rows the release seeded, which stay. + expected.insert("control_plane_delay_seconds".to_owned()); + let delay_key = if state.legacy_delay_key { + "gsi_propagation_delay_ms" + } else { + "index_propagation_delay_ms" + }; + expected.insert(delay_key.to_owned()); + let keys: BTreeSet = up_settings.keys().cloned().collect(); + assert_eq!(keys, expected, "{}: settings keys", state.name); + assert_eq!( + up_settings["catalog_version"], + LATEST.to_string(), + "{}", + state.name + ); + assert_eq!( + up_settings["control_plane_delay_seconds"], "0.25", + "{}", + state.name + ); + assert_eq!(up_settings[delay_key], "10", "{}", state.name); + assert_eq!( + up_settings["data_database_name"], s.data_name, + "{}", + state.name + ); + assert_eq!(legacy_rows(&up_c).await.len(), 3, "{}", state.name); + assert_eq!(legacy_rows(&up_d).await.len(), 5, "{}", state.name); + assert_eq!( + ledger(&up_c).await.len(), + MigrationId::ALL.len(), + "{}", + state.name + ); + for p in [up_c, up_d, fr_c, fr_d] { + p.close().await; + } +} + +#[tokio::test] +async fn released_0_1_5() { + upgrade_and_compare(&image("0.1.5", true, true, false)).await; +} + +#[tokio::test] +async fn released_0_1_6() { + upgrade_and_compare(&image("0.1.6", true, true, false)).await; +} + +#[tokio::test] +async fn released_0_1_8() { + upgrade_and_compare(&image("0.1.8", true, false, false)).await; +} + +#[tokio::test] +async fn released_0_1_10() { + upgrade_and_compare(&image("0.1.10", true, false, false)).await; +} + +#[tokio::test] +async fn released_0_1_11() { + upgrade_and_compare(&image("0.1.11", true, false, false)).await; +} + +#[tokio::test] +async fn released_0_1_12() { + upgrade_and_compare(&image("0.1.12", true, false, false)).await; +} + +#[tokio::test] +async fn released_0_1_13() { + upgrade_and_compare(&image("0.1.13", false, false, false)).await; +} + +#[tokio::test] +async fn released_0_1_14() { + upgrade_and_compare(&image("0.1.14", false, false, false)).await; +} + +#[tokio::test] +async fn released_pgvector_0_1_11() { + upgrade_and_compare(&image("pv-0.1.11", true, false, true)).await; +} + +#[tokio::test] +async fn released_pgvector_0_1_14() { + upgrade_and_compare(&image("pv-0.1.14", false, false, true)).await; +} + +/// v0.1.0 lacks the three `tables` columns CreateTable writes (D1) and keeps +/// no data ledger. +#[tokio::test] +async fn tag_v0_1_0_catalog_only() { + upgrade_and_compare(&State { + name: "v0.1.0", + catalog_only: true, + vector: false, + database_collation: true, + legacy_delay_key: true, + default_account: false, + }) + .await; +} + +#[tokio::test] +async fn tag_v0_1_2_catalog_only() { + upgrade_and_compare(&State { + name: "v0.1.2", + catalog_only: true, + vector: false, + database_collation: true, + legacy_delay_key: true, + default_account: false, + }) + .await; +} diff --git a/crates/storage-postgres/tests/migrations_runs.rs b/crates/storage-postgres/tests/migrations_runs.rs new file mode 100644 index 000000000..ffd30fca1 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_runs.rs @@ -0,0 +1,398 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! The runs a fresh install gets after `init`: `--reconverge` on a catalog +//! that a later migration changed, `--forget`, a ledger that cannot be read, +//! the data database the server serves, a migrator that finds an earlier +//! migrator's statement still running, a zero lock timeout, and one timeout +//! across every wait of a run. See `migrations_support` for the database +//! these tests need. + +mod migrations_support; + +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, MigrationError, MigrationId, Runner, StoreKind, Target, +}; +use extenddb_storage_postgres::migrations::{ + MigrationLock, PostgresCatalogMigrations, PostgresMigrations, PostgresTarget, + ServerRegistration, ServingRegistration, catalog_status, migrate, +}; +use migrations_support::{LATEST, Scratch, base_conn, count, ids_in, ledger, pool, skip}; +use sqlx::PgPool; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; + +const RELEASE: &str = "1.0.0-test"; + +/// A scratch pair with a fresh install, as `init` leaves it. +async fn installed(base: &str, prefix: &str) -> Scratch { + let s = Scratch::new(base, prefix).await; + s.migrate_through(MigrationId::LATEST).await; + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + s +} + +/// A migration after migration 1 may change a table migration 1 created; a +/// `--reconverge` runs migration 1 again over it, and its check must not +/// refuse what the later migration added. +#[tokio::test] +async fn a_reconverge_keeps_what_a_later_migration_added() { + let Some(base) = base_conn() else { + return skip("reconverge"); + }; + let s = installed(&base, "pgm_reconv").await; + let catalog = s.catalog().await; + sqlx::query("ALTER TABLE settings ADD COLUMN note TEXT") + .execute(&catalog) + .await + .unwrap(); + sqlx::query("UPDATE settings SET note = 'kept' WHERE key = 'catalog_version'") + .execute(&catalog) + .await + .unwrap(); + let rows_before = ledger(&catalog).await; + let m = PostgresMigrations::open_unlocked(&s.target()) + .await + .unwrap(); + let report = Runner::new(&m, RELEASE).reconverge().await; + m.close().await; + let report = report.expect("reconverge on a catalog a later migration changed"); + assert_eq!(report.to, Some(LATEST)); + let note: Option = + sqlx::query_scalar("SELECT note FROM settings WHERE key = 'catalog_version'") + .fetch_one(&catalog) + .await + .unwrap(); + assert_eq!(note.as_deref(), Some("kept")); + assert_eq!(ledger(&catalog).await, rows_before); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + catalog.close().await; +} + +/// `forget` deletes the rows of the id it is given in every store that +/// records it, and the next run applies that id again. +#[tokio::test] +async fn forget_removes_one_migration_and_the_next_run_applies_it() { + let Some(base) = base_conn() else { + return skip("forget"); + }; + let s = installed(&base, "pgm_forget").await; + let lock = MigrationLock::acquire(&s.options(), Duration::from_secs(5)) + .await + .unwrap(); + let m = PostgresMigrations::open(&s.target(), &lock).await.unwrap(); + let forgot = Runner::new(&m, RELEASE).forget(LATEST).await; + m.close().await; + lock.release().await; + forgot.expect("forget the latest migration"); + let latest = i32::try_from(LATEST).unwrap(); + let (catalog, data) = (s.catalog().await, s.data().await); + for (pool, kind) in [(&catalog, StoreKind::Catalog), (&data, StoreKind::Data)] { + let expected: Vec = ids_in(kind) + .into_iter() + .filter(|id| *id != latest) + .collect(); + let ids: Vec = ledger(pool).await.iter().map(|r| r.0).collect(); + assert_eq!(ids, expected, "{kind:?}"); + } + let CatalogStatus::Behind { version, pending } = s.status().await else { + panic!("a forgotten migration is pending"); + }; + assert_eq!(version, Some(LATEST - 1)); + let mut pending: Vec<(u32, StoreKind)> = pending.iter().map(|p| (p.id, p.store)).collect(); + pending.sort(); + let expected: Vec<(u32, StoreKind)> = [StoreKind::Catalog, StoreKind::Data] + .into_iter() + .filter(|kind| ids_in(*kind).contains(&latest)) + .map(|kind| (LATEST, kind)) + .collect(); + assert_eq!(pending, expected); + let report = s.migrate_through(MigrationId::LATEST).await; + assert_eq!(report.applied.len(), expected.len()); + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + for p in [catalog, data] { + p.close().await; + } +} + +/// A ledger the status cannot read is an error naming it, never a catalog +/// read as behind. +#[tokio::test] +async fn a_ledger_that_cannot_be_read_is_an_error() { + let Some(base) = base_conn() else { + return skip("unreadable ledger"); + }; + let s = installed(&base, "pgm_ledger").await; + let data = s.data().await; + sqlx::query("ALTER TABLE extenddb_migrations RENAME COLUMN checksum TO checksum_before") + .execute(&data) + .await + .unwrap(); + let err = catalog_status(&s.target()) + .await + .expect_err("an unreadable ledger is an error"); + assert!(err.to_string().contains("extenddb_migrations"), "{err}"); + data.close().await; +} + +/// The data database the runner and the status use is the one the +/// connection string row names, which the server serves; a target naming +/// another one is refused unless the command line gave it. +#[tokio::test] +async fn the_status_refuses_when_the_two_data_rows_name_different_databases() { + let Some(base) = base_conn() else { + return skip("data rows"); + }; + let s = installed(&base, "pgm_rows").await; + let other = PostgresTarget::new(s.options(), format!("{}_other", s.data_name)); + let err = catalog_status(&other) + .await + .expect_err("the rows disagree with the target"); + let message = err.to_string(); + assert!( + message.contains(&s.data_name) && message.contains("data_database_connection_string"), + "{message}" + ); + assert_eq!(err.exit_code(), 1, "{message}"); + // The row's database is the one the status reads, whatever credentials + // the catalog was reached with. + assert_eq!( + catalog_status(&PostgresTarget::new(s.options(), &s.data_name)) + .await + .unwrap(), + CatalogStatus::Current { version: LATEST } + ); +} + +/// A session named as another runner's, running a statement in the +/// database at `url`: what a migrator that died while a statement ran +/// leaves behind. +async fn earlier_runner_statement(s: &Scratch, url: &str, seconds: u32) -> (PgPool, i32) { + let options: PgConnectOptions = url.parse().unwrap(); + let pool = PgPoolOptions::new() + .max_connections(1) + .connect_with(options.application_name("extenddb migrate run 0123456789ab")) + .await + .unwrap(); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + let running = pool.clone(); + tokio::spawn(async move { + let _ = sqlx::query(&format!("SELECT pg_sleep({seconds})")) + .execute(&running) + .await; + }); + let admin = pool_for_admin(s).await; + for _ in 0..200 { + let active: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1 AND state = 'active')", + ) + .bind(pid) + .fetch_one(&admin) + .await + .unwrap(); + if active { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + admin.close().await; + (pool, pid) +} + +async fn pool_for_admin(s: &Scratch) -> PgPool { + pool(&format!("{}/postgres", s.base), 1).await +} + +#[tokio::test] +async fn a_migrator_waits_for_an_earlier_migrators_statement_to_end() { + let Some(base) = base_conn() else { + return skip("earlier migrator"); + }; + let s = Scratch::new(&base, "pgm_dead").await; + s.load("0.1.14").await; + let (sleeper, _) = earlier_runner_statement(&s, &s.data_url(), 3).await; + let started = Instant::now(); + let report = migrate(&s.target(), RELEASE, Duration::from_secs(30)) + .await + .expect("migrate once the earlier statement ended"); + assert!( + started.elapsed() >= Duration::from_millis(2500), + "migrate ran beside the earlier statement ({:?})", + started.elapsed() + ); + assert_eq!(report.to, Some(LATEST)); + sleeper.close().await; + + // The same for a statement in the catalog database, which a migrator + // that died in a catalog body leaves behind. + let (sleeper, _) = earlier_runner_statement(&s, &s.catalog_url(), 3).await; + let started = Instant::now(); + migrate(&s.target(), RELEASE, Duration::from_secs(30)) + .await + .expect("migrate once the earlier statement ended"); + assert!( + started.elapsed() >= Duration::from_millis(2500), + "migrate ran beside the earlier statement in the catalog database ({:?})", + started.elapsed() + ); + sleeper.close().await; + + // The wait is bounded by the lock timeout, and names the session and + // the statement that ends it, with no advice to wait for it. + let (sleeper, pid) = earlier_runner_statement(&s, &s.data_url(), 60).await; + let err = migrate(&s.target(), RELEASE, Duration::from_secs(1)) + .await + .expect_err("the earlier statement outlasts the lock timeout"); + assert!(matches!(err, MigrationError::Blocked(_)), "{err}"); + assert!(err.to_string().contains("pg_cancel_backend"), "{err}"); + assert_eq!(err.exit_code(), 6); + assert!(err.to_string().contains(&format!("pid {pid}")), "{err}"); + let admin = pool_for_admin(&s).await; + sqlx::query("SELECT pg_cancel_backend($1)") + .bind(pid) + .execute(&admin) + .await + .unwrap(); + admin.close().await; + sleeper.close().await; +} + +/// `f`, which must end within 10 s, and how long it took. +async fn timed(f: impl std::future::Future) -> (T, Duration) { + let started = Instant::now(); + let out = tokio::time::timeout(Duration::from_secs(10), f) + .await + .expect("it ends within 10 s"); + (out, started.elapsed()) +} + +/// A zero lock timeout tries each wait once and exits 6 at once: while +/// another migrator holds the migration key, through the run `migrate +/// --yes` takes (the backend's lock) and through the library's run, and the +/// server's registration refuses at once; while a server holds the serving +/// key; and while an earlier migrator's statement runs. Nothing is written. +#[tokio::test] +async fn a_zero_lock_timeout_tries_once_and_exits_6_at_once() { + let Some(base) = base_conn() else { + return skip("zero lock timeout"); + }; + let s = Scratch::new(&base, "pgm_zero").await; + s.load("0.1.14").await; + let at_once = Duration::from_secs(2); + let catalog = s.catalog().await; + let mut migrator = catalog.begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(4539458, 1)") + .execute(&mut *migrator) + .await + .unwrap(); + let target = Target::at(s.catalog_url().as_str()); + let (locked, took) = timed(PostgresCatalogMigrations.lock(&target, Duration::ZERO)).await; + let Err(err) = locked else { + panic!("the command took the lock while a migrator held it"); + }; + assert_eq!(err.exit_code(), 6, "{err}"); + assert!( + err.to_string() + .contains("another migrator holds the migration lock"), + "{err}" + ); + assert!(took < at_once, "the command's lock took {took:?}"); + let (run, took) = timed(migrate(&s.target(), RELEASE, Duration::ZERO)).await; + let err = run.expect_err("a run while a migrator holds the lock"); + assert_eq!(err.exit_code(), 6, "{err}"); + assert!(took < at_once, "the run took {took:?}"); + let (registered, took) = timed(ServerRegistration::register(&s.catalog_url())).await; + let Err(err) = registered else { + panic!("a server registered while a migrator held the lock"); + }; + assert_eq!(err.exit_code(), 6, "{err}"); + assert!(took < at_once, "the registration took {took:?}"); + migrator.rollback().await.unwrap(); + + let serving = ServingRegistration::register(&s.options()).await.unwrap(); + let (run, took) = timed(migrate(&s.target(), RELEASE, Duration::ZERO)).await; + let err = run.expect_err("a run while a server holds its key"); + assert_eq!(err.exit_code(), 6, "{err}"); + assert!(err.to_string().contains("\"extenddb serve\""), "{err}"); + assert!(took < at_once, "the run took {took:?}"); + serving.release().await; + + let (sleeper, pid) = earlier_runner_statement(&s, &s.data_url(), 60).await; + let (run, took) = timed(migrate(&s.target(), RELEASE, Duration::ZERO)).await; + let err = run.expect_err("a run while an earlier migrator's statement runs"); + assert!(matches!(err, MigrationError::Blocked(_)), "{err}"); + assert!(err.to_string().contains(&format!("pid {pid}")), "{err}"); + assert!(took < at_once, "the run took {took:?}"); + let admin = pool_for_admin(&s).await; + sqlx::query("SELECT pg_cancel_backend($1)") + .bind(pid) + .execute(&admin) + .await + .unwrap(); + admin.close().await; + sleeper.close().await; + assert_eq!( + count( + &catalog, + "SELECT count(*) FROM pg_class WHERE relname = 'extenddb_migrations'" + ) + .await, + 0, + "a run at a zero lock timeout wrote the ledger" + ); + catalog.close().await; +} + +/// Every wait of a run shares the one lock timeout: another migrator holds +/// the migration key for 2 s, a server holds the serving key for 2 s more, +/// and an earlier migrator's statement outlasts the rest, so a run with a +/// 6 s timeout waits for all three and exits 6 within 7 s. +#[tokio::test] +async fn one_lock_timeout_bounds_every_wait_of_a_run() { + let Some(base) = base_conn() else { + return skip("one deadline"); + }; + let s = Scratch::new(&base, "pgm_deadline").await; + s.load("0.1.14").await; + let timeout = Duration::from_secs(6); + let (sleeper, pid) = earlier_runner_statement(&s, &s.data_url(), 60).await; + // The server registers first: it refuses to register beside a holder of + // the migration key. + let serving = ServingRegistration::register(&s.options()).await.unwrap(); + let catalog = s.catalog().await; + let mut migrator = catalog.begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(4539458, 1)") + .execute(&mut *migrator) + .await + .unwrap(); + let holders = tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2)).await; + migrator.rollback().await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + serving.release().await; + }); + let started = Instant::now(); + let err = migrate(&s.target(), RELEASE, timeout) + .await + .expect_err("the earlier statement outlasts the timeout"); + let took = started.elapsed(); + holders.await.unwrap(); + assert!(matches!(err, MigrationError::Blocked(_)), "{err}"); + assert!(err.to_string().contains(&format!("pid {pid}")), "{err}"); + assert!( + took >= Duration::from_secs(5) && took < timeout + Duration::from_secs(1), + "the run took {took:?} with a {timeout:?} lock timeout: {err}" + ); + let admin = pool_for_admin(&s).await; + sqlx::query("SELECT pg_cancel_backend($1)") + .bind(pid) + .execute(&admin) + .await + .unwrap(); + admin.close().await; + sleeper.close().await; + catalog.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_serving.rs b/crates/storage-postgres/tests/migrations_serving.rs new file mode 100644 index 000000000..e3697db51 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_serving.rs @@ -0,0 +1,311 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! The server's registration after the database ends its session, as a +//! restart, a failover or `pg_terminate_backend` does: the backend's side of +//! registering again. The loss is reported at the next renewal; an attempt +//! to register again takes the serving key on a new session when no +//! migration holds the catalog and its status is the one the server started +//! with, and is refused otherwise, leaving no key behind. See +//! `migrations_support` for the database these tests need. + +mod migrations_support; + +use std::time::Duration; + +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +use extenddb_storage::migrations::{CatalogStatus, MigrationId, Runner}; +use extenddb_storage_postgres::migrations::{ + MigrationLock, PostgresMigrations, ServerRegistration, migrate, +}; +use migrations_support::{LATEST, Scratch, base_conn, pool, skip}; +use sqlx::PgPool; + +const RELEASE: &str = "1.0.0-test"; + +/// The renewal interval of these tests, so a loss is reported within 1 s. +const KEEPALIVE: Duration = Duration::from_secs(1); + +/// A scratch pair with a fresh install, as `init` leaves it. +async fn installed(base: &str, prefix: &str) -> Scratch { + let s = Scratch::new(base, prefix).await; + s.migrate_through(MigrationId::LATEST).await; + assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); + s +} + +/// The sessions that hold the serving key on the scratch catalog, with +/// their `application_name`. +async fn serving_sessions(admin: &PgPool, s: &Scratch) -> Vec<(i32, String)> { + sqlx::query_as( + "SELECT l.pid, coalesce(a.application_name, '') FROM pg_locks l \ + JOIN pg_database d ON d.oid = l.database \ + LEFT JOIN pg_stat_activity a ON a.pid = l.pid \ + WHERE l.locktype = 'advisory' AND l.classid = 4539458 AND l.objid = 2 \ + AND l.objsubid = 2 AND l.mode = 'ShareLock' AND l.granted AND d.datname = $1 \ + ORDER BY l.pid", + ) + .bind(&s.catalog_name) + .fetch_all(admin) + .await + .unwrap() +} + +/// End the one session that holds the serving key, and return its pid. +async fn end_the_serving_session(admin: &PgPool, s: &Scratch) -> i32 { + let sessions = serving_sessions(admin, s).await; + assert_eq!(sessions.len(), 1, "{sessions:?}"); + let (pid, _) = sessions[0]; + let ended: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)") + .bind(pid) + .fetch_one(admin) + .await + .unwrap(); + assert!(ended, "pid {pid} was not ended"); + pid +} + +/// The reason [`ServingRegistrar::lost`] reports, within 10 s. +async fn lost(registration: &ServerRegistration) -> String { + tokio::time::timeout(Duration::from_secs(10), registration.lost()) + .await + .expect("the loss is reported within 10 s") +} + +/// The registration is reported lost at the next renewal after the database +/// ends its session, and is taken again on a new session, which excludes a +/// migration again; a second loss is handled the same way. +#[tokio::test] +async fn a_registration_the_database_ended_is_taken_again_on_a_new_session() { + let Some(base) = base_conn() else { + return skip("registered again"); + }; + let s = installed(&base, "pgs_again").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = ServerRegistration::register_with(&s.catalog_url(), KEEPALIVE) + .await + .expect("register"); + let mut ended = Vec::new(); + for round in 1..=2 { + ended.push(end_the_serving_session(&admin, &s).await); + let reason = lost(®istration).await; + assert!( + reason.contains("serving registration's connection was lost"), + "round {round}: {reason}" + ); + assert_eq!( + serving_sessions(&admin, &s).await, + [], + "round {round}: the database kept the key" + ); + assert_eq!( + registration.register_again().await, + Reregistration::Registered, + "round {round}" + ); + let sessions = serving_sessions(&admin, &s).await; + assert_eq!(sessions.len(), 1, "round {round}: {sessions:?}"); + assert_eq!(sessions[0].1, "extenddb serve", "round {round}"); + assert!( + !ended.contains(&sessions[0].0), + "round {round}: the key is on an ended session {sessions:?}" + ); + // The new registration excludes a migration, and is not reported + // lost over two renewals. + let err = migrate(&s.target(), RELEASE, Duration::from_secs(1)) + .await + .expect_err("a migration while the server is registered again"); + assert_eq!(err.exit_code(), 6, "round {round}: {err}"); + assert!( + err.to_string().contains("\"extenddb serve\""), + "round {round}: {err}" + ); + assert!( + tokio::time::timeout(Duration::from_millis(2500), registration.lost()) + .await + .is_err(), + "round {round}: the new registration was reported lost" + ); + } + admin.close().await; +} + +/// A migration that holds the catalog when the server tries again refuses +/// the attempt, which leaves no key behind: one that holds both keys, and one +/// of a release before 1.0 that holds the migration key alone. Once it has +/// finished without a change, the server registers again. +#[tokio::test] +async fn a_migration_that_holds_the_catalog_refuses_the_registration() { + let Some(base) = base_conn() else { + return skip("refused by a migration"); + }; + let s = installed(&base, "pgs_held").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = ServerRegistration::register_with(&s.catalog_url(), KEEPALIVE) + .await + .expect("register"); + end_the_serving_session(&admin, &s).await; + lost(®istration).await; + let lock = MigrationLock::acquire(&s.options(), Duration::from_secs(5)) + .await + .expect("the migration lock, once the server's key is gone"); + match registration.register_again().await { + Reregistration::Refused(why) => assert!( + why.contains("a migration is running on this catalog") && why.contains("pid "), + "{why}" + ), + other => panic!("registered beside a migration: {other:?}"), + } + assert_eq!(serving_sessions(&admin, &s).await, []); + lock.release().await; + + let catalog = s.catalog().await; + let mut legacy = catalog.begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(4539458, 1)") + .execute(&mut *legacy) + .await + .unwrap(); + match registration.register_again().await { + Reregistration::Refused(why) => { + assert!(why.contains("holds the migration lock"), "{why}"); + } + other => panic!("registered beside a migrator: {other:?}"), + } + assert_eq!(serving_sessions(&admin, &s).await, []); + legacy.rollback().await.unwrap(); + catalog.close().await; + + assert_eq!( + registration.register_again().await, + Reregistration::Registered + ); + assert_eq!(serving_sessions(&admin, &s).await.len(), 1); + admin.close().await; +} + +/// A catalog whose status changed while the server held no registration +/// refuses the attempt, naming both statuses; the key it took is released. +#[tokio::test] +async fn a_status_that_changed_while_unregistered_refuses_the_registration() { + let Some(base) = base_conn() else { + return skip("refused on a changed status"); + }; + let s = installed(&base, "pgs_status").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = ServerRegistration::register_with(&s.catalog_url(), KEEPALIVE) + .await + .expect("register"); + end_the_serving_session(&admin, &s).await; + lost(®istration).await; + // A migrator clears the latest migration in the meantime. + let lock = MigrationLock::acquire(&s.options(), Duration::from_secs(5)) + .await + .unwrap(); + let m = PostgresMigrations::open(&s.target(), &lock).await.unwrap(); + let forgot = Runner::new(&m, RELEASE).forget(LATEST).await; + m.close().await; + lock.release().await; + forgot.expect("forget the latest migration"); + match registration.register_again().await { + Reregistration::Refused(why) => assert!( + why.contains("status changed") + && why.contains(&format!("migration {LATEST}")) + && why.contains("start the server again"), + "{why}" + ), + other => panic!("registered on a changed catalog: {other:?}"), + } + assert_eq!(serving_sessions(&admin, &s).await, []); + admin.close().await; +} + +/// A database that accepts no connection makes the attempt unavailable, +/// not refused, so the server tries again; once it accepts them, the server +/// registers. +#[tokio::test] +async fn an_attempt_the_database_cannot_answer_is_unavailable() { + let Some(base) = base_conn() else { + return skip("unavailable attempt"); + }; + let s = installed(&base, "pgs_closed").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = ServerRegistration::register_with(&s.catalog_url(), KEEPALIVE) + .await + .expect("register"); + let allow = |allowed: bool| { + format!( + "ALTER DATABASE \"{}\" WITH ALLOW_CONNECTIONS {allowed}", + s.catalog_name + ) + }; + sqlx::query(&allow(false)).execute(&admin).await.unwrap(); + end_the_serving_session(&admin, &s).await; + lost(®istration).await; + let attempt = registration.register_again().await; + sqlx::query(&allow(true)).execute(&admin).await.unwrap(); + match attempt { + Reregistration::Unavailable(why) => assert!(why.contains("connect"), "{why}"), + other => panic!("an attempt the database cannot answer: {other:?}"), + } + assert_eq!( + registration.register_again().await, + Reregistration::Registered + ); + admin.close().await; +} + +/// The server drops an attempt at the bound of its window, and at +/// shutdown. An attempt dropped after it took the serving key, while its +/// status read waits on a lock, leaves no key held: a migration that then +/// asks for the catalog sees no server. +#[tokio::test] +async fn an_attempt_dropped_while_its_status_read_waits_leaves_no_key() { + let Some(base) = base_conn() else { + return skip("a dropped attempt"); + }; + let s = installed(&base, "pgm_drop").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = ServerRegistration::register_with(&s.catalog_url(), KEEPALIVE) + .await + .expect("register"); + end_the_serving_session(&admin, &s).await; + lost(®istration).await; + // The status read of the next attempt waits on this lock. + let catalog = s.catalog().await; + let mut blocker = catalog.begin().await.expect("begin"); + sqlx::query("LOCK TABLE extenddb_migrations IN ACCESS EXCLUSIVE MODE") + .execute(&mut *blocker) + .await + .expect("lock the ledger"); + let attempt = tokio::spawn(async move { + let answer = + tokio::time::timeout(Duration::from_secs(3), registration.register_again()).await; + (answer.is_err(), registration) + }); + let mut held_while_waiting = Vec::new(); + for _ in 0..20 { + held_while_waiting = serving_sessions(&admin, &s).await; + if !held_while_waiting.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let (dropped, _registration) = attempt.await.expect("the attempt's task"); + let since = std::time::Instant::now(); + let mut held = serving_sessions(&admin, &s).await; + while !held.is_empty() && since.elapsed() < Duration::from_secs(5) { + tokio::time::sleep(Duration::from_millis(50)).await; + held = serving_sessions(&admin, &s).await; + } + blocker.rollback().await.expect("release the ledger"); + catalog.close().await; + assert!( + dropped, + "the attempt finished within 3 s, so it was not dropped" + ); + assert!( + !held_while_waiting.is_empty(), + "the attempt never took the key, so its drop shows nothing" + ); + assert_eq!(held, [], "a dropped attempt left the serving key held"); + admin.close().await; +} diff --git a/crates/storage-postgres/tests/migrations_support/mod.rs b/crates/storage-postgres/tests/migrations_support/mod.rs new file mode 100644 index 000000000..31646f90e --- /dev/null +++ b/crates/storage-postgres/tests/migrations_support/mod.rs @@ -0,0 +1,568 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared by the migration integration tests: scratch database pairs, the +//! released fixtures, a fresh install seeded the way the fixtures were, and +//! the comparison of two installs with table and index ids mapped to names. +//! +//! Every test needs `EXTENDDB_TEST_PG_CONNECTION_STRING`, a base URL +//! `postgres://user:pass@host:port` with no database, for a role that may +//! create databases; without it every test reports a skip and passes. + +#![allow(dead_code)] + +pub mod proxy; + +use std::collections::{BTreeMap, BTreeSet}; + +use extenddb_core::types::CreateTableInput; +use extenddb_storage::migrations::{MigrationId, RunReport, Runner, StoreKind}; +use extenddb_storage::{MetadataEngine, TableEngine}; +use extenddb_storage_postgres::migrations::{ + Definitions, PostgresMigrations, PostgresTarget, describe, migration, +}; +use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +use sqlx::PgPool; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; + +pub const ACCOUNT: &str = "123456789012"; + +/// The highest migration this binary carries. Tests derive what a complete +/// run does from it and from this backend's table, so adding a migration +/// needs no edit here; a literal id in a test means that migration itself. +pub const LATEST: u32 = MigrationId::LATEST.number(); + +/// The kinds of store `id` records rows in, catalog first, as this backend's +/// table declares them and the runner applies them. +fn store_kinds(id: MigrationId) -> Vec { + migration(id).store_kinds() +} + +/// The ids a complete run records in a store of `kind`: every id in the +/// catalog, and in the data database the ids with a data body. +pub fn ids_in(kind: StoreKind) -> Vec { + MigrationId::ALL + .into_iter() + .filter(|id| store_kinds(*id).contains(&kind)) + .map(|id| i32::try_from(id.number()).expect("an id fits the ledger's INTEGER")) + .collect() +} + +/// The steps a run from an empty or a legacy catalog applies: one for each +/// id in each store it records rows in. +pub fn step_count() -> usize { + MigrationId::ALL + .into_iter() + .map(|id| store_kinds(id).len()) + .sum() +} + +pub fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; + (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) +} + +/// Whether a missing server fails the test instead of skipping it: set +/// `EXTENDDB_TEST_PG_REQUIRED` to any value but `0` in a job that must run +/// these tests. +pub fn required() -> bool { + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0") +} + +/// Report a skip, or fail when [`required`]. +pub fn skip(test: &str) { + assert!( + !required(), + "{test}: EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED \ + says this job must run the PostgreSQL tests" + ); + eprintln!("SKIP {test}: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); +} + +/// Report a skip for a test that needs pgvector on a server without it. +pub fn skip_without_pgvector(test: &str) { + eprintln!( + "SKIP {test}: the server has no pgvector package, and \ + EXTENDDB_TEST_PGVECTOR_CONNECTION_STRING names no server that has it" + ); +} + +/// Whether the test writes its expected files instead of comparing with +/// them: `EXTENDDB_BLESS_EXPECTED=1`. +pub fn bless() -> bool { + std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") +} + +/// Whether the server has the pgvector package, which the vector fixtures +/// need. +pub async fn has_pgvector(base: &str) -> bool { + let admin = pool(&format!("{base}/postgres"), 1).await; + let found: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector')", + ) + .fetch_one(&admin) + .await + .expect("read pg_available_extensions"); + admin.close().await; + found +} + +pub async fn pool(url: &str, size: u32) -> PgPool { + PgPoolOptions::new() + .max_connections(size) + .connect(url) + .await + .unwrap_or_else(|e| panic!("connect to {url}: {e}")) +} + +/// A catalog and a data database, dropped by [`Scratch::drop_all`] or, after +/// a failed assertion, by the guard. +pub struct Scratch { + pub base: String, + pub catalog_name: String, + pub data_name: String, + _guard: Guard, +} + +struct Guard { + base: String, + names: Vec, +} + +impl Drop for Guard { + fn drop(&mut self) { + let (base, names) = (self.base.clone(), self.names.clone()); + let _ = std::thread::spawn(move || { + let Ok(rt) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + rt.block_on(async { + if let Ok(admin) = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + { + for n in &names { + let _ = + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{n}\" WITH (FORCE)")) + .execute(&admin) + .await; + } + admin.close().await; + } + }); + }) + .join(); + } +} + +impl Scratch { + /// Two empty databases named `__c` and `_d`. + pub async fn new(base: &str, prefix: &str) -> Self { + Self::with_names(base, prefix, "_c", "_d", &[]).await + } + + /// Empty databases `` and ``, and one + /// `` for each of `others`, all dropped with the scratch; + /// `` is `_`. + pub async fn with_names( + base: &str, + prefix: &str, + catalog: &str, + data: &str, + others: &[&str], + ) -> Self { + let stem = format!( + "{prefix}_{}", + &uuid::Uuid::new_v4().simple().to_string()[..10] + ); + let (catalog_name, data_name) = (format!("{stem}{catalog}"), format!("{stem}{data}")); + let mut names = vec![catalog_name.clone(), data_name.clone()]; + names.extend(others.iter().map(|suffix| format!("{stem}{suffix}"))); + let admin = pool(&format!("{base}/postgres"), 1).await; + for n in &names { + sqlx::query(&format!("CREATE DATABASE \"{n}\"")) + .execute(&admin) + .await + .expect("create a scratch database"); + } + admin.close().await; + Self { + base: base.to_owned(), + _guard: Guard { + base: base.to_owned(), + names, + }, + catalog_name, + data_name, + } + } + + pub fn catalog_url(&self) -> String { + format!("{}/{}", self.base, self.catalog_name) + } + + pub fn data_url(&self) -> String { + format!("{}/{}", self.base, self.data_name) + } + + pub fn options(&self) -> PgConnectOptions { + self.catalog_url().parse().expect("a postgres URL") + } + + /// The runner's target, which writes the data connection rows as `init`. + pub fn target(&self) -> PostgresTarget { + PostgresTarget::new(self.options(), &self.data_name) + .with_data_connection_row(self.data_url()) + } + + pub async fn catalog(&self) -> PgPool { + pool(&self.catalog_url(), 2).await + } + + pub async fn data(&self) -> PgPool { + pool(&self.data_url(), 2).await + } + + /// Load a released fixture: `tests/fixtures/released//catalog.sql` + /// and `data.sql`, each on a connection of its own. + pub async fn load(&self, state: &str) { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/released") + .join(state); + for (file, url) in [ + ("catalog.sql", self.catalog_url()), + ("data.sql", self.data_url()), + ] { + let sql = std::fs::read_to_string(dir.join(file)) + .unwrap_or_else(|e| panic!("{state}/{file}: {e}")); + let p = pool(&url, 1).await; + sqlx::raw_sql(&sql) + .execute(&p) + .await + .unwrap_or_else(|e| panic!("load {state}/{file}: {e}")); + p.close().await; + } + // The fixture names the data database it was dumped from. + let catalog = self.catalog().await; + sqlx::query("UPDATE settings SET value = $1 WHERE key = 'data_database_connection_string'") + .bind(self.data_url()) + .execute(&catalog) + .await + .expect("point the catalog at the data database"); + sqlx::query("UPDATE settings SET value = $1 WHERE key = 'data_database_name'") + .bind(&self.data_name) + .execute(&catalog) + .await + .expect("name the data database"); + catalog.close().await; + } + + /// Run the migrations up to `id` with no lock, as a test drives them. + pub async fn migrate_through(&self, id: MigrationId) -> RunReport { + let m = PostgresMigrations::open_unlocked(&self.target()) + .await + .expect("open the runner"); + let report = Runner::new(&m, "test").migrate_through(id).await; + m.close().await; + report.unwrap_or_else(|e| panic!("migrate through {id}: {e}")) + } + + /// `(id, database)` of every step a run from an empty or a legacy + /// catalog applies, in the order the runner applies them: ids in order, + /// and for each id the catalog first. + pub fn all_steps(&self) -> Vec<(u32, String)> { + MigrationId::ALL + .into_iter() + .flat_map(|id| { + store_kinds(id).into_iter().map(move |kind| { + let database = match kind { + StoreKind::Catalog => self.catalog_name.clone(), + StoreKind::Data => self.data_name.clone(), + StoreKind::Account => panic!("PostgreSQL has no account store"), + }; + (id.number(), database) + }) + }) + .collect() + } + + pub async fn status(&self) -> extenddb_storage::migrations::CatalogStatus { + extenddb_storage_postgres::migrations::catalog_status(&self.target()) + .await + .expect("read the status") + } +} + +fn seed_input(json: serde_json::Value) -> CreateTableInput { + serde_json::from_value(json).expect("a CreateTable request") +} + +/// The tables the released fixtures were seeded with, created through the +/// engine on an install this binary migrated: seed_tbl with a sort key, a GSI, +/// an LSI, a stream and TTL, seed_simple, and with `vector` vec_tbl with a +/// vector index. Per-table objects come from these requests; items and +/// backups are rows, not objects. +pub async fn seed_fresh(s: &Scratch, vector: bool) { + let catalog = s.catalog().await; + for (key, value) in [ + ("control_plane_delay_seconds", "0"), + ("index_propagation_delay_ms", "0"), + ] { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ($1, $2) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(key) + .bind(value) + .execute(&catalog) + .await + .expect("set a delay"); + } + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, 'seed')") + .bind(ACCOUNT) + .execute(&catalog) + .await + .expect("the seed account"); + if vector { + // `migrate` makes this attempt on every run, beside the ledger. + let line = extenddb_storage_postgres::migrations::try_create_vector_extension( + &s.data_url().parse().expect("a postgres URL"), + None, + ) + .await; + assert!(line.starts_with("pgvector available"), "{line}"); + } + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: s.catalog_url(), + pool_size: 4, + max_item_size_bytes: 400_000, + }, + "us-east-1", + ) + .await + .expect("open the engine"); + let mut requests = vec![ + serde_json::json!({ + "TableName": "seed_tbl", + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "lsk", "AttributeType": "N"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + "GlobalSecondaryIndexes": [{"IndexName": "gsi1", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}}], + "LocalSecondaryIndexes": [{"IndexName": "lsi1", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "lsk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}}], + "BillingMode": "PAY_PER_REQUEST", + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"} + }), + serde_json::json!({ + "TableName": "seed_simple", + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "N"}], + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST" + }), + ]; + if vector { + requests.push(serde_json::json!({ + "TableName": "vec_tbl", + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{"IndexName": "vidx", "Dimensions": 4, "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, "Projection": {"ProjectionType": "ALL"}}] + })); + } + for request in requests { + engine + .create_table(ACCOUNT, seed_input(request)) + .await + .expect("CreateTable"); + } + engine + .update_ttl(ACCOUNT, "seed_tbl", "exp", true) + .await + .expect("UpdateTimeToLive"); + // The TTL worker builds the index in a server; here the engine does. + engine + .create_ttl_index(ACCOUNT, "seed_tbl", "exp") + .await + .expect("the TTL index"); + catalog.close().await; +} + +/// `_ddb_` names replaced with `T:
`, `I:
/` or +/// `V:
/` from the catalog's rows, longest id first. +pub async fn id_names(catalog: &PgPool) -> Vec<(String, String)> { + let mut names: Vec<(String, String)> = + sqlx::query_as("SELECT table_id, 'T:' || table_name FROM tables") + .fetch_all(catalog) + .await + .expect("read tables"); + names.extend( + sqlx::query_as::<_, (String, String)>( + "SELECT i.index_id, 'I:' || t.table_name || '/' || i.index_name \ + FROM indexes i JOIN tables t ON t.table_id = i.table_id", + ) + .fetch_all(catalog) + .await + .expect("read indexes"), + ); + names.extend( + sqlx::query_as::<_, (String, String)>( + "SELECT v.index_id, 'V:' || t.table_name || '/' || v.index_name \ + FROM vector_indexes v JOIN tables t ON t.table_id = v.table_id", + ) + .fetch_all(catalog) + .await + .unwrap_or_default(), + ); + names +} + +fn mapped(text: &str, names: &[(String, String)]) -> String { + let mut out = text.to_owned(); + for (id, name) in names { + out = out.replace(id.as_str(), name); + // PostgreSQL truncates generated index names to 63 bytes, cutting ids. + let short: String = id.replace('-', "_"); + out = out.replace(&short, name); + } + out +} + +/// A database's per-table objects with ids mapped: columns, constraints, and +/// indexes keyed by table and definition with the index's own name removed, +/// because PostgreSQL generates some of them and truncates the id inside. +pub fn per_table(defs: &Definitions, names: &[(String, String)]) -> BTreeSet { + let mut out = BTreeSet::new(); + for (key, column) in &defs.columns { + if key.starts_with("_ddb_") { + out.insert(format!("column {} {column:?}", mapped(key, names))); + } + } + for (key, definition) in &defs.constraints { + if key.starts_with("_ddb_") { + let constraint = key.split_once('.').map_or("", |(_, c)| c); + let table = key.split_once('.').map_or(key.as_str(), |(t, _)| t); + let kind = if constraint.ends_with("_pkey") { + "pkey" + } else { + constraint + }; + out.insert(format!( + "constraint {} {kind} {}", + mapped(table, names), + mapped(definition, names) + )); + } + } + for (name, index) in &defs.indexes { + if index.table.starts_with("_ddb_") { + let definition = + index + .definition + .replacen(&format!("INDEX \"{name}\" "), "INDEX ? ", 1); + let definition = definition.replacen(&format!("INDEX {name} "), "INDEX ? ", 1); + out.insert(format!( + "index {} {} valid={}", + mapped(&index.table, names), + mapped(&definition, names), + index.valid + )); + } + } + out +} + +/// The objects that are not per-table. +pub fn shared(defs: &Definitions) -> Definitions { + Definitions { + columns: defs + .columns + .iter() + .filter(|(k, _)| !k.starts_with("_ddb_")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + indexes: defs + .indexes + .iter() + .filter(|(_, i)| !i.table.starts_with("_ddb_")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + constraints: defs + .constraints + .iter() + .filter(|(k, _)| !k.starts_with("_ddb_")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + sequences: defs.sequences.clone(), + } +} + +pub async fn settings(catalog: &PgPool) -> BTreeMap { + sqlx::query_as::<_, (String, String)>("SELECT key, value FROM settings") + .fetch_all(catalog) + .await + .expect("read settings") + .into_iter() + .collect() +} + +pub async fn describe_pool(p: &PgPool) -> Definitions { + describe(p).await.expect("describe") +} + +/// `(id, name, outcome)` of a store's ledger. +pub async fn ledger(p: &PgPool) -> Vec<(i32, String, String)> { + sqlx::query_as("SELECT id, name, outcome FROM extenddb_migrations ORDER BY id") + .fetch_all(p) + .await + .unwrap_or_default() +} + +pub async fn legacy_rows(p: &PgPool) -> Vec { + sqlx::query_scalar("SELECT filename FROM schema_history ORDER BY filename") + .fetch_all(p) + .await + .unwrap_or_default() +} + +pub async fn count(p: &PgPool, sql: &str) -> i64 { + sqlx::query_scalar(sql) + .fetch_one(p) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")) +} + +/// The statements an ownership note prints, as (who runs it, statement), in +/// the order the note numbers them: `(1) as : (2) as ...`. +/// The last one ends with the note's line. +pub fn note_steps(note: &str) -> Vec<(String, String)> { + let mut steps = Vec::new(); + let Some(start) = note.find("(1) as ") else { + return steps; + }; + let mut rest = ¬e[start..]; + let mut n = 1; + while let Some(body) = rest.strip_prefix(&format!("({n}) as ")) { + let (who, after) = body.split_once(": ").expect("who runs the statement"); + let next = format!(" ({}) as ", n + 1); + let (statement, tail) = match after.find(&next) { + Some(i) => (&after[..i], &after[i + 1..]), + None => (after.lines().next().unwrap_or(after), ""), + }; + steps.push((who.to_owned(), statement.trim().to_owned())); + rest = tail; + n += 1; + } + steps +} diff --git a/crates/storage-postgres/tests/migrations_support/proxy.rs b/crates/storage-postgres/tests/migrations_support/proxy.rs new file mode 100644 index 000000000..8981a7f59 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_support/proxy.rs @@ -0,0 +1,90 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A TCP proxy to the test server that can stop forwarding while it keeps +//! both connections open, what a client sees when the database host is +//! paused or vanishes without closing the connection, and can close both +//! connections, what the database sees when the client process dies. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use sqlx::postgres::PgConnectOptions; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +use super::Scratch; + +pub struct Proxy { + pub port: u16, + /// While set, nothing is forwarded and both connections stay open. + pub paused: Arc, + /// Once set, every connection is closed on both sides within 50 ms. + pub cut: Arc, +} + +async fn pump( + mut from: tokio::net::tcp::OwnedReadHalf, + mut to: tokio::net::tcp::OwnedWriteHalf, + paused: Arc, + cut: Arc, +) { + let mut buf = vec![0_u8; 16 * 1024]; + loop { + // Returning drops both halves; once both pumps of a connection + // return, its two sockets are closed. + if cut.load(Ordering::SeqCst) { + return; + } + if paused.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + } + // A read that times out is dropped before it completes, which loses + // nothing on a TcpStream, so the pause is noticed within 50 ms. + match tokio::time::timeout(Duration::from_millis(50), from.read(&mut buf)).await { + Err(_) => {} + Ok(Ok(0) | Err(_)) => return, + Ok(Ok(n)) => { + if to.write_all(&buf[..n]).await.is_err() { + return; + } + } + } + } +} + +impl Proxy { + /// Listen on a loopback port and forward each connection to `upstream`. + pub async fn start(upstream: (String, u16)) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let paused = Arc::new(AtomicBool::new(false)); + let cut = Arc::new(AtomicBool::new(false)); + let (pause_flag, cut_flag) = (Arc::clone(&paused), Arc::clone(&cut)); + tokio::spawn(async move { + while let Ok((client, _)) = listener.accept().await { + let Ok(server) = TcpStream::connect((upstream.0.as_str(), upstream.1)).await else { + continue; + }; + let (client_read, client_write) = client.into_split(); + let (server_read, server_write) = server.into_split(); + for (from, to) in [(client_read, server_write), (server_read, client_write)] { + tokio::spawn(pump( + from, + to, + Arc::clone(&pause_flag), + Arc::clone(&cut_flag), + )); + } + } + }); + Self { port, paused, cut } + } + + /// The scratch catalog's options, through this proxy. + pub fn options(&self, s: &Scratch) -> PgConnectOptions { + s.options().host("127.0.0.1").port(self.port) + } +} diff --git a/crates/storage-postgres/tests/migrations_tag_blobs.rs b/crates/storage-postgres/tests/migrations_tag_blobs.rs new file mode 100644 index 000000000..b70a12d16 --- /dev/null +++ b/crates/storage-postgres/tests/migrations_tag_blobs.rs @@ -0,0 +1,51 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! Tags without a released fixture ship the same PostgreSQL schema files, +//! blob for blob, as the tag of a fixture that stands for them +//! (`tests/fixtures/released/README.md`). Needs `git` and the release tags +//! in the repository; without them it reports a skip and passes. + +use std::process::Command; + +/// `(tag without a fixture, tag whose fixture stands for it)`. +const SAME_FILES: &[(&str, &str)] = &[ + ("v0.1.1", "v0.1.0"), + ("v0.1.3", "v0.1.5"), + ("v0.1.4", "v0.1.5"), + ("v0.1.7", "v0.1.8"), + ("v0.1.9", "v0.1.8"), +]; + +/// The names and blob ids of the tag's schema files, or `None` when git or +/// the tag is not available. +fn schema_files(tag: &str) -> Option { + let root = concat!(env!("CARGO_MANIFEST_DIR"), "/../.."); + let out = Command::new("git") + .args([ + "-C", + root, + "ls-tree", + tag, + "--", + "crates/storage-postgres/migrations/", + "crates/storage-postgres/data_migrations/", + ]) + .output() + .ok()?; + let text = String::from_utf8(out.stdout).ok()?; + (out.status.success() && !text.trim().is_empty()).then_some(text) +} + +#[test] +fn tags_without_a_fixture_ship_the_schema_files_of_one_with_a_fixture() { + for (tag, stands_for) in SAME_FILES { + let (Some(files), Some(expected)) = (schema_files(tag), schema_files(stands_for)) else { + eprintln!("SKIP: git or the release tags {tag} and {stands_for} are not available"); + return; + }; + assert_eq!( + files, expected, + "{tag} and {stands_for} ship different schema files" + ); + } +} diff --git a/crates/storage-postgres/tests/put_create_race.rs b/crates/storage-postgres/tests/put_create_race.rs index 98b02757a..981d29bfb 100644 --- a/crates/storage-postgres/tests/put_create_race.rs +++ b/crates/storage-postgres/tests/put_create_race.rs @@ -67,9 +67,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } /// Every `.sql` file under `migrations/` and `data_migrations/`, each @@ -126,10 +139,13 @@ async fn scratch() -> Scratch { .await .expect("apply a shipped migration"); } - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/stream_records_page_order.rs b/crates/storage-postgres/tests/stream_records_page_order.rs index 11e55be1a..06ba9e33d 100644 --- a/crates/storage-postgres/tests/stream_records_page_order.rs +++ b/crates/storage-postgres/tests/stream_records_page_order.rs @@ -25,10 +25,22 @@ const ACCOUNT: &str = "111111111111"; const REGION: &str = "us-east-1"; const LABEL: &str = "2026-01-01T00:00:00.000"; +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING") - .ok() - .filter(|s| !s.is_empty()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn) } fn shipped_migrations() -> Vec { @@ -118,10 +130,13 @@ async fn a_page_is_in_sequence_order_when_the_records_were_written_in_reverse() .execute(&db) .await .expect("match the two-database layout"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{ACCOUNT}")) diff --git a/crates/storage-postgres/tests/stream_shard_ownership.rs b/crates/storage-postgres/tests/stream_shard_ownership.rs index 8c5a1cc90..6ff100327 100644 --- a/crates/storage-postgres/tests/stream_shard_ownership.rs +++ b/crates/storage-postgres/tests/stream_shard_ownership.rs @@ -140,9 +140,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } /// Every `.sql` file under `migrations/` and `data_migrations/`, each @@ -195,10 +208,13 @@ async fn scratch() -> Scratch { .execute(&db) .await .expect("match the two-database layout"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); for account in [ACCOUNT, OTHER_ACCOUNT] { sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(account) diff --git a/crates/storage-postgres/tests/stream_shard_retention.rs b/crates/storage-postgres/tests/stream_shard_retention.rs index 6c07a0cb0..a68833d23 100644 --- a/crates/storage-postgres/tests/stream_shard_retention.rs +++ b/crates/storage-postgres/tests/stream_shard_retention.rs @@ -53,9 +53,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } /// Every `.sql` file under `migrations/` and `data_migrations/`, each @@ -114,10 +127,13 @@ async fn scratch() -> Scratch { .execute(&db) .await .expect("match the two-database layout"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/table_snapshot.rs b/crates/storage-postgres/tests/table_snapshot.rs index adfad63b2..7a985c88d 100644 --- a/crates/storage-postgres/tests/table_snapshot.rs +++ b/crates/storage-postgres/tests/table_snapshot.rs @@ -259,9 +259,9 @@ async fn create_databases(base: &str, names: &[String; 2], pgvector: bool) -> bo /// Apply every catalog and data migration the crate ships, as `extenddb /// migrate --yes` does (the data ones on `data_url`, which is the catalog -/// database on the shared layout), and pgvector if asked. The lists live in -/// the crate (`migrations.rs`), so a migration added there is applied here -/// without this file knowing. +/// database on the shared layout), and pgvector if asked. The table lives in +/// the crate (`migrations/mod.rs`), so a migration added there is applied +/// here without this file knowing. async fn migrate(catalog: &PgPool, data_url: &str, pgvector: bool) { let data = PgPoolOptions::new() .max_connections(2) @@ -281,14 +281,19 @@ async fn migrate(catalog: &PgPool, data_url: &str, pgvector: bool) { } /// Zero the control-plane and propagation delays, and point the engine at -/// `separate_data_url` when there is one. +/// `separate_data_url` when there is one. The rows are written, not only +/// updated: migration 1 seeds no setting, and the tests below that change a +/// delay update the row written here. async fn configure_catalog(catalog: &PgPool, separate_data_url: Option<&String>) { for key in ["control_plane_delay_seconds", "index_propagation_delay_ms"] { - sqlx::query("UPDATE settings SET value = '0' WHERE key = $1") - .bind(key) - .execute(catalog) - .await - .expect("setting"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ($1, '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(key) + .execute(catalog) + .await + .expect("setting"); } if let Some(data_url) = separate_data_url { sqlx::query( diff --git a/crates/storage-postgres/tests/twi_conflict.rs b/crates/storage-postgres/tests/twi_conflict.rs index 5d7c6fa14..8d5205469 100644 --- a/crates/storage-postgres/tests/twi_conflict.rs +++ b/crates/storage-postgres/tests/twi_conflict.rs @@ -65,9 +65,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } /// Every `.sql` file under `migrations/` and `data_migrations/`, each @@ -124,10 +137,13 @@ async fn scratch() -> Scratch { .await .expect("apply a shipped migration"); } - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index 6118f9a0a..6ff14fd35 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -112,9 +112,22 @@ impl Scratch { } } +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) } /// Every `.sql` file under `migrations/` and `data_migrations/`, each @@ -192,17 +205,23 @@ async fn scratch(pgvector: Pgvector) -> Scratch { // Zero control-plane delay so CreateTable and DeleteTable complete inline: // these tests run no background workers, so a scheduled transition would // never happen and every table would sit in CREATING. - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&catalog) - .await - .expect("pin the control-plane delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&catalog) + .await + .expect("pin the control-plane delay to zero"); // And the propagation delay, for the same reason and the same way // devtools/run-tests does it: these tests run no queue workers, so anything // enqueued would never be applied. A test that wants the queue sets this itself. - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") - .execute(&catalog) - .await - .expect("pin the propagation delay to zero"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&catalog) + .await + .expect("pin the propagation delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) diff --git a/crates/storage-sqlite/Cargo.toml b/crates/storage-sqlite/Cargo.toml index 09447140d..c74c4c45a 100644 --- a/crates/storage-sqlite/Cargo.toml +++ b/crates/storage-sqlite/Cargo.toml @@ -39,8 +39,14 @@ bigdecimal = { workspace = true } crc32fast = { workspace = true } zeroize = { workspace = true } libc = { workspace = true } +libsqlite3-sys = { workspace = true } [dev-dependencies] +extenddb-storage = { workspace = true, features = ["test-util"] } +# The slow-statement test sets sqlx's threshold, a `log` level, and prints +# every tracing event of a child process. +log = "0.4" +tracing-subscriber = { workspace = true } # The crate's own tests need `test-hooks`; a dev-dependency on itself turns # the feature on for test builds only. extenddb-storage-sqlite = { path = ".", features = ["test-hooks"] } diff --git a/crates/storage-sqlite/migrations/0001_baseline.expected.json b/crates/storage-sqlite/migrations/0001_baseline.expected.json new file mode 100644 index 000000000..ecf562ed7 --- /dev/null +++ b/crates/storage-sqlite/migrations/0001_baseline.expected.json @@ -0,0 +1,514 @@ +{ + "objects": [ + { + "type": "index", + "name": "idx_backup_items_arn", + "table": "backup_items", + "definition": [ + "CREATE INDEX idx_backup_items_arn ON backup_items(backup_arn)" + ] + }, + { + "type": "index", + "name": "idx_backups_table", + "table": "backups", + "definition": [ + "CREATE INDEX idx_backups_table ON backups(account_id,table_name)" + ] + }, + { + "type": "index", + "name": "idx_gsi_pending_claim", + "table": "gsi_pending", + "definition": [ + "CREATE INDEX idx_gsi_pending_claim ON gsi_pending(worker_partition,ready_at,id)" + ] + }, + { + "type": "index", + "name": "idx_idempotency_tokens_created", + "table": "idempotency_tokens", + "definition": [ + "CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens(created_at)" + ] + }, + { + "type": "index", + "name": "idx_login_attempts_principal_time", + "table": "login_attempts", + "definition": [ + "CREATE INDEX idx_login_attempts_principal_time ON login_attempts(principal,attempted_at DESC)" + ] + }, + { + "type": "index", + "name": "idx_login_attempts_source_ip_time", + "table": "login_attempts", + "definition": [ + "CREATE INDEX idx_login_attempts_source_ip_time ON login_attempts(source_ip,attempted_at DESC)WHERE source_ip IS NOT NULL" + ] + }, + { + "type": "index", + "name": "idx_metrics_bucket", + "table": "metrics", + "definition": [ + "CREATE INDEX idx_metrics_bucket ON metrics(bucket)" + ] + }, + { + "type": "index", + "name": "idx_stream_records_created", + "table": "stream_records", + "definition": [ + "CREATE INDEX idx_stream_records_created ON stream_records(created_at)" + ] + }, + { + "type": "index", + "name": "idx_stream_shards_table", + "table": "stream_shards", + "definition": [ + "CREATE INDEX idx_stream_shards_table ON stream_shards(table_id)" + ] + }, + { + "type": "index", + "name": "idx_table_restores_backup", + "table": "table_restores", + "definition": [ + "CREATE INDEX idx_table_restores_backup ON table_restores(source_backup_arn)" + ] + }, + { + "type": "index", + "name": "idx_tables_pending_transition", + "table": "tables", + "definition": [ + "CREATE INDEX idx_tables_pending_transition ON tables(status_transition_at)WHERE status_transition_at IS NOT NULL" + ] + }, + { + "type": "index", + "name": "idx_vector_indexes_index_id", + "table": "vector_indexes", + "definition": [ + "CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes(index_id)" + ] + }, + { + "type": "table", + "name": "access_keys", + "table": "access_keys", + "definition": [ + "FOREIGN KEY(account_id,user_name)REFERENCES iam_users(account_id,user_name)ON DELETE CASCADE", + "access_key_id TEXT PRIMARY KEY", + "account_id TEXT NOT NULL", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "is_active INTEGER NOT NULL DEFAULT 1", + "secret_key_encrypted BLOB NOT NULL", + "user_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "accounts", + "table": "accounts", + "definition": [ + "account_id TEXT PRIMARY KEY", + "account_name TEXT NOT NULL UNIQUE", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))" + ] + }, + { + "type": "table", + "name": "admin_users", + "table": "admin_users", + "definition": [ + "admin_name TEXT PRIMARY KEY", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "password_hash TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "backup_definitions", + "table": "backup_definitions", + "definition": [ + "backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn)ON DELETE CASCADE", + "definition TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "backup_items", + "table": "backup_items", + "definition": [ + "backup_arn TEXT NOT NULL REFERENCES backups(backup_arn)ON DELETE CASCADE", + "id INTEGER PRIMARY KEY AUTOINCREMENT", + "item_data TEXT NOT NULL", + "pk TEXT NOT NULL", + "sk TEXT" + ] + }, + { + "type": "table", + "name": "backups", + "table": "backups", + "definition": [ + "account_id TEXT NOT NULL", + "attribute_definitions TEXT NOT NULL", + "backup_arn TEXT PRIMARY KEY", + "backup_name TEXT NOT NULL", + "backup_size_bytes INTEGER NOT NULL DEFAULT 0", + "backup_status TEXT NOT NULL DEFAULT 'AVAILABLE'", + "backup_type TEXT NOT NULL DEFAULT 'USER'", + "billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST'", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "item_count INTEGER NOT NULL DEFAULT 0", + "key_schema TEXT NOT NULL", + "provisioned_throughput TEXT", + "stream_specification TEXT", + "table_id TEXT NOT NULL", + "table_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "continuous_backups", + "table": "continuous_backups", + "definition": [ + "PRIMARY KEY(account_id,table_name)", + "account_id TEXT NOT NULL", + "earliest_restorable TEXT", + "latest_restorable TEXT", + "pitr_enabled INTEGER NOT NULL DEFAULT 0", + "table_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "gsi_pending", + "table": "gsi_pending", + "definition": [ + "id INTEGER PRIMARY KEY AUTOINCREMENT", + "index_context TEXT NOT NULL", + "new_item TEXT", + "old_item TEXT", + "ready_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "table_id TEXT NOT NULL", + "worker_partition INTEGER NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_group_members", + "table": "iam_group_members", + "definition": [ + "FOREIGN KEY(account_id,group_name)REFERENCES iam_groups(account_id,group_name)ON DELETE CASCADE", + "FOREIGN KEY(account_id,user_name)REFERENCES iam_users(account_id,user_name)ON DELETE CASCADE", + "PRIMARY KEY(account_id,group_name,user_name)", + "account_id TEXT NOT NULL", + "group_name TEXT NOT NULL", + "user_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_groups", + "table": "iam_groups", + "definition": [ + "PRIMARY KEY(account_id,group_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "group_arn TEXT NOT NULL UNIQUE", + "group_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_permissions_boundaries", + "table": "iam_permissions_boundaries", + "definition": [ + "PRIMARY KEY(account_id,principal_type,principal_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "policy_document TEXT NOT NULL", + "principal_name TEXT NOT NULL", + "principal_type TEXT NOT NULL CHECK(principal_type IN('user','role'))" + ] + }, + { + "type": "table", + "name": "iam_policies", + "table": "iam_policies", + "definition": [ + "PRIMARY KEY(account_id,principal_type,principal_name,policy_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "policy_document TEXT NOT NULL", + "policy_name TEXT NOT NULL", + "principal_name TEXT NOT NULL", + "principal_type TEXT NOT NULL CHECK(principal_type IN('user','group','role'))" + ] + }, + { + "type": "table", + "name": "iam_role_tags", + "table": "iam_role_tags", + "definition": [ + "FOREIGN KEY(account_id,role_name)REFERENCES iam_roles(account_id,role_name)ON DELETE CASCADE", + "PRIMARY KEY(account_id,role_name,tag_key)", + "account_id TEXT NOT NULL", + "role_name TEXT NOT NULL", + "tag_key TEXT NOT NULL", + "tag_value TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_roles", + "table": "iam_roles", + "definition": [ + "PRIMARY KEY(account_id,role_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "permissions_boundary_arn TEXT", + "role_arn TEXT NOT NULL UNIQUE", + "role_name TEXT NOT NULL", + "trust_policy TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_sessions", + "table": "iam_sessions", + "definition": [ + "FOREIGN KEY(account_id,role_name)REFERENCES iam_roles(account_id,role_name)ON DELETE CASCADE", + "access_key_id TEXT NOT NULL UNIQUE", + "account_id TEXT NOT NULL", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "expires_at TEXT NOT NULL", + "role_name TEXT NOT NULL", + "secret_key_encrypted BLOB NOT NULL", + "session_name TEXT NOT NULL", + "session_policy TEXT", + "session_tags TEXT", + "session_token TEXT PRIMARY KEY" + ] + }, + { + "type": "table", + "name": "iam_user_tags", + "table": "iam_user_tags", + "definition": [ + "FOREIGN KEY(account_id,user_name)REFERENCES iam_users(account_id,user_name)ON DELETE CASCADE", + "PRIMARY KEY(account_id,user_name,tag_key)", + "account_id TEXT NOT NULL", + "tag_key TEXT NOT NULL", + "tag_value TEXT NOT NULL", + "user_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "iam_users", + "table": "iam_users", + "definition": [ + "PRIMARY KEY(account_id,user_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "password_hash TEXT", + "user_arn TEXT NOT NULL UNIQUE", + "user_name TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "idempotency_tokens", + "table": "idempotency_tokens", + "definition": [ + "PRIMARY KEY(account_id,token)", + "account_id TEXT NOT NULL", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "fingerprint TEXT NOT NULL", + "token TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "indexes", + "table": "indexes", + "definition": [ + "CONSTRAINT chk_propagation_delay_ms_non_negative CHECK(propagation_delay_ms IS NULL OR propagation_delay_ms >= 0)", + "CONSTRAINT indexes_table_id_fkey FOREIGN KEY(table_id)REFERENCES tables(table_id)ON DELETE CASCADE", + "PRIMARY KEY(table_id,index_name)", + "index_id TEXT NOT NULL", + "index_name TEXT NOT NULL", + "index_status TEXT NOT NULL DEFAULT 'ACTIVE'", + "index_type TEXT NOT NULL", + "key_schema TEXT NOT NULL", + "projection TEXT NOT NULL", + "propagation_delay_ms INTEGER", + "provisioned_throughput TEXT", + "table_id TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "login_attempts", + "table": "login_attempts", + "definition": [ + "attempted_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "principal TEXT NOT NULL", + "source_ip TEXT", + "success INTEGER NOT NULL" + ] + }, + { + "type": "table", + "name": "metrics", + "table": "metrics", + "definition": [ + "PRIMARY KEY(bucket,metric,table_name,index_name,operation)", + "bucket TEXT NOT NULL", + "count INTEGER NOT NULL DEFAULT 0", + "index_name TEXT NOT NULL DEFAULT ''", + "max REAL NOT NULL DEFAULT -9e999", + "metric TEXT NOT NULL", + "min REAL NOT NULL DEFAULT 9e999", + "operation TEXT NOT NULL DEFAULT ''", + "sum REAL NOT NULL DEFAULT 0", + "table_name TEXT NOT NULL DEFAULT ''" + ] + }, + { + "type": "table", + "name": "schema_history", + "table": "schema_history", + "definition": [ + "applied_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "filename TEXT PRIMARY KEY" + ] + }, + { + "type": "table", + "name": "seq_counters", + "table": "seq_counters", + "definition": [ + "name TEXT PRIMARY KEY", + "value INTEGER NOT NULL DEFAULT 0" + ] + }, + { + "type": "table", + "name": "settings", + "table": "settings", + "definition": [ + "key TEXT PRIMARY KEY", + "value TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "stream_records", + "table": "stream_records", + "definition": [ + "PRIMARY KEY(shard_id,sequence_number)", + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "event_name TEXT NOT NULL", + "record_data TEXT NOT NULL", + "sequence_number TEXT NOT NULL", + "shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id)ON DELETE CASCADE", + "table_id TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "stream_shards", + "table": "stream_shards", + "definition": [ + "created_at TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "ending_sequence_number TEXT", + "parent_shard_id TEXT", + "shard_id TEXT PRIMARY KEY", + "starting_sequence_number TEXT NOT NULL", + "table_id TEXT NOT NULL REFERENCES tables(table_id)ON DELETE CASCADE" + ] + }, + { + "type": "table", + "name": "table_restores", + "table": "table_restores", + "definition": [ + "restore_date_time TEXT NOT NULL", + "source_backup_arn TEXT NOT NULL", + "source_table_arn TEXT", + "table_id TEXT PRIMARY KEY REFERENCES tables(table_id)ON DELETE CASCADE" + ] + }, + { + "type": "table", + "name": "tables", + "table": "tables", + "definition": [ + "CONSTRAINT tables_table_id_unique UNIQUE(table_id)", + "PRIMARY KEY(account_id,table_name)", + "account_id TEXT NOT NULL REFERENCES accounts(account_id)ON DELETE CASCADE", + "attribute_definitions TEXT NOT NULL", + "billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST'", + "creation_date_time TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "deletion_protection_enabled INTEGER NOT NULL DEFAULT 0", + "item_count INTEGER NOT NULL DEFAULT 0", + "key_schema TEXT NOT NULL", + "on_demand_throughput TEXT", + "provisioned_throughput TEXT", + "sse_specification TEXT", + "status_transition_at TEXT", + "stream_label TEXT", + "stream_specification TEXT", + "table_arn TEXT NOT NULL", + "table_class TEXT", + "table_id TEXT NOT NULL", + "table_name TEXT NOT NULL", + "table_size_bytes INTEGER NOT NULL DEFAULT 0", + "table_status TEXT NOT NULL DEFAULT 'CREATING'", + "ttl_attribute TEXT", + "ttl_index_ready INTEGER NOT NULL DEFAULT 0" + ] + }, + { + "type": "table", + "name": "tags", + "table": "tags", + "definition": [ + "PRIMARY KEY(resource_arn,tag_key)", + "resource_arn TEXT NOT NULL", + "tag_key TEXT NOT NULL", + "tag_value TEXT NOT NULL" + ] + }, + { + "type": "table", + "name": "vector_indexes", + "table": "vector_indexes", + "definition": [ + "CONSTRAINT chk_vector_active_has_no_backfilling CHECK(index_status <> 'ACTIVE' OR backfilling IS NULL)", + "CONSTRAINT chk_vector_backfilling_bool CHECK(backfilling IS NULL OR backfilling IN(0,1))", + "CONSTRAINT chk_vector_dimensions_positive CHECK(dimensions > 0)", + "CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY(table_id)REFERENCES tables(table_id)ON DELETE CASCADE", + "PRIMARY KEY(table_id,index_name)", + "backfilling INTEGER", + "dimensions INTEGER NOT NULL", + "distance_function TEXT NOT NULL", + "index_id TEXT NOT NULL", + "index_name TEXT NOT NULL", + "index_status TEXT NOT NULL DEFAULT 'CREATING'", + "projection TEXT NOT NULL", + "search_schema TEXT", + "skipped_item_count INTEGER", + "table_id TEXT NOT NULL", + "vector_attribute TEXT NOT NULL" + ] + } + ] +} diff --git a/crates/storage-sqlite/migrations/0001_baseline.sql b/crates/storage-sqlite/migrations/0001_baseline.sql new file mode 100644 index 000000000..5f90c4408 --- /dev/null +++ b/crates/storage-sqlite/migrations/0001_baseline.sql @@ -0,0 +1,437 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 1, baseline, SQLite. Creates the v1.0 catalog schema in an empty +-- database and brings every released catalog to the same schema. The code +-- body in `src/migrations/m0001_baseline.rs` executes this file and then the +-- steps a file cannot make conditional (the `backup_items` rebuild and the +-- `table_restores.source_table_arn` column, each only where it is missing). +-- The runner runs the body on its dedicated connection with foreign keys +-- off inside one BEGIN IMMEDIATE transaction, checks foreign keys, and +-- commits it with its ledger row. The runner writes the version row; this +-- file writes no version and seeds no settings, because every seeded setting +-- has an equal default in code. Every statement is safe to run again on a +-- catalog at a later migration. +-- +-- Per-DynamoDB-table data and index tables (`_ddb_`) are not created +-- here; CreateTable creates them, and later migrations change them. + +-- Accounts, for multi-account support (REQ-AUTH-005). +CREATE TABLE IF NOT EXISTS accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +-- Table metadata. +CREATE TABLE IF NOT EXISTS tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); + +CREATE INDEX IF NOT EXISTS idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; + +-- Index metadata. index_id is always supplied by the engine (no DB-side UUID). +CREATE TABLE IF NOT EXISTS indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); + +-- Vector index metadata. Kept out of `indexes` deliberately: a vector index is +-- not described by a key schema, so reusing that table's `key_schema` column +-- would mean storing something meaningless in a NOT NULL column. The engine +-- supplies index_id, as it does for GSIs. +-- +-- `search_schema` is nullable because the HASH element is optional (measured +-- against the live service): with one the search is partition-scoped and +-- SearchConditionExpression is required, without one it spans the table. +-- +-- `backfilling` mirrors the measured lifecycle: false while CREATING before the +-- scan starts, true while it runs, and the member is absent once ACTIVE. Stored +-- as an integer so the ACTIVE state is representable as NULL rather than as a +-- third boolean value. +CREATE TABLE IF NOT EXISTS vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id + ON vector_indexes (index_id); + +-- Resource tags. +CREATE TABLE IF NOT EXISTS tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); + +-- The ledger of releases before 1.0. Nothing writes it; it is kept so that +-- fresh and upgraded catalogs have the same tables. +CREATE TABLE IF NOT EXISTS schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +-- Settings: the version row the runner writes, the data database name, and +-- runtime configuration. +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- Stream shards. +CREATE TABLE IF NOT EXISTS stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +CREATE INDEX IF NOT EXISTS idx_stream_shards_table ON stream_shards (table_id); + +-- Stream records. +CREATE TABLE IF NOT EXISTS stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); + +CREATE INDEX IF NOT EXISTS idx_stream_records_created ON stream_records (created_at); + +-- Admin users. +CREATE TABLE IF NOT EXISTS admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +-- IAM users. +CREATE TABLE IF NOT EXISTS iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); + +-- IAM user tags. +CREATE TABLE IF NOT EXISTS iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- Access keys. +CREATE TABLE IF NOT EXISTS access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- IAM groups. +CREATE TABLE IF NOT EXISTS iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); + +-- IAM group membership. +CREATE TABLE IF NOT EXISTS iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- IAM roles. +CREATE TABLE IF NOT EXISTS iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); + +-- IAM role tags. +CREATE TABLE IF NOT EXISTS iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); + +-- IAM sessions. +CREATE TABLE IF NOT EXISTS iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); + +-- IAM policies. +CREATE TABLE IF NOT EXISTS iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); + +-- IAM permissions boundaries. +CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); + +-- Idempotency tokens for TransactWriteItems. +CREATE TABLE IF NOT EXISTS idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created ON idempotency_tokens (created_at); + +-- Metrics (1-minute aggregation). ±9e999 store as ±Infinity in SQLite REAL, +-- matching PostgreSQL's Infinity / -Infinity seed for min / max. +CREATE TABLE IF NOT EXISTS metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); + +CREATE INDEX IF NOT EXISTS idx_metrics_bucket ON metrics (bucket); + +-- Login attempt tracking. +CREATE TABLE IF NOT EXISTS login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); + +CREATE INDEX IF NOT EXISTS idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); + +CREATE INDEX IF NOT EXISTS idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; + +-- Backup metadata. +CREATE TABLE IF NOT EXISTS backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +CREATE INDEX IF NOT EXISTS idx_backups_table ON backups (account_id, table_name); + +-- Backup items. `id` is the stable restore cursor: unlike an implicit rowid, +-- an INTEGER PRIMARY KEY value is not renumbered by VACUUM. +CREATE TABLE IF NOT EXISTS backup_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_backup_items_arn ON backup_items (backup_arn); + +-- A backup's table definition beyond its keys: secondary indexes, billing +-- mode and throughput, table class, encryption. JSON in the wire's shape +-- behind a version marker; see `extenddb_storage::backup_definition`. A +-- backup taken by a release that did not write it has no row and restores +-- with its keys only. +CREATE TABLE IF NOT EXISTS backup_definitions ( + backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, + definition TEXT NOT NULL +); + +-- Which backup a restored table came from: reported by DescribeTable as +-- RestoreSummary, and used to refuse DeleteBackup while the restore is +-- running. Not a foreign key to backups, which may be deleted later. +-- `restore_date_time` is the source backup's creation time, as the service +-- reports it, supplied by the engine. `source_table_arn` is nullable because +-- a pre-release build created the table without it; every row this build +-- writes sets it. +CREATE TABLE IF NOT EXISTS table_restores ( + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, + source_backup_arn TEXT NOT NULL, + source_table_arn TEXT, + restore_date_time TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_table_restores_backup ON table_restores (source_backup_arn); + +-- Continuous backups / PITR status. +CREATE TABLE IF NOT EXISTS continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); + +-- Persistent queue for async GSI propagation. A row is inserted inside the +-- base write transaction (zero crash window) and consumed by a background +-- worker once `ready_at` has passed; survives process crash/restart. +-- +-- One row per async index, and each row is self-describing: `index_context` +-- carries the base key schema, attribute definitions, and the single target +-- index definition captured at enqueue, so the worker applies with zero +-- catalog reads. `worker_partition` is a stable hash of the base table key; +-- all updates to a given base item share a partition and `ready_at` is kept +-- monotonically non-decreasing within it, so the worker (which drains in `id` +-- order) preserves per-key FIFO even with randomized propagation jitter. +CREATE TABLE IF NOT EXISTS gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); + +CREATE INDEX IF NOT EXISTS idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); + +-- Monotonic counters. Replaces PostgreSQL sequences. The stream counter is +-- seeded to the current epoch in microseconds so sequence numbers are +-- time-ordered and survive restarts (mirrors the PostgreSQL setval seed). +CREATE TABLE IF NOT EXISTS seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); + +INSERT OR IGNORE INTO seq_counters (name, value) + VALUES ('stream', CAST(strftime('%s','now') AS INTEGER) * 1000000); + +-- Every released baseline seeded this marker, and every released server +-- since v0.1.14 refuses a catalog without it. Removing it keeps those +-- servers off a v1.0 catalog; a release before 1.0 that writes it again is +-- reported as WrittenByOlderRelease. +DELETE FROM settings WHERE key = 'resource_tag_migration'; diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index fca131e5c..6d5d5b22c 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -1501,12 +1501,12 @@ mod restore_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("delay"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") + sqlx::query("INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("propagation"); @@ -2119,9 +2119,9 @@ mod restore_tests { /// A database with the schema catalog 0.0.3 shipped, holding a backup in /// the row shape 0.0.3 wrote (`pk` empty, `sk` NULL, no definition), is - /// brought to 0.0.4 by re-applying the schema, which is what - /// `extenddb migrate` does on this backend; the old backup then restores - /// as it did before: keys and items, no secondary indexes. + /// brought to the latest migration by the runner, as `extenddb migrate + /// --yes` does; the old backup then restores as it did before: keys and + /// items, no secondary indexes. #[tokio::test] async fn a_0_0_3_database_migrates_and_its_backups_restore() { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) @@ -2131,7 +2131,7 @@ mod restore_tests { .execute(&engine.pool) .await .expect("0.0.3 schema"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + sqlx::query("INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("delay"); @@ -2140,7 +2140,16 @@ mod restore_tests { .execute(&engine.pool) .await .expect("account"); - assert!(engine.check_catalog_version().await.is_err()); + let on_pool = crate::migrations::SqliteMigrations::on_pool(engine.pool.clone(), ":memory:") + .expect("one connection"); + let runner = extenddb_storage::migrations::Runner::new(&on_pool, "test"); + assert!( + matches!( + runner.status().await.expect("status"), + extenddb_storage::migrations::CatalogStatus::Behind { version: None, .. } + ), + "a 0.0.3 database has no ledger" + ); // A backup exactly as the 0.0.3 binary wrote one. let arn = format!("arn:aws:dynamodb:us-east-1:{ACCOUNT}:table/old/backup/1-00000000"); @@ -2170,38 +2179,20 @@ mod restore_tests { .expect("old backup item"); } - // The dry run names the rebuild, with the row count. - assert_eq!( - crate::schema::pending_shape_migrations(&engine.pool) - .await - .expect("pending"), - vec!["rebuild backup_items (3 rows)".to_owned()] - ); - let (log_guard, logs) = crate::schema::log_capture::capture(); - crate::schema::apply(&engine.pool).await.expect("migrate"); - drop(log_guard); - let logs = logs.lock().expect("logs").clone(); + // Migration 1 rebuilds backup_items and says how many rows it copied. + let report = runner.migrate().await.expect("migrate"); + let baseline_notes = report.applied[0].notes.join("\n"); assert!( - logs.iter().any(|l| l.starts_with("INFO") - && l.contains("rebuilding backup_items") - && l.contains("(3 rows)")), - "{logs:#?}" + baseline_notes.contains("copying 3 row(s)"), + "{baseline_notes}" ); assert!( - logs.iter() - .any(|l| l.starts_with("INFO") && l.contains("rebuilt backup_items (3 rows)")), - "{logs:#?}" - ); - assert!( - crate::schema::pending_shape_migrations(&engine.pool) - .await - .expect("pending after") - .is_empty() + matches!( + runner.status().await.expect("status after"), + extenddb_storage::migrations::CatalogStatus::Current { .. } + ), + "current after the run" ); - engine - .check_catalog_version() - .await - .expect("0.0.4 after migrate"); let id_is_primary_key: i64 = sqlx::query_scalar( "SELECT pk FROM pragma_table_info('backup_items') WHERE name = 'id'", ) @@ -2426,12 +2417,12 @@ mod restore_tests { ) .await .expect("file engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("delay"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") + sqlx::query("INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("propagation"); @@ -2446,7 +2437,7 @@ mod restore_tests { async fn close_file_engine(engine: SqliteEngine, path: &std::path::Path) { engine.pool.close().await; drop(engine); - for suffix in ["", "-wal", "-shm"] { + for suffix in ["", "-wal", "-shm", ".lock"] { let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); } } diff --git a/crates/storage-sqlite/src/bootstrapper.rs b/crates/storage-sqlite/src/bootstrapper.rs index 293a39c8d..d4c65b16c 100644 --- a/crates/storage-sqlite/src/bootstrapper.rs +++ b/crates/storage-sqlite/src/bootstrapper.rs @@ -9,17 +9,12 @@ //! user. Destruction removes the database file and its WAL/SHM sidecars. use async_trait::async_trait; -use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType}; -use extenddb_storage::bootstrapper::{ - AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair, helpers, -}; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper, helpers}; use extenddb_storage::error::StorageError; use extenddb_storage::management_store::{OpError, OpResult}; -use extenddb_storage::util::recover_sort_key_definitions; use sqlx::SqlitePool; use sqlx::sqlite::SqlitePoolOptions; -use crate::schema::{self, CATALOG_VERSION}; use crate::serve_lock::ServeLock; use crate::sqlite_util::sqlite_url; @@ -30,17 +25,11 @@ use crate::sqlite_util::sqlite_url; /// paths, not the hot serving path. pub struct SqliteBootstrapper { path: String, - /// The database-file lock held between [`Bootstrapper::acquire_migration_lock`] - /// and [`Bootstrapper::release_migration_lock`]; see `serve_lock`. - held_lock: std::sync::Mutex>, } impl SqliteBootstrapper { pub fn new(path: impl Into) -> Self { - Self { - path: path.into(), - held_lock: std::sync::Mutex::new(None), - } + Self { path: path.into() } } /// Take the exclusive lock a running `extenddb serve` holds on this @@ -121,14 +110,9 @@ impl SqliteBootstrapper { SqlitePoolOptions::new() .max_connections(1) .after_connect(|conn, _| { - Box::pin(async move { - use sqlx::Executor; - conn.execute("PRAGMA journal_mode = WAL").await?; - conn.execute("PRAGMA foreign_keys = ON").await?; - conn.execute("PRAGMA synchronous = NORMAL").await?; - conn.execute("PRAGMA busy_timeout = 5000").await?; - Ok(()) - }) + Box::pin( + async move { crate::sqlite_util::set_connection_pragmas(conn, true).await }, + ) }) .connect(&sqlite_url(&self.path)) .await @@ -146,16 +130,49 @@ impl Bootstrapper for SqliteBootstrapper { Ok(()) // SQLite has no role concept. } + /// Create the database file, empty and with mode 0600, and its directory. + /// An existing file is refused so `init` is not silently destructive. + /// The commands' migration lock opens only an existing file, so the file + /// exists before `init` takes it. async fn create_catalog_db(&self) -> OpResult<()> { - // Connecting with mode=rwc creates the file; refuse to clobber an - // existing database so `init` is not silently destructive. - if !self.is_memory() && std::path::Path::new(&self.path).exists() { - return Err(OpError::AlreadyExists(format!( + if self.is_memory() { + return Ok(()); + } + let exists = || { + OpError::AlreadyExists(format!( "SQLite database '{}' already exists. Run 'destroy' first, then 'init'.", self.path - ))); + )) + }; + let path = std::path::Path::new(&self.path); + if path.exists() { + return Err(exists()); + } + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).map_err(|e| { + OpError::Internal(format!( + "create parent directory for SQLite database '{}': {e}", + self.path + )) + })?; + } + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(path) { + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Err(exists()), + Err(e) => Err(OpError::Internal(format!( + "create SQLite database '{}': {e}", + self.path + ))), } - Ok(()) } async fn create_data_db(&self) -> OpResult<()> { @@ -163,35 +180,15 @@ impl Bootstrapper for SqliteBootstrapper { } async fn run_catalog_migrations(&self) -> OpResult<()> { - let pool = self.pool().await?; - schema::apply(&pool).await + Err(through_the_runner("run_catalog_migrations")) } async fn run_data_migrations(&self) -> OpResult<()> { - // Catalog and data share one file, so the schema itself is applied in - // run_catalog_migrations. What remains here are the table-shape steps - // an already-0.0.4 catalog may still need (an earlier 0.0.4 build's - // backup_items / table_restores), which `migrate` skips - // run_catalog_migrations for because the version already matches. - if !self.is_catalog_initialized().await? { - return Ok(()); - } - let pool = self.pool().await?; - schema::apply_shape_migrations(&pool).await + Err(through_the_runner("run_data_migrations")) } async fn pending_data_migrations(&self) -> OpResult> { - // No migration files: the complete schema is applied in - // run_catalog_migrations. Reported here instead are the table rebuilds - // and column additions that applying it would perform, so that - // `migrate`'s dry run names them (the backup_items rebuild copies every - // backup row in one write transaction) and so that `migrate --yes` - // runs them on a catalog already stamped 0.0.4. - if !self.is_catalog_initialized().await? { - return Ok(Vec::new()); - } - let pool = self.pool().await?; - schema::pending_shape_migrations(&pool).await + Err(through_the_runner("pending_data_migrations")) } async fn record_data_connection(&self) -> OpResult<()> { @@ -310,161 +307,7 @@ impl Bootstrapper for SqliteBootstrapper { } async fn is_catalog_initialized(&self) -> OpResult { - if !self.is_memory() && !std::path::Path::new(&self.path).exists() { - return Ok(false); - } - let Ok(pool) = self.pool().await else { - return Ok(false); - }; - schema::table_exists(&pool, "settings").await - } - - /// Repair table metadata damaged by the pre-fix `UpdateTable` (#259). - /// - /// SQLite keeps the catalog and the data tables in one file, so the physical - /// sort key columns are read with `PRAGMA table_info`. Otherwise identical to - /// the PostgreSQL implementation: for any base sort key with no attribute - /// definition, recover the type from its column name. - async fn repair_lost_sort_key_definitions(&self, apply: bool) -> OpResult { - let mut report = KeyDefinitionRepair::default(); - let Ok(pool) = self.pool().await else { - return Ok(report); - }; - // A never-initialised catalog has no `tables` table, and `pool()` opens - // with mode=rwc, which silently creates an empty database file, so the - // SELECT below would hard-error rather than find nothing. PostgreSQL - // degrades gracefully through its get_data_db_name() guard; this is the - // SQLite equivalent. - if !schema::table_exists(&pool, "tables").await? { - return Ok(report); - } - - let rows: Vec<(String, String, String, String, String)> = sqlx::query_as( - "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ - FROM tables ORDER BY table_name", - ) - .fetch_all(&pool) - .await - .map_err(|e| OpError::Internal(format!("Cannot read tables: {e}")))?; - - for (account_id, table_name, table_id, ks_json, ad_json) in rows { - let key_schema: Vec = match serde_json::from_str(&ks_json) { - Ok(v) => v, - Err(e) => { - report - .needs_attention - .push(format!("{table_name}: unreadable key_schema ({e})")); - continue; - } - }; - let attr_defs: Vec = - serde_json::from_str(&ad_json).unwrap_or_default(); - - // Reported for every table on every run, not only when a sort key is - // repaired: the partition key definition is dropped by the same write and - // is not recoverable from the schema, because the pk column is always TEXT. - // It is not needed for correctness either, since partition key values are - // encoded from the key schema alone, so it is reported rather than guessed - // and keeps being reported until a human restores it. - let pk_missing: Vec<&str> = key_schema - .iter() - .filter(|ks| ks.key_type == KeyType::Hash) - .filter(|ks| { - !attr_defs - .iter() - .any(|ad| ad.attribute_name == ks.attribute_name) - }) - .map(|ks| ks.attribute_name.as_str()) - .collect(); - if !pk_missing.is_empty() { - report.needs_attention.push(format!( - "{table_name}: partition key definition(s) [{}] are absent; reads and \ - writes are unaffected, but index key type validation cannot check them", - pk_missing.join(", ") - )); - } - - let missing: Vec<&KeySchemaElement> = key_schema - .iter() - .filter(|ks| ks.key_type == KeyType::Range) - .filter(|ks| { - !attr_defs - .iter() - .any(|ad| ad.attribute_name == ks.attribute_name) - }) - .collect(); - if missing.is_empty() { - continue; - } - - // PRIMARY KEY columns only, in key order. Every data table carries all - // three typed columns for each sort key position (sk_s, sk_n, sk_b), so - // column existence says nothing about the declared type: only the pk - // position reported by table_info does. `table_id` is interpolated - // because PRAGMA takes no bind parameters; it is checked as a UUID first - // so a malformed catalog row cannot reach the statement. - if uuid::Uuid::parse_str(&table_id).is_err() { - report - .needs_attention - .push(format!("{table_name}: table_id {table_id:?} is not a UUID")); - continue; - } - let mut key_columns: Vec<(i64, String)> = sqlx::query_as::< - _, - (i64, String, String, i64, Option, i64), - >(&format!( - "PRAGMA table_info(\"_ddb_{table_id}\")" - )) - .fetch_all(&pool) - .await - .map_err(|e| OpError::Internal(format!("Cannot read columns of {table_name}: {e}")))? - .into_iter() - .filter(|(.., pk_pos)| *pk_pos > 0) - .map(|(_, name, _, _, _, pk_pos)| (pk_pos, name)) - .collect(); - key_columns.sort_unstable(); - let columns: Vec = key_columns.into_iter().map(|(_, name)| name).collect(); - - let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); - if recovered.is_empty() { - report.needs_attention.push(format!( - "{table_name}: sort key(s) [{}] have no attribute definition and no \ - matching PRIMARY KEY column was found to recover the type from", - missing - .iter() - .map(|ks| ks.attribute_name.as_str()) - .collect::>() - .join(", ") - )); - continue; - } - - let mut merged = attr_defs; - merged.extend(recovered.iter().cloned()); - let merged_json = serde_json::to_string(&merged) - .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; - if apply { - sqlx::query( - "UPDATE tables SET attribute_definitions = ?1 \ - WHERE account_id = ?2 AND table_name = ?3", - ) - .bind(&merged_json) - .bind(&account_id) - .bind(&table_name) - .execute(&pool) - .await - .map_err(|e| OpError::Internal(format!("Cannot repair {table_name}: {e}")))?; - } - - for def in &recovered { - report.repaired.push(format!( - "{table_name}: {} ({:?})", - def.attribute_name, def.attribute_type - )); - } - } - - Ok(report) + Err(through_the_runner("is_catalog_initialized")) } async fn list_table_names(&self) -> OpResult> { @@ -491,28 +334,6 @@ impl Bootstrapper for SqliteBootstrapper { Ok(row.map(|(v,)| v)) } - /// `init` and `migrate` rewrite the schema, so they take the same - /// exclusive lock `extenddb serve` holds on the database file. Unlike the - /// trait's default contract this does not wait for the lock: the holder is - /// a server that runs until stopped, so waiting would hang, and the - /// operator is told to stop it instead. - async fn acquire_migration_lock(&self) -> OpResult<()> { - let lock = self.exclusive_lock("this command")?; - *self - .held_lock - .lock() - .map_err(|_| OpError::Internal("lock state poisoned".to_owned()))? = lock; - Ok(()) - } - - async fn release_migration_lock(&self) -> OpResult<()> { - self.held_lock - .lock() - .map_err(|_| OpError::Internal("lock state poisoned".to_owned()))? - .take(); - Ok(()) - } - async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { if self.is_memory() { return Ok(()); @@ -533,22 +354,13 @@ impl Bootstrapper for SqliteBootstrapper { } async fn read_catalog_version(&self) -> OpResult> { - let Ok(pool) = self.pool().await else { - return Ok(None); - }; - if !schema::table_exists(&pool, "settings").await? { - return Ok(None); - } - let row: Option<(String,)> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_optional(&pool) - .await - .map_err(|e| OpError::Internal(format!("read catalog version: {e}")))?; - Ok(row.map(|(v,)| v)) + Err(through_the_runner("read_catalog_version")) } fn expected_catalog_version(&self) -> String { - CATALOG_VERSION.to_string() + extenddb_storage::migrations::MigrationId::LATEST + .number() + .to_string() } fn catalog_database_name(&self) -> String { @@ -573,16 +385,26 @@ impl Bootstrapper for SqliteBootstrapper { } } +/// The error of a bootstrapper method the commands do not call on this +/// backend: its catalog migrates through +/// [`crate::migrations::SqliteCatalogMigrations`] and the shared runner. +fn through_the_runner(method: &str) -> OpError { + OpError::Internal(format!( + "sqlite migrates through its CatalogMigrations; the bootstrapper method {method} is \ + not used" + )) +} + #[cfg(all(test, unix))] mod lock_tests { use super::SqliteBootstrapper; use crate::serve_lock::ServeLock; use extenddb_storage::bootstrapper::Bootstrapper; - /// `destroy` and `migrate` are refused while a server holds the database, - /// and a server is refused while `migrate` holds it. + /// `destroy` is refused while a server holds the database, and runs once + /// the server has gone. #[tokio::test] - async fn destroy_and_migrate_are_refused_while_a_server_holds_the_file() { + async fn destroy_is_refused_while_a_server_holds_the_file() { let dir = std::env::temp_dir().join(format!( "extenddb-bootstrap-lock-{}", uuid::Uuid::new_v4().simple() @@ -605,90 +427,10 @@ mod lock_tests { db.exists(), "destroy must not unlink a file a server has open" ); - let err = bootstrap - .acquire_migration_lock() - .await - .expect_err("migrate under a server"); - assert!( - format!("{err:?}").contains("another extenddb process"), - "{err:?}" - ); drop(server); - bootstrap.acquire_migration_lock().await.expect("free now"); - assert!(ServeLock::acquire(&db).is_err(), "serve during migrate"); - bootstrap.release_migration_lock().await.expect("release"); - ServeLock::acquire(&db).expect("free after release"); - bootstrap.drop_databases("unused").await.expect("destroy"); assert!(!db.exists()); let _ = std::fs::remove_dir_all(&dir); } } - -#[cfg(test)] -mod migrate_report_tests { - use super::SqliteBootstrapper; - use extenddb_storage::bootstrapper::Bootstrapper; - - /// `extenddb migrate`'s dry run prints `pending_data_migrations`. On a - /// 0.0.3 database it names the backup_items rebuild with its row count, - /// `run_data_migrations` performs it even when the catalog version - /// already matches (an earlier 0.0.4 build), and then nothing is pending. - #[tokio::test] - async fn the_dry_run_names_the_backup_items_rebuild() { - let target = std::env::var_os("CARGO_TARGET_DIR") - .map(std::path::PathBuf::from) - .unwrap_or_else(|| { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target") - }); - let path = target.join(format!("migrate-report-{}.sqlite", uuid::Uuid::new_v4())); - let bootstrap = SqliteBootstrapper::new(path.to_string_lossy().into_owned()); - assert!( - bootstrap - .pending_data_migrations() - .await - .expect("no database yet") - .is_empty() - ); - { - let pool = bootstrap.pool().await.expect("pool"); - sqlx::raw_sql(include_str!("../testdata/schema_0_0_3.sql")) - .execute(&pool) - .await - .expect("0.0.3 schema"); - // Stamped as an earlier 0.0.4 build left it: version current, - // backup_items still without its cursor column. - sqlx::raw_sql( - "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, \ - key_schema, attribute_definitions) VALUES ('a', 'b', 't', 't', '0', '[]', '[]'); \ - INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES \ - ('a', '', NULL, '{}'), ('a', '', NULL, '{}'); \ - UPDATE settings SET value = '0.0.4' WHERE key = 'catalog_version';", - ) - .execute(&pool) - .await - .expect("rows"); - pool.close().await; - } - assert_eq!( - bootstrap.read_catalog_version().await.expect("version"), - Some(bootstrap.expected_catalog_version()) - ); - assert_eq!( - bootstrap.pending_data_migrations().await.expect("pending"), - vec!["rebuild backup_items (2 rows)".to_owned()] - ); - bootstrap.run_data_migrations().await.expect("apply"); - assert!( - bootstrap - .pending_data_migrations() - .await - .expect("pending after") - .is_empty() - ); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } - } -} diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index ae4ae05e7..ce068463a 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -118,7 +118,7 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))? .and_then(|v| v.parse::().ok()) - .unwrap_or(0.25); + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS); let now = time::OffsetDateTime::now_utc(); let creation_ts = format_timestamp(now); diff --git a/crates/storage-sqlite/src/data/index.rs b/crates/storage-sqlite/src/data/index.rs index fa02deeb1..9c0a4113f 100644 --- a/crates/storage-sqlite/src/data/index.rs +++ b/crates/storage-sqlite/src/data/index.rs @@ -731,7 +731,7 @@ mod pending_context_tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let the_item: extenddb_core::types::Item = serde_json::from_str(r#"{"pk":{"S":"shared-key"},"emb":{"L":[{"N":"1"}]}}"#) @@ -772,9 +772,9 @@ mod pending_context_tests { /// A catalog created before the rename must keep honouring its operator's value. /// /// This is the compatibility property the whole fallback exists for, and the cost - /// of getting it wrong is not cosmetic: the server refuses to start on a - /// catalog-version mismatch rather than migrating, so nothing ever rewrites the old - /// row. Reading past it would silently reset a configured delay to the default, and + /// of getting it wrong is not cosmetic: no catalog migration rewrites the old row + /// yet, though one could. Reading past it would silently reset a configured delay + /// to the default, and /// since 0 means synchronous, the silent change would be from strict to eventually /// consistent, which is exactly the direction that turns a passing test suite into /// a flaky one somewhere else. @@ -783,7 +783,7 @@ mod pending_context_tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; // Reshape the catalog to look as it did before the rename: the legacy key // only, carrying a deliberately non-default value. @@ -814,7 +814,7 @@ mod pending_context_tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query( "INSERT OR REPLACE INTO settings (key, value) \ diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 1f578edb7..02bc09836 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -719,7 +719,7 @@ mod tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let ks = vec![ KeySchemaElement { @@ -834,7 +834,7 @@ mod tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let ks = vec![KeySchemaElement { attribute_name: "pk".to_owned(), @@ -937,7 +937,7 @@ mod tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let ks = vec![KeySchemaElement { attribute_name: "pk".to_owned(), @@ -1059,7 +1059,7 @@ mod tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let build = || SqliteVectorBuild { pool: engine.pool.clone(), diff --git a/crates/storage-sqlite/src/delete_table.rs b/crates/storage-sqlite/src/delete_table.rs index 476177d6c..272695c89 100644 --- a/crates/storage-sqlite/src/delete_table.rs +++ b/crates/storage-sqlite/src/delete_table.rs @@ -93,7 +93,7 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))? .and_then(|v| v.parse::().ok()) - .unwrap_or(0.25); + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS); let index_ids: Vec = index_rows.iter().map(|r| r.index_id.clone()).collect(); let table_id = row.table_id.clone(); diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 7e8085ff9..7e9b278cb 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -33,6 +33,7 @@ mod delete_table; mod hooks; mod management_store; mod metadata; +pub mod migrations; mod number_key; mod operations; mod schema; @@ -52,17 +53,23 @@ mod worker; mod workers; /// Default secondary-index propagation delay (milliseconds) when the -/// `index_propagation_delay_ms` setting is absent. Mirrors the value seeded by -/// the catalog schema, and is the single definition used by both the live read -/// on the write path and the background refresh worker. +/// `index_propagation_delay_ms` setting is absent, as it is on a catalog the +/// migrations created (releases before 1.0 seeded the same value). The single +/// definition used by both the live read on the write path and the background +/// refresh worker. pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; +/// Default control-plane delay (seconds) when the `control_plane_delay_seconds` +/// setting is absent or unparseable: CreateTable, DeleteTable, the +/// control-plane worker and the console's default all read this one value. +pub(crate) const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; + /// Read the propagation-delay setting, preferring the canonical key and falling back /// to the pre-rename one. /// /// A catalog created before the rename holds the operator's value under the old name, -/// and the server refuses to start on a catalog-version mismatch rather than migrating, -/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// and no catalog migration rewrites that row yet, though one could. Reading past it +/// would silently reset a /// configured delay to the default; since 0 means synchronous, the silent change would /// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference /// deterministic when both rows exist. @@ -74,7 +81,6 @@ pub use bootstrapper::SqliteBootstrapper; pub use catalog_store::SqliteCatalogStore; pub use config::SqliteConfig; pub use credential_store::SqliteCredentialStore; -pub use schema::CATALOG_VERSION; pub use store::SqliteEngine; /// The SQLite backend, for installation via @@ -94,6 +100,7 @@ pub fn backend() -> extenddb_storage::Backend { management_store: sqlite_management_store_factory, diagnostics_store: sqlite_diagnostics_store_factory, server_components: sqlite_server_components_factory, + catalog_migrations: Some(&migrations::SqliteCatalogMigrations), } } @@ -106,18 +113,12 @@ use extenddb_storage::settings_store::SettingsStoreError; use crate::sqlite_util::sqlite_url; /// Open a small catalog-only pool for the settings/diagnostics factories, -/// applying the same PRAGMAs as the engine. +/// with the pragmas of every SQLite pool. async fn catalog_pool(connection_string: &str) -> Result { sqlx::sqlite::SqlitePoolOptions::new() .max_connections(4) .after_connect(|conn, _| { - Box::pin(async move { - use sqlx::Executor; - conn.execute("PRAGMA journal_mode = WAL").await?; - conn.execute("PRAGMA foreign_keys = ON").await?; - conn.execute("PRAGMA busy_timeout = 5000").await?; - Ok(()) - }) + Box::pin(async move { crate::sqlite_util::set_connection_pragmas(conn, true).await }) }) .connect(&sqlite_url(connection_string)) .await @@ -192,6 +193,7 @@ fn sqlite_diagnostics_store_factory( use std::sync::Arc; use extenddb_auth::CredentialStore; +use extenddb_storage::migrations::{MigrationError, Runner}; use extenddb_storage::server_components::{BackendError, ServerComponents}; use crate::hooks::SqliteRuntimeHooks; @@ -209,14 +211,29 @@ fn sqlite_server_components_factory( let region = region.to_owned(); Box::pin(async move { // One server per database file, taken before the database is opened: - // startup recovery below assumes no other server is using it. A file - // database only; an in-memory one belongs to this process alone. - let serve_lock = serve_lock::ServeLock::acquire_for_location(&db_path) - .map_err(BackendError::InitializationFailed)? + // startup recovery below assumes no other server is using it, and a + // migration holds the same lock. A file database only; an in-memory + // one belongs to this process alone. + let serve_lock = serve_lock::ServeLock::try_acquire_for_location(&db_path) + .map_err(|failure| { + BackendError::Catalog(match failure { + serve_lock::LockFailure::Held(m) => MigrationError::Locked(m), + serve_lock::LockFailure::Failed(m) => MigrationError::Store(m), + }) + })? .map(|lock| { tracing::debug!("holding serve lock {}", lock.path().display()); std::sync::Arc::new(lock) }); + // The catalog's status is read before the engine opens the file, so + // a server that refuses has written nothing. A dev build migrates + // instead, as `migrate --yes` would, copy included. + let dev = options.bootstrap_if_uninitialized; + if let Some(lock) = &serve_lock { + migrations::prepare_for_serve(&db_path, options.release, lock, dev) + .await + .map_err(BackendError::Catalog)?; + } let mut engine = SqliteEngine::new(&db_path, pool_size, ®ion, MAX_ITEM_SIZE_BYTES) .await .map_err(|e| BackendError::ConnectionFailed { @@ -224,20 +241,28 @@ fn sqlite_server_components_factory( details: e.to_string(), })?; - // In-memory databases do not persist across the `init` process, so the - // catalog must be bootstrapped here, at serve time, on the engine's own - // shared connection (a second pool would open a separate empty DB). - // A dev-mode build extends the same bootstrap to an uninitialized FILE - // database (`bootstrap_if_uninitialized`), so zero-config dev serve - // works with a persistent path too; every bootstrap step is guarded - // (IF NOT EXISTS / INSERT OR IGNORE), so re-running on an initialized - // file is a no-op. Production builds keep the explicit-`init` contract. + // An in-memory database exists only on the engine's single + // connection, so its catalog is created there, by the runner, every + // time the server starts. let in_memory = engine.in_memory; - if in_memory || options.bootstrap_if_uninitialized { + if in_memory { + let on_pool = migrations::SqliteMigrations::on_pool(engine.pool.clone(), &db_path) + .map_err(BackendError::Catalog)?; + let report = Runner::new(&on_pool, options.release) + .migrate() + .await + .map_err(BackendError::Catalog)?; + migrations::log_report(&report); + } + + // The rows a server needs that `init` writes on a production + // deployment: written here for an in-memory database and in a dev + // build, which need no `init`. Each row is written only when absent. + if in_memory || dev { let admin_user = std::env::var("EXTENDDB_ADMIN_USER").ok(); let admin_password = std::env::var("EXTENDDB_ADMIN_PASSWORD").ok(); if let Some(password) = engine - .bootstrap_ephemeral(admin_user.as_deref(), admin_password.as_deref()) + .bootstrap_dev_rows(admin_user.as_deref(), admin_password.as_deref()) .await .map_err(|e| BackendError::InitializationFailed(e.to_string()))? { @@ -251,42 +276,6 @@ fn sqlite_server_components_factory( } } - engine.check_catalog_version().await.map_err(|e| match e { - extenddb_storage::error::StorageError::CatalogVersionMismatch { expected, found } => { - BackendError::CatalogVersionMismatch { expected, found } - } - // The unreadable-version refusal: printed as is, not as - // "Internal error: ...". - extenddb_storage::error::StorageError::Internal(refusal) => { - BackendError::InitializationFailed(refusal) - } - other => BackendError::InitializationFailed(other.to_string()), - })?; - - // The version string matches, but a table can still be in an earlier - // 0.0.4 build's shape (on which every DescribeTable fails). Refuse, as - // for a version mismatch: the steps can rewrite a table (the - // backup_items rebuild locks the database for writes while it runs), - // so they run under `extenddb migrate --yes`, not at startup. - let pending = schema::pending_shape_migrations(&engine.pool) - .await - .map_err(|e| { - use extenddb_storage::management_store::OpError; - let (OpError::Validation(m) - | OpError::AlreadyExists(m) - | OpError::NotFound(m) - | OpError::HasDependents(m) - | OpError::Internal(m)) = e; - BackendError::InitializationFailed(m) - })?; - if !pending.is_empty() { - return Err(BackendError::InitializationFailed(format!( - "the catalog's tables need migration steps this server does not run at \ - startup: {}. Stop the server and run 'extenddb migrate --yes'.", - pending.join(", ") - ))); - } - // Recover any in-flight control-plane transitions from a prior run. match engine.process_control_plane_transitions().await { Ok(transitions) => { @@ -435,67 +424,3 @@ mod serve_lock_tests { let _ = std::fs::remove_dir_all(&dir); } } - -#[cfg(test)] -mod shape_gate_tests { - /// `serve` on a file database whose catalog version matches but whose - /// `table_restores` is in the earlier 0.0.4 shape refuses to start, - /// naming the step and `extenddb migrate --yes`; after the migrate step - /// it starts. - #[tokio::test] - async fn serve_refuses_a_file_database_with_pending_shape_migrations() { - let dir = std::env::temp_dir().join(format!( - "extenddb-shape-gate-{}", - uuid::Uuid::new_v4().simple() - )); - std::fs::create_dir_all(&dir).expect("dir"); - let config = crate::config::SqliteConfig { - path: dir.join("db.sqlite").to_string_lossy().into_owned(), - pool_size: 2, - }; - let mut bootstrap = extenddb_storage::server_components::ServerComponentsOptions::default(); - bootstrap.bootstrap_if_uninitialized = true; - let plain = extenddb_storage::server_components::ServerComponentsOptions::default(); - - // A current database, then the earlier shape: no source_table_arn. - drop( - super::sqlite_server_components_factory(&config, "us-east-1", bootstrap) - .await - .expect("bootstrap a file database"), - ); - { - let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", config.path)) - .await - .expect("open"); - sqlx::query("ALTER TABLE table_restores DROP COLUMN source_table_arn") - .execute(&pool) - .await - .expect("the earlier table_restores shape"); - pool.close().await; - } - let Err(err) = super::sqlite_server_components_factory(&config, "us-east-1", plain).await - else { - panic!("serve started on the earlier shape"); - }; - assert!( - err.to_string() - .contains("add table_restores.source_table_arn"), - "{err}" - ); - assert!(err.to_string().contains("extenddb migrate --yes"), "{err}"); - - let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", config.path)) - .await - .expect("open"); - crate::schema::apply_shape_migrations(&pool) - .await - .expect("migrate --yes"); - pool.close().await; - drop( - super::sqlite_server_components_factory(&config, "us-east-1", plain) - .await - .expect("serve starts after migrate"), - ); - let _ = std::fs::remove_dir_all(&dir); - } -} diff --git a/crates/storage-sqlite/src/management_store/mod.rs b/crates/storage-sqlite/src/management_store/mod.rs index 3222d325f..563327b41 100644 --- a/crates/storage-sqlite/src/management_store/mod.rs +++ b/crates/storage-sqlite/src/management_store/mod.rs @@ -18,7 +18,8 @@ mod users; use extenddb_storage::management_store::{ AccessKeyCreated, AccountDetail, GroupDetail, GroupListEntry, ManagementStore, OpError, - OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, UserListEntry, + OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, StoredDocument, UserDetail, + UserListEntry, }; use futures::future::BoxFuture; use time::OffsetDateTime; @@ -395,6 +396,23 @@ impl ManagementStore for SqliteCatalogStore { }) } + fn list_policy_documents( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let (account_id, principal_type, principal_name) = ( + account_id.to_owned(), + principal_type.to_owned(), + principal_name.to_owned(), + ); + Box::pin(async move { + self.list_policy_documents_impl(&account_id, &principal_type, &principal_name) + .await + }) + } + // ── Permissions boundaries ───────────────────────────────────── fn set_user_boundary( @@ -461,6 +479,23 @@ impl ManagementStore for SqliteCatalogStore { Box::pin(async move { self.get_role_boundary_impl(&account_id, &role_name).await }) } + fn boundary_document( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let (account_id, principal_type, principal_name) = ( + account_id.to_owned(), + principal_type.to_owned(), + principal_name.to_owned(), + ); + Box::pin(async move { + self.boundary_document_impl(&account_id, &principal_type, &principal_name) + .await + }) + } + fn delete_role_boundary( &self, account_id: &str, @@ -720,7 +755,7 @@ mod migration_tests { .execute(&pool) .await .unwrap(); - crate::schema::apply(&pool).await.expect("apply schema"); + crate::migrations::apply_for_tests(&pool).await; SqliteCatalogStore::new(pool) } @@ -879,4 +914,55 @@ mod migration_tests { let p = stored.iter().find(|(n, ..)| n == "p").unwrap(); assert_eq!(p.1, collide); } + + /// A stored document that does not decode is named with its account, + /// principal, policy and decode error, and the principal's other + /// policies and its boundary are still read and rewritten. + #[tokio::test] + async fn an_undecodable_policy_is_named_and_the_others_are_rewritten() { + let s = store().await; + s.create_account("333333333333", "acct").await.unwrap(); + s.create_user("333333333333", "carol", None).await.unwrap(); + for name in ["a_broken", "b_tagged"] { + s.put_policy( + "333333333333", + "user", + "carol", + name, + &policy("dynamodb:ResourceTag/env"), + ) + .await + .unwrap(); + } + s.set_user_boundary("333333333333", "carol", &policy("dynamodb:ResourceTag/env")) + .await + .unwrap(); + sqlx::query( + "UPDATE iam_policies SET policy_document = '{not json' \ + WHERE account_id = '333333333333' AND policy_name = 'a_broken'", + ) + .execute(s.pool()) + .await + .unwrap(); + + let report = migrate_resource_tag_condition_keys(&s, true).await; + assert_eq!(report.errors, 0, "{:?}", report.failures); + assert_eq!(report.unreadable.len(), 1, "{report:?}"); + let broken = &report.unreadable[0]; + assert_eq!( + ( + broken.account_id.as_str(), + broken.principal.as_str(), + broken.policy_name.as_str() + ), + ("333333333333", "user/carol", "a_broken") + ); + assert!(broken.error.contains("line 1"), "{}", broken.error); + let names: Vec<&str> = report + .rewritten + .iter() + .map(|r| r.policy_name.as_str()) + .collect(); + assert_eq!(names, ["b_tagged", ""]); + } } diff --git a/crates/storage-sqlite/src/management_store/policies.rs b/crates/storage-sqlite/src/management_store/policies.rs index da9e82e03..1cc31e1c4 100644 --- a/crates/storage-sqlite/src/management_store/policies.rs +++ b/crates/storage-sqlite/src/management_store/policies.rs @@ -3,7 +3,7 @@ //! IAM policy CRUD and permissions boundaries for `SqliteCatalogStore`. -use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::management_store::{OpError, OpResult, StoredDocument}; use time::OffsetDateTime; use crate::catalog_store::SqliteCatalogStore; @@ -107,6 +107,59 @@ impl SqliteCatalogStore { .collect() } + /// A principal's policies for the resource-tag migration, each document + /// decoded on its own: one that does not decode comes back as its decode + /// error, and the others are still read. + pub(crate) async fn list_policy_documents_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT policy_name, policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ? \ + ORDER BY policy_name", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_policy_documents: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(rows + .into_iter() + .map(|(name, doc)| (name, serde_json::from_str(&doc).map_err(|e| e.to_string()))) + .collect()) + } + + /// A principal's permissions boundary for the resource-tag migration: + /// the decoded document, or its decode error. + pub(crate) async fn boundary_document_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let row: Option<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ?", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("boundary_document: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.map(|(doc,)| serde_json::from_str(&doc).map_err(|e| e.to_string()))) + } + // ── Permissions boundaries ───────────────────────────────────── async fn set_boundary( diff --git a/crates/storage-sqlite/src/migrations/backend.rs b/crates/storage-sqlite/src/migrations/backend.rs new file mode 100644 index 000000000..e7490aece --- /dev/null +++ b/crates/storage-sqlite/src/migrations/backend.rs @@ -0,0 +1,661 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The runner's SQLite parts. +//! +//! Every body runs on one dedicated connection, a pool of exactly one +//! connection that this module opens and closes, so code bodies that take a +//! pool (the management store, for migration 2) run inside the runner's +//! transaction. Per body: `PRAGMA foreign_keys = OFF`, `BEGIN IMMEDIATE`, +//! `PRAGMA foreign_key_check`, the body, the check again (a violating row the +//! body added rolls back and stops the run; one that was there before is +//! noted), the post-check the first time migration 1 is applied, the ledger +//! row and the version row, `COMMIT`, then `PRAGMA foreign_keys = ON`. The +//! pragma has no effect inside a transaction, and a table rebuild with +//! foreign keys on deletes child rows through `ON DELETE CASCADE`. The +//! connection is closed when the runner is done, on every exit path, because +//! every other pool assumes the pragma its `after_connect` set. +//! +//! An in-memory database exists only on the engine's one connection, so the +//! runner runs on that connection, with no lock and no copy. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use async_trait::async_trait; +use extenddb_storage::management_store::ManagementStore; +use extenddb_storage::migrations::{ + CatalogTraces, Done, HasManagementStore, LedgerColumns, LedgerRow, LedgerWrites, Migration, + MigrationBackend, MigrationError, MigrationId, Released, RunReport, Runner, Step, StoreKind, + StoreRef, in_transaction, record_rows, sql_ledger_ddl, +}; +use sqlx::sqlite::SqlitePoolOptions; +use sqlx::{ConnectOptions, Connection, SqlitePool}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::{set_connection_pragmas, sqlite_url}; + +/// What SQLite code bodies receive: the runner's connection, and a +/// management store over it. +pub struct SqliteStores { + pub(crate) pool: SqlitePool, + management: SqliteCatalogStore, +} + +impl HasManagementStore for SqliteStores { + fn management_store(&self) -> &dyn ManagementStore { + &self.management + } +} + +/// The ledger's `applied_at` column in SQLite, which has no time type: the +/// UTC time as text, to the millisecond. +const APPLIED_AT: &str = "TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))"; + +/// The ledger rows of the runner's connection, in SQLite's statements. +struct Rows<'a> { + pool: &'a SqlitePool, + /// Tests make the ledger row's insert fail, after the body ran. + #[cfg(test)] + fail_insert: bool, +} + +#[async_trait] +impl LedgerWrites for Rows<'_> { + type Error = MigrationError; + + async fn create_ledger(&self) -> Result<(), MigrationError> { + let ddl = sql_ledger_ddl(APPLIED_AT); + sqlx::raw_sql(&ddl) + .execute(self.pool) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store(&ddl, e)) + } + + async fn insert_row(&self, row: &LedgerRow) -> Result<(), MigrationError> { + #[cfg(test)] + if self.fail_insert { + return Err(MigrationError::Store(format!( + "record migration {} {}: the test failed the insert", + row.id, row.name + ))); + } + sqlx::query( + "INSERT INTO extenddb_migrations \ + (id, name, kind, checksum, outcome, notes, applied_by) \ + VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind(i64::from(row.id)) + .bind(&row.name) + .bind(row.kind.as_str()) + .bind(&row.checksum) + .bind(row.outcome.as_str()) + .bind(&row.notes) + .bind(&row.applied_by) + .execute(self.pool) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store(format!("record migration {} {}", row.id, row.name), e)) + } + + async fn write_version(&self, version: u32) -> Result<(), MigrationError> { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('catalog_version', ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(version.to_string()) + .execute(self.pool) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store("write catalog_version", e)) + } +} + +/// The SQLite backend of the migration runner. +pub struct SqliteMigrations { + stores: SqliteStores, + location: String, + /// The database file; `None` for an in-memory database. + file: Option, + /// Whether the connection is this runner's own, closed by [`Self::close`]. + owned: bool, + /// The copy this run took before its first write, if it took one. + copy: Mutex>, + /// The lines naming the earlier copies this run's copy replaced, + /// reported with the run, also when it stops before its first commit. + replaced: Mutex>, + /// The lines naming the partial copies of stopped runs that opening the + /// database for writing removed, reported with the run. + removed: Mutex>, + /// `PRAGMA foreign_keys = ON` failed after a step committed: the step is + /// reported as applied, and the run stops with this error at its next + /// call ([`Self::discard_unused_copy`] when the step was its last). + restore_failed: Mutex>, + /// Tests make the pragma after a commit fail with this. + #[cfg(test)] + pub(crate) fail_restore: std::sync::atomic::AtomicBool, + /// Tests make the ledger row's insert fail after a body ran. + #[cfg(test)] + pub(crate) fail_insert_row: std::sync::atomic::AtomicBool, +} + +impl SqliteMigrations { + /// Open the runner's connection to the database at `location`, creating + /// the file when it does not exist: `init` creates it before its first + /// run, and a dev-mode server opens a new database this way. + /// + /// # Errors + /// + /// The location does not parse, or the database cannot be opened. + pub async fn open(location: &str) -> Result { + Self::open_with(location, true).await + } + + /// Open a read-only connection for reading the status: SQLite opens the + /// file read-only, so reading changes nothing in it (no journal mode + /// switch, no WAL checkpoint) and creates no file. + /// + /// # Errors + /// + /// As [`Self::open`]. + pub async fn open_for_status(location: &str) -> Result { + Self::open_with(location, false).await + } + + async fn open_with(location: &str, write: bool) -> Result { + let file = crate::serve_lock::database_file(location).map_err(MigrationError::Store)?; + if write && let Some(path) = &file { + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).map_err(|e| { + MigrationError::store(format!("create the directory of {location}"), e) + })?; + } + super::open::create_private(path)?; + } + let options = sqlx::sqlite::SqliteConnectOptions::from_str(&sqlite_url(location)) + .map_err(|e| MigrationError::store(format!("open SQLite database {location}"), e))? + .read_only(!write) + .create_if_missing(write); + // A test build counts every statement as slow, so the test that a run + // prints no slow-statement warning needs no large catalog. + #[cfg(test)] + let options = + options.log_slow_statements(log::LevelFilter::Warn, std::time::Duration::ZERO); + // The operator lines report every step; sqlx's slow-statement + // warnings would print SQL between them. + let options = options.disable_statement_logging(); + // A database that cannot be opened, or whose pragmas fail, is + // reported at once with SQLite's cause: the pool would run the + // failing connection again until its acquire timeout. + let mut probe = options + .connect() + .await + .map_err(|e| super::open::open_error(location, &e))?; + let pragmas = set_connection_pragmas(&mut probe, write).await; + let _ = probe.close().await; + pragmas.map_err(|e| super::open::open_error(location, &e))?; + // The connection is opened once. A second connect would mean the first + // was lost, together with any open transaction, and statements would + // then commit one by one, so it is refused. A connection counts as + // opened once its pragmas succeeded; the first failure is returned as + // it is, without waiting for the pool's acquire timeout. + let opened = Arc::new(AtomicUsize::new(0)); + let first_error: Arc>> = Arc::new(Mutex::new(None)); + let failure = Arc::clone(&first_error); + let connected = SqlitePoolOptions::new() + .max_connections(1) + .min_connections(1) + .idle_timeout(None) + .max_lifetime(None) + .acquire_timeout(std::time::Duration::from_secs(5)) + .after_connect(move |conn, _| { + let opened = Arc::clone(&opened); + let failure = Arc::clone(&failure); + Box::pin(async move { + if opened.load(Ordering::SeqCst) > 0 { + return Err(sqlx::Error::Protocol( + "the migration connection was lost and is not reopened".to_owned(), + )); + } + match set_connection_pragmas(conn, write).await { + Ok(()) => { + opened.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + Err(e) => { + let mut first = failure.lock().unwrap_or_else(PoisonError::into_inner); + first.get_or_insert_with(|| e.to_string()); + Err(e) + } + } + }) + }) + .connect_with(options) + .await; + let pool = match connected { + Ok(pool) => pool, + Err(e) => { + let first = first_error + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let cause = first.unwrap_or_else(|| e.to_string()); + return Err(MigrationError::Store(format!( + "open SQLite database {location}: {cause}" + ))); + } + }; + let migrations = Self::with_pool(pool, location, file, true); + // Every run that writes holds the database's lock, so a partial copy + // beside the database now is a stopped run's, whether or not this + // run copies. + if write && let Some(path) = &migrations.file { + *migrations + .removed + .lock() + .unwrap_or_else(PoisonError::into_inner) = + super::copy::remove_stopped_partials(path); + } + Ok(migrations) + } + + /// Run on the engine's pool of an in-memory database, which is left + /// open. The pool must hold exactly one connection: the runner's + /// transaction and the body's statements have to share it. + /// + /// # Errors + /// + /// [`MigrationError::Refused`] for a pool that allows more than one + /// connection, which a file database's engine pool does. + pub fn on_pool(pool: SqlitePool, location: &str) -> Result { + let max = pool.options().get_max_connections(); + if max != 1 { + return Err(MigrationError::Refused(format!( + "SqliteMigrations::on_pool runs on an in-memory database's single connection, \ + and this pool allows {max}; migrate a file database with migrate()" + ))); + } + Ok(Self::with_pool(pool, location, None, false)) + } + + fn with_pool(pool: SqlitePool, location: &str, file: Option, owned: bool) -> Self { + Self { + stores: SqliteStores { + management: SqliteCatalogStore::new(pool.clone()), + pool, + }, + location: location.to_owned(), + file, + owned, + copy: Mutex::new(None), + replaced: Mutex::new(Vec::new()), + removed: Mutex::new(Vec::new()), + restore_failed: Mutex::new(None), + #[cfg(test)] + fail_restore: std::sync::atomic::AtomicBool::new(false), + #[cfg(test)] + fail_insert_row: std::sync::atomic::AtomicBool::new(false), + } + } + + /// [`Runner::forget`] in one transaction, so the ledger row and the + /// version row change together. + /// + /// # Errors + /// + /// As [`Runner::forget`]; the database is then as it was. + pub(crate) async fn forget(&self, id: u32) -> Result<(), MigrationError> { + in_transaction( + self.exec("BEGIN IMMEDIATE"), + Runner::new(self, "").forget(id), + self.exec("COMMIT"), + async { + let _ = self.exec("ROLLBACK").await; + }, + ) + .await + } + + /// The ledger writes on the runner's connection. + fn rows(&self) -> Rows<'_> { + Rows { + pool: &self.stores.pool, + #[cfg(test)] + fail_insert: self.fail_insert_row.load(Ordering::SeqCst), + } + } + + /// The error a failed `PRAGMA foreign_keys = ON` left after a commit. + fn take_restore_failure(&self) -> Option { + self.restore_failed + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + } + + /// `result` as the caller reports it: a run that stopped before it + /// committed anything left the database as it was, so the copy it took + /// is removed, and the error is returned with the lines naming the files + /// the run removed, or alone when it removed none. A run whose last step + /// committed and whose pragma after it failed stops with that error and + /// keeps its copy. A report starts with the partial copies of stopped + /// runs that opening the database removed. + pub(crate) fn discard_unused_copy( + &self, + result: Result, + ) -> Result { + let result = match (result, self.take_restore_failure()) { + (Ok(report), Some(error)) => Err(MigrationError::Stopped { + error: Box::new(error), + report, + }), + (result, _) => result, + }; + let removed = + std::mem::take(&mut *self.removed.lock().unwrap_or_else(PoisonError::into_inner)); + let replaced = + std::mem::take(&mut *self.replaced.lock().unwrap_or_else(PoisonError::into_inner)); + match result { + Ok(mut report) => { + report.notes.splice(0..0, removed); + report.notes.extend(replaced); + Ok(report) + } + Err(MigrationError::Stopped { error, report }) if report.applied.is_empty() => { + // The run's own notes name the copy removed here, so only + // the lines naming the other files it removed are kept. + let copy = self + .copy + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let mut notes = removed; + notes.extend(replaced); + if let Some(Err(note)) = copy.map(|copy| super::copy::remove_copy(©)) { + notes.push(note); + } + if notes.is_empty() { + Err(*error) + } else { + Err(MigrationError::Stopped { + error, + report: RunReport { notes, ..report }, + }) + } + } + Err(MigrationError::Stopped { error, mut report }) => { + report.notes.splice(0..0, removed); + report.notes.extend(replaced); + Err(MigrationError::Stopped { error, report }) + } + other => other, + } + } + + /// Close the runner's own connection. Callers close it on every exit + /// path; see [`migrate`](super::migrate) for the pattern. + pub async fn close(self) { + if self.owned { + self.stores.pool.close().await; + } + } + + #[cfg(test)] + pub(crate) fn pool(&self) -> &SqlitePool { + &self.stores.pool + } + + async fn exec(&self, sql: &str) -> Result<(), MigrationError> { + sqlx::raw_sql(sql) + .execute(&self.stores.pool) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store(sql, e)) + } + + async fn table_exists(&self, name: &str) -> Result { + crate::schema::table_exists(&self.stores.pool, name) + .await + .map_err(|e| MigrationError::Store(format!("look up table {name}: {e:?}"))) + } + + /// The checks, the body and the rows, inside the open transaction. + async fn body_and_rows( + &self, + id: MigrationId, + step: Step<'_, SqliteStores>, + row: Option, + catalog_version: Option, + ) -> Result { + let pool = &self.stores.pool; + let before = foreign_key_violations(pool).await?; + let mut done = match step { + Step::File { name, text } => { + sqlx::raw_sql(text) + .execute(pool) + .await + .map_err(|e| MigrationError::store(format!("migration {id} ({name})"), e))?; + Done::default() + } + Step::Code(body) => body.run(&self.stores).await?, + Step::Record => Done::default(), + }; + let added: Vec = foreign_key_violations(pool) + .await? + .into_iter() + .filter(|v| !before.contains(v)) + .collect(); + if !added.is_empty() { + return Err(MigrationError::Store(format!( + "migration {id} left {} row(s) that violate a foreign key, in {}; nothing was \ + committed", + added.len(), + by_table(&added) + ))); + } + if !before.is_empty() { + done.notes.push(format!( + "{} row(s) violated a foreign key before migration {id} ran, in {}; they are \ + left as they are", + before.len(), + by_table(&before) + )); + } + // The first application only: on a catalog at a later migration the + // tables migration 1 creates may carry what later migrations added. + if id == MigrationId::Baseline && row.is_some() { + done.notes + .extend(super::postcheck::check(pool, super::BASELINE_EXPECTED).await?); + } + record_rows(&self.rows(), row, &done.notes, catalog_version).await?; + Ok(done) + } +} + +/// One row of `PRAGMA foreign_key_check`: the child table, its rowid (none +/// for a table without one), the parent table and the constraint's index. +type Violation = (String, Option, String, i64); + +async fn foreign_key_violations(pool: &SqlitePool) -> Result, MigrationError> { + sqlx::query_as("PRAGMA foreign_key_check") + .fetch_all(pool) + .await + .map_err(|e| MigrationError::store("PRAGMA foreign_key_check", e)) +} + +/// `table (n), ...` for a list of violations. +fn by_table(violations: &[Violation]) -> String { + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for (table, ..) in violations { + *counts.entry(table.as_str()).or_default() += 1; + } + counts + .into_iter() + .map(|(table, n)| format!("{table} ({n})")) + .collect::>() + .join(", ") +} + +#[async_trait] +impl MigrationBackend for SqliteMigrations { + type Stores = SqliteStores; + + fn backend_name(&self) -> &'static str { + "sqlite" + } + + fn migration(&self, id: MigrationId) -> Migration { + super::migration(id) + } + + fn released(&self) -> &'static [Released] { + super::RELEASED + } + + fn stores(&self) -> &SqliteStores { + &self.stores + } + + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError> { + Ok(match kind { + StoreKind::Catalog => vec![self.location.clone()], + StoreKind::Data | StoreKind::Account => Vec::new(), + }) + } + + async fn read_ledger(&self, _store: &StoreRef) -> Result, MigrationError> { + if !self.table_exists("extenddb_migrations").await? { + return Ok(Vec::new()); + } + let rows: Vec> = sqlx::query_as( + "SELECT id, name, kind, checksum, outcome, notes, applied_at, applied_by \ + FROM extenddb_migrations ORDER BY id", + ) + .fetch_all(&self.stores.pool) + .await + .map_err(|e| MigrationError::store("read extenddb_migrations", e))?; + rows.into_iter() + .map(|columns| LedgerRow::from_columns(None, columns)) + .collect() + } + + async fn read_traces(&self) -> Result { + if !self.table_exists("settings").await? { + return Ok(CatalogTraces::default()); + } + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT key, value FROM settings \ + WHERE key IN ('catalog_version', 'resource_tag_migration')", + ) + .fetch_all(&self.stores.pool) + .await + .map_err(|e| MigrationError::store("read settings", e))?; + Ok(CatalogTraces::from_settings(&rows)) + } + + async fn catalog_is_empty(&self) -> Result { + let tables: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'") + .fetch_one(&self.stores.pool) + .await + .map_err(|e| MigrationError::store("count tables", e))?; + Ok(tables == 0) + } + + async fn before_first_write( + &self, + from: Option, + to: u32, + ) -> Result, MigrationError> { + let Some(file) = &self.file else { + return Ok(Vec::new()); + }; + if self.catalog_is_empty().await? { + // An empty database (init) holds nothing to keep. + return Ok(Vec::new()); + } + let from = from.map_or_else(|| "legacy".to_owned(), |v| v.to_string()); + let copy = super::copy::copy_path(file, &from, to, std::time::SystemTime::now()); + let lines = super::copy::pre_migration_copy( + &self.stores.pool, + file, + ©, + super::copy::Hooks::RUN, + ) + .await?; + // An earlier copy with the same bytes holds nothing this copy does + // not, so it is removed now, before the run's first commit: a crash + // loop that kills each run before that commit then leaves one copy, + // not one per run. + let replaced = super::copy::remove_superseded(file, ©).await; + self.replaced + .lock() + .unwrap_or_else(PoisonError::into_inner) + .extend(replaced); + *self.copy.lock().unwrap_or_else(PoisonError::into_inner) = Some(copy); + Ok(lines) + } + + async fn apply( + &self, + _store: &StoreRef, + id: MigrationId, + step: Step<'_, SqliteStores>, + row: Option, + catalog_version: Option, + ) -> Result { + if let Some(error) = self.take_restore_failure() { + return Err(error); + } + self.exec("PRAGMA foreign_keys = OFF").await?; + let result = in_transaction( + self.exec("BEGIN IMMEDIATE"), + self.body_and_rows(id, step, row, catalog_version), + self.exec("COMMIT"), + async { + let _ = self.exec("ROLLBACK").await; + }, + ) + .await; + let restored = self.exec("PRAGMA foreign_keys = ON").await; + #[cfg(test)] + let restored = if self.fail_restore.load(Ordering::SeqCst) { + Err(MigrationError::Store( + "PRAGMA foreign_keys = ON: the connection was lost".to_owned(), + )) + } else { + restored + }; + let done = result?; + // The step is committed, so it is reported as applied whatever the + // pragma did; a failed pragma stops the run after it. + if let Err(e) = restored { + *self + .restore_failed + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(MigrationError::Store(format!( + "migration {id} was committed, and then {e}; the run stops here" + ))); + } + Ok(done) + } + + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError> { + self.rows().write_version(version).await + } + + async fn delete_ledger_row(&self, _store: &StoreRef, id: u32) -> Result<(), MigrationError> { + sqlx::query("DELETE FROM extenddb_migrations WHERE id = ?") + .bind(i64::from(id)) + .execute(&self.stores.pool) + .await + .map(|_| ()) + .map_err(|e| MigrationError::store(format!("forget migration {id}"), e)) + } +} diff --git a/crates/storage-sqlite/src/migrations/commands.rs b/crates/storage-sqlite/src/migrations/commands.rs new file mode 100644 index 000000000..51fcc1a41 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/commands.rs @@ -0,0 +1,192 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The SQLite backend's [`CatalogMigrations`], which `init`, `migrate` and +//! `verify` call through the installed backend. + +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, HasManagementStore, HeldLock, LedgerRow, Listed, + MigrationBackend, MigrationError, MigrationId, RunKind, RunReport, Runner, Target, + policy_rewrite_lines, policy_rewrite_pending, +}; + +use super::{ + MigrationLock, SqliteMigrations, catalog_status, in_memory_refusal, lock_for_migration, +}; + +/// The SQLite backend's catalog migrations for the commands. +pub struct SqliteCatalogMigrations; + +#[async_trait] +impl CatalogMigrations for SqliteCatalogMigrations { + fn listed(&self, id: MigrationId) -> Listed { + Listed::of(&super::migration(id)) + } + + fn backup_advice(&self, target: &Target) -> String { + let location = &target.location; + format!( + "back up the SQLite database file {location} with its -wal and -shm files \ + (migrate --yes also copies the database beside it before its first change)" + ) + } + + /// `EXTENDDB__STORAGE__SQLITE__PATH=`, + /// quoted for a POSIX shell when it needs it; none in memory. + fn location_assignment(&self, target: &Target) -> Option { + let file = crate::serve_lock::database_file(&target.location).ok()??; + let path = std::path::absolute(&file).unwrap_or(file); + Some(format!( + "EXTENDDB__STORAGE__SQLITE__PATH={}", + shell_word(&path.to_string_lossy()) + )) + } + + /// `EXTENDDB__STORAGE__SQLITE__PATH` is set, which overrides the config + /// file's `path`. + fn location_from_environment(&self) -> bool { + std::env::var_os("EXTENDDB__STORAGE__SQLITE__PATH").is_some_and(|v| !v.is_empty()) + } + + async fn status(&self, target: &Target) -> Result { + catalog_status(&target.location).await + } + + /// Reads the policies on a read-only connection, as the status does. + /// The installed backend's management store opens the database for + /// writing: it sets WAL mode on a rollback-journal database, and its + /// connection, when it is the last one to close, checkpoints the WAL + /// into the database file. + async fn pending_changes( + &self, + target: &Target, + status: &CatalogStatus, + ) -> Result, MigrationError> { + if !policy_rewrite_pending(status) { + return Ok(Vec::new()); + } + let migrations = SqliteMigrations::open_for_status(&target.location).await?; + let lines = policy_rewrite_lines(migrations.stores().management_store()).await; + migrations.close().await; + lines + } + + async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError> { + let location = target.location.as_str(); + let file = crate::serve_lock::database_file(location) + .map_err(MigrationError::Store)? + .ok_or_else(in_memory_refusal)?; + if !file.exists() { + return Ok(Vec::new()); + } + let migrations = SqliteMigrations::open_for_status(location).await?; + let rows = Runner::new(&migrations, "").catalog_ledger().await; + migrations.close().await; + rows + } + + /// The lock on an existing database file. A file that does not exist is + /// not a catalog: `migrate` refuses it as `Uninitialized` and creates + /// nothing, not even the lock file. `init` creates the file before it + /// takes the lock. + async fn lock(&self, target: &Target, timeout: Duration) -> Result { + let location = target.location.as_str(); + let file = crate::serve_lock::database_file(location) + .map_err(MigrationError::Store)? + .ok_or_else(in_memory_refusal)?; + if !file.exists() { + return Err(MigrationError::Status(CatalogStatus::Uninitialized)); + } + let lock = lock_for_migration(location, timeout) + .await? + .ok_or_else(in_memory_refusal)?; + Ok(HeldLock::new(location, lock)) + } + + async fn run( + &self, + lock: &HeldLock, + release: &str, + kind: RunKind, + ) -> Result { + let location = locked_location(lock)?; + let migrations = SqliteMigrations::open(location).await?; + let runner = Runner::new(&migrations, release); + let result = match kind { + RunKind::Create | RunKind::Migrate => runner.migrate().await, + RunKind::Reconverge => runner.reconverge().await, + }; + let result = migrations.discard_unused_copy(result); + migrations.close().await; + result + } + + async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError> { + let location = locked_location(lock)?; + let migrations = SqliteMigrations::open(location).await?; + let result = migrations.forget(id).await; + migrations.close().await; + result + } +} + +/// `word` as one word of a POSIX shell command: as it is when every +/// character is one a shell leaves alone, else in single quotes. +fn shell_word(word: &str) -> String { + let plain = !word.is_empty() + && word + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"/._-+:@%".contains(&b)); + if plain { + word.to_owned() + } else { + format!("'{}'", word.replace('\'', "'\\''")) + } +} + +/// The location of a lock this backend took. +fn locked_location(lock: &HeldLock) -> Result<&str, MigrationError> { + lock.get::() + .map(|_| lock.location()) + .ok_or_else(|| { + MigrationError::Refused(format!( + "the lock on {} was not taken by the SQLite backend", + lock.location() + )) + }) +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::{SqliteCatalogMigrations, shell_word}; + + #[test] + fn the_assignment_names_the_absolute_path_as_one_shell_word() { + assert_eq!(shell_word("/srv/x.sqlite"), "/srv/x.sqlite"); + assert_eq!(shell_word("/a b/it's"), "'/a b/it'\\''s'"); + let shown = SqliteCatalogMigrations + .location_assignment(&Target::at("/srv/db/x.sqlite")) + .expect("a file"); + assert_eq!(shown, "EXTENDDB__STORAGE__SQLITE__PATH=/srv/db/x.sqlite"); + let relative = SqliteCatalogMigrations + .location_assignment(&Target::at("x.sqlite")) + .expect("a file"); + let cwd = std::env::current_dir().expect("cwd"); + assert_eq!( + relative, + format!( + "EXTENDDB__STORAGE__SQLITE__PATH={}", + shell_word(&cwd.join("x.sqlite").to_string_lossy()) + ) + ); + assert_eq!( + SqliteCatalogMigrations.location_assignment(&Target::at(":memory:")), + None + ); + } +} diff --git a/crates/storage-sqlite/src/migrations/copy.rs b/crates/storage-sqlite/src/migrations/copy.rs new file mode 100644 index 000000000..0379dd5c8 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/copy.rs @@ -0,0 +1,807 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The copy of the database taken before the first write of a run that +//! applies anything. +//! +//! The copy is taken with SQLite's online backup API, on a connection to the +//! database and one to the copy, both opened for it on a blocking thread. +//! The backup reads the database, the committed frames of its WAL included, +//! in one read transaction, so the copy holds every write committed when +//! that transaction began, whatever another process writes meanwhile. It +//! takes time in proportion to the database's size. `VACUUM INTO`, which it +//! replaces, creates every table and index again, and its time grows faster +//! than the number of tables. The copy is then switched to rollback-journal +//! mode, as `VACUUM INTO` left it: the backup copies the database's WAL +//! mark, and a WAL database cannot be read without a `-shm` file beside it. +//! +//! Only SQLite opens the database file. POSIX advisory locks belong to a +//! process and a file, so closing any file descriptor of the database that +//! other code in this process opened would release every lock the process +//! holds on it, the shared lock of the runner's connection included. Another +//! process that closed its connection then would take itself for the last +//! one, checkpoint the WAL and delete it, and the commits the runner made +//! after that would go to the deleted file and be lost. SQLite keeps its own +//! descriptors of a file open while a connection of the process holds a lock +//! on it. The search for earlier copies below opens files under a copy's +//! name, so it skips a symbolic link and a hard link to the database, its +//! WAL or its index, which a restore with `ln` can leave under such a name. +//! +//! The copy is created owner-only (0600) before SQLite writes to it, +//! whatever the database's mode: the database holds the encryption key, and +//! a catalog that 0.1.x `init` created and no server opened is 0644. +//! +//! Every copy's name carries the UTC time the run took it, +//! `.pre-migration--to--`, and a name that +//! already exists is an error, so no run reuses or overwrites a copy. +//! +//! A copy is written to `.partial` first. A run holds the database's +//! lock before it copies, so a partial file beside the database when a run +//! starts was left by a run that was stopped while it copied; the run +//! removes every one, with the files SQLite keeps beside it, and names +//! them. +//! +//! A run that stops before it commits anything removes its copy, since the +//! database is unchanged; a run that committed a step keeps its copy and +//! names it in its error. A run that was killed after its copy was complete +//! and before it committed anything leaves the copy. Once the next run's +//! own copy is complete, the run removes such a copy, and names it, when +//! the copy has the same bytes as its own: the same database contents, so +//! its own copy holds everything the earlier one does, and runs killed in +//! that place one after the other leave one copy. A run that then stops +//! before its first commit removes its own copy as well and names the +//! earlier one with its error. The same `` alone would not show that +//! nothing changed: a reconverge runs bodies again without changing the +//! catalog version, and `--forget` brings a catalog back to a version it had +//! before. A copy with a `-wal` or `-journal` file beside it, which a +//! process opened, is left as it is. Nothing else removes a copy. + +use std::fs::File; +use std::io::{Read as _, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use extenddb_storage::migrations::MigrationError; +use sqlx::SqlitePool; + +/// Free space a run needs, as a multiple of the database's size: the copy +/// takes one; migration 1's rebuild of `backup_items` writes the table again +/// and its WAL holds the new pages until the checkpoint. Measured on a +/// catalog of 250,000 backup items: a copy of 1.0 times the database, a WAL +/// of 1.0 times at migration 1's commit, and a database file of 2.0 times +/// after the run. +const SPACE_FACTOR: u64 = 3; + +/// What a copy calls, or waits for, that tests replace. +#[derive(Clone, Copy)] +pub(crate) struct Hooks { + /// The free bytes of a directory's filesystem. + pub(crate) free_space: fn(&Path) -> Option, + /// Runs between the sync and the rename; tests stop the process there. + pub(crate) before_rename: fn(&Path), + /// How long the copy waits while another connection holds a lock the + /// backup needs. + pub(crate) locked_for: Duration, +} + +impl Hooks { + /// The hooks of a run. + pub(crate) const RUN: Self = Self { + free_space, + before_rename: no_hook, + locked_for: Duration::from_secs(30), + }; +} + +fn no_hook(_: &Path) {} + +/// Take the copy at `copy`. The database is written to `.partial`, +/// synced, and renamed to `copy`, so a process that is stopped while it +/// copies leaves no file under the copy's name. Prints a line before the +/// copy with its path and the database's size, and one after it with the +/// time it took, since a run holds the lock while it copies. Returns the +/// lines for the operator: the partial copies of stopped runs it removed, +/// then the copy. +pub(crate) async fn pre_migration_copy( + pool: &SqlitePool, + db: &Path, + copy: &Path, + hooks: Hooks, +) -> Result, MigrationError> { + if std::fs::symlink_metadata(copy).is_ok() { + return Err(MigrationError::Refused(format!( + "{} already exists, and a copy taken before migrating is never reused or replaced; \ + run the migration again in a second, or move the file away", + copy.display() + ))); + } + // Before the space check: the partial copies of stopped runs may be what + // fills the disk. + let mut lines = remove_stopped_partials(db); + let live = live_size(pool).await?; + let dir = parent_dir(copy); + if let Some(free) = (hooks.free_space)(&dir) { + refuse_short_space(live, free, &dir)?; + } + let partial = partial_path(copy); + progress(&format!( + "--- Copying the database ({live} bytes) to {}...", + copy.display() + )); + let started = Instant::now(); + if let Err(e) = write_partial(db, copy, &partial, hooks.locked_for).await { + remove_partial(&partial); + return Err(e); + } + (hooks.before_rename)(&partial); + if let Err(e) = std::fs::rename(&partial, copy) { + remove_partial(&partial); + return Err(MigrationError::store( + format!("rename {} to {}", partial.display(), copy.display()), + e, + )); + } + // The rename is durable once the directory is synced. + let _ = sync_dir(&dir).await; + let seconds = started.elapsed().as_secs_f64(); + progress(&format!("--- Copied the database in {seconds:.1} s")); + lines.push(format!( + "copied the database to {} before migrating, in {seconds:.1} s", + copy.display() + )); + Ok(lines) +} + +/// Write the copy of `db` to `partial`, created owner-only, with SQLite's +/// online backup API on a blocking thread, and sync it: a large database +/// takes a while, and a server that migrates at startup goes on with its +/// other tasks meanwhile. +async fn write_partial( + db: &Path, + copy: &Path, + partial: &Path, + locked_for: Duration, +) -> Result<(), MigrationError> { + create_partial(partial).map_err(|e| copy_failed(copy, e))?; + let (source, target) = (db.to_path_buf(), partial.to_path_buf()); + tokio::task::spawn_blocking(move || backup::copy(&source, &target, locked_for)) + .await + .map_err(|e| copy_failed(copy, e))? + .map_err(|e| copy_failed(copy, e))?; + sync_file(partial).await +} + +fn copy_failed(copy: &Path, e: impl std::fmt::Display) -> MigrationError { + MigrationError::store(format!("copy the database to {}", copy.display()), e) +} + +/// SQLite's online backup API, which sqlx does not expose, on connections +/// opened for one copy and closed after it. +mod backup { + use std::ffi::{CStr, CString, c_int}; + use std::path::Path; + use std::time::{Duration, Instant}; + + use libsqlite3_sys as ffi; + + /// A connection, closed when dropped. + struct Connection(*mut ffi::sqlite3); + + impl Connection { + /// Open the existing database file at `path` for reading and + /// writing. SQLite creates no database file here. + fn open(path: &Path) -> Result { + let name = path + .to_str() + .and_then(|name| CString::new(name).ok()) + .ok_or_else(|| format!("{} is not a UTF-8 path without NUL", path.display()))?; + let mut handle = std::ptr::null_mut(); + // SAFETY: `name` is NUL-terminated and outlives the call, and + // `handle` is a valid place for the connection's pointer. + let rc = unsafe { + ffi::sqlite3_open_v2( + name.as_ptr(), + &raw mut handle, + ffi::SQLITE_OPEN_READWRITE, + std::ptr::null(), + ) + }; + // SQLite returns a connection to close even when the open fails, + // unless it could not allocate one. + let connection = Self(handle); + if rc != ffi::SQLITE_OK { + return Err(format!( + "open {}: {}", + path.display(), + connection.last_error(rc) + )); + } + // SAFETY: the connection is open. + unsafe { ffi::sqlite3_busy_timeout(connection.0, 5_000) }; + Ok(connection) + } + + /// Run `sql` on the connection; rows it returns are not read. + fn execute(&self, sql: &CStr) -> Result<(), String> { + // SAFETY: the connection is open and `sql` is NUL-terminated. + // With no callback and no place for an error string, SQLite + // allocates nothing for the caller to free, finalizes its own + // statements, and leaves an error on the connection. + let rc = unsafe { + ffi::sqlite3_exec( + self.0, + sql.as_ptr(), + None, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if rc == ffi::SQLITE_OK { + Ok(()) + } else { + Err(self.last_error(rc)) + } + } + + /// The text of the connection's last error, or of `rc` when SQLite + /// returned no connection. + fn last_error(&self, rc: c_int) -> String { + if self.0.is_null() { + return result_text(rc); + } + // SAFETY: the connection is open; `sqlite3_errmsg` returns a + // NUL-terminated string that SQLite owns, copied here at once. + unsafe { CStr::from_ptr(ffi::sqlite3_errmsg(self.0)) } + .to_string_lossy() + .into_owned() + } + } + + impl Drop for Connection { + fn drop(&mut self) { + // SAFETY: the connection is not used after this, and closing a + // null pointer does nothing. The connection has no prepared + // statement, and `copy` finishes its backup before the + // connections are dropped, so the close does not return + // SQLITE_BUSY and leave the connection open. + unsafe { ffi::sqlite3_close(self.0) }; + } + } + + /// The English text of a result code. + fn result_text(rc: c_int) -> String { + // SAFETY: `sqlite3_errstr` takes any result code and returns a + // static NUL-terminated string. + unsafe { CStr::from_ptr(ffi::sqlite3_errstr(rc)) } + .to_string_lossy() + .into_owned() + } + + /// Copy the database at `source` into the database file at `target`, + /// which is empty, in one step: the source is read in one read + /// transaction, so the target is the database as that transaction saw + /// it. Waits up to `locked_for` while another connection holds a lock + /// the backup needs. The target is then put in rollback-journal mode: + /// the backup copies page 1 with the source's WAL mark, and a WAL + /// database cannot be read without a `-shm` file beside it, so a copy + /// in a directory its reader may not write could not be opened. + pub(super) fn copy(source: &Path, target: &Path, locked_for: Duration) -> Result<(), String> { + let from = Connection::open(source)?; + let to = Connection::open(target)?; + // SAFETY: both connections are open, used by this thread alone, and + // outlive the backup, which is finished below. + let backup = + unsafe { ffi::sqlite3_backup_init(to.0, c"main".as_ptr(), from.0, c"main".as_ptr()) }; + if backup.is_null() { + return Err(to.last_error(ffi::SQLITE_ERROR)); + } + let deadline = Instant::now() + locked_for; + let stepped = loop { + // SAFETY: the backup is live until it is finished below. + let rc = unsafe { ffi::sqlite3_backup_step(backup, -1) }; + if matches!(rc, ffi::SQLITE_BUSY | ffi::SQLITE_LOCKED) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(100)); + continue; + } + break rc; + }; + // SAFETY: the backup is live and is not used after this call, which + // frees it. It returns the last step's result unless that was + // SQLITE_DONE, and leaves a failed step's error on `to`. + let finished = unsafe { ffi::sqlite3_backup_finish(backup) }; + match stepped { + ffi::SQLITE_DONE if finished == ffi::SQLITE_OK => {} + ffi::SQLITE_DONE => return Err(to.last_error(finished)), + ffi::SQLITE_BUSY | ffi::SQLITE_LOCKED => { + return Err(format!( + "another connection held a lock on the database for {} s", + locked_for.as_secs() + )); + } + _ => return Err(to.last_error(stepped)), + } + to.execute(c"PRAGMA journal_mode = DELETE") + } +} + +/// Create `partial` owner-only; it must not exist. +fn create_partial(partial: &Path) -> std::io::Result { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(partial) +} + +/// A line the operator reads while the run holds the lock. A closed stdout +/// does not stop the run. +fn progress(line: &str) { + let _ = writeln!(std::io::stdout().lock(), "{line}"); +} + +/// The directory of `path`, `.` for a bare file name. +fn parent_dir(path: &Path) -> PathBuf { + match path.parent() { + Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(), + _ => PathBuf::from("."), + } +} + +/// Once this run keeps its copy (its first step committed), remove each +/// complete copy an earlier run took that has the same ``, the same +/// size and the same bytes as this run's copy, and no `-wal` or `-journal` +/// file beside it. Such a copy holds the database contents this run's copy +/// holds, so it is a stopped run's copy that this one replaces. Reads the +/// files on a blocking thread. Returns one line naming each file removed, +/// and each one that could not be. +pub(crate) async fn remove_superseded(db: &Path, copy: &Path) -> Vec { + let (db, copy) = (db.to_path_buf(), copy.to_path_buf()); + tokio::task::spawn_blocking(move || superseded(&db, ©)) + .await + .unwrap_or_default() + .into_iter() + .map(|path| match std::fs::remove_file(&path) { + Ok(()) => format!( + "removed {}, a copy an earlier run took of the same database contents, which \ + this run's copy replaces", + path.display() + ), + Err(e) => format!( + "could not remove {} ({e}), a copy an earlier run took of the same database \ + contents, which this run's copy replaces", + path.display() + ), + }) + .collect() +} + +/// The complete copies beside `db` that [`remove_superseded`] removes, in +/// name order. +fn superseded(db: &Path, copy: &Path) -> Vec { + let Some(prefix) = copy_prefix(db) else { + return Vec::new(); + }; + let Some(name) = copy.file_name().and_then(|n| n.to_str()) else { + return Vec::new(); + }; + let Some((from, _)) = copy_versions(&prefix, name) else { + return Vec::new(); + }; + let Ok(size) = std::fs::metadata(copy).map(|m| m.len()) else { + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(parent_dir(db)) else { + return Vec::new(); + }; + // A link under a copy's name to the database, its WAL or its index is + // not a copy, and it is never opened: closing a descriptor of the + // database that this code opened would release every lock the process + // holds on it. A symbolic link is skipped by its own metadata, a hard + // link by its device and inode. + let database: Vec = ["", "-wal", "-shm"] + .iter() + .filter_map(|suffix| { + let mut name = db.as_os_str().to_owned(); + name.push(suffix); + std::fs::metadata(PathBuf::from(name)).ok() + }) + .collect(); + let mut found: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name().and_then(|n| n.to_str()).is_some_and(|n| { + n != name && copy_versions(&prefix, n).is_some_and(|(f, _)| f == from) + }) && !opened(path) + && std::fs::symlink_metadata(path).is_ok_and(|m| { + m.file_type().is_file() + && m.len() == size + && !database.iter().any(|d| same_file(d, &m)) + }) + }) + .collect(); + found.sort(); + found.retain(|path| same_bytes(path, copy)); + found +} + +/// Whether a `-wal` or `-journal` file is beside `copy`: a process opened it, +/// and its contents may not be the main file's alone. +fn opened(copy: &Path) -> bool { + ["-wal", "-journal"].iter().any(|suffix| { + let mut side = copy.as_os_str().to_owned(); + side.push(suffix); + std::fs::symlink_metadata(PathBuf::from(side)).is_ok() + }) +} + +/// Whether `a` and `b` are the metadata of one file. +#[cfg(unix)] +fn same_file(a: &std::fs::Metadata, b: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + a.dev() == b.dev() && a.ino() == b.ino() +} + +/// On Windows a lock belongs to the handle that took it, so closing +/// another handle of the file releases nothing; no link is told apart. +#[cfg(not(unix))] +fn same_file(_: &std::fs::Metadata, _: &std::fs::Metadata) -> bool { + false +} + +/// Whether the files at `a` and `b` hold the same bytes; `false` when either +/// cannot be read. +fn same_bytes(a: &Path, b: &Path) -> bool { + let (Ok(mut a), Ok(mut b)) = (File::open(a), File::open(b)) else { + return false; + }; + let (mut x, mut y) = (vec![0; 1 << 20], vec![0; 1 << 20]); + loop { + let (Ok(n), Ok(m)) = (fill(&mut a, &mut x), fill(&mut b, &mut y)) else { + return false; + }; + if x[..n] != y[..m] { + return false; + } + if n == 0 { + return true; + } + } +} + +/// Read from `file` until `buf` is full or the file ends; the bytes read. +fn fill(file: &mut File, buf: &mut [u8]) -> std::io::Result { + let mut filled = 0; + while filled < buf.len() { + match file.read(&mut buf[filled..])? { + 0 => break, + n => filled += n, + } + } + Ok(filled) +} + +/// `.pre-migration-`, which every copy's name starts with. +fn copy_prefix(db: &Path) -> Option { + db.file_name() + .and_then(|n| n.to_str()) + .map(|name| format!("{name}.pre-migration-")) +} + +/// The `` and `` of a complete copy's name, +/// `-to--`; `None` for any other name, +/// a partial copy's included. +fn copy_versions<'a>(prefix: &str, name: &'a str) -> Option<(&'a str, u32)> { + let rest = name.strip_prefix(prefix)?; + let (from, rest) = rest.split_once("-to-")?; + let (to, stamp) = rest.split_once('-')?; + let stamp = stamp.as_bytes(); + let stamped = stamp.len() == 16 + && stamp[8] == b'T' + && stamp[15] == b'Z' + && stamp[..8] + .iter() + .chain(&stamp[9..15]) + .all(u8::is_ascii_digit); + if !stamped || from.is_empty() { + return None; + } + Some((from, to.parse().ok()?)) +} + +/// Remove every `.pre-migration-*.partial` beside `db`, with the files +/// SQLite keeps beside it: the partial copies of runs that were stopped +/// while they copied. +/// Every run that holds the database's lock calls it, whether or not it +/// copies, so a run with nothing to apply removes them too. Returns one +/// line naming each file removed, and each one that could not be. +pub(crate) fn remove_stopped_partials(db: &Path) -> Vec { + let Some(name) = db.file_name().and_then(|n| n.to_str()) else { + return Vec::new(); + }; + let dir = match db.parent() { + Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(), + _ => PathBuf::from("."), + }; + let prefix = format!("{name}.pre-migration-"); + let Ok(entries) = std::fs::read_dir(&dir) else { + return Vec::new(); + }; + let mut found: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name().and_then(|n| n.to_str()).is_some_and(|n| { + n.starts_with(&prefix) + && (n.ends_with(".partial") + || PARTIAL_SIDE_FILES + .iter() + .any(|suffix| n.ends_with(&format!(".partial{suffix}")))) + }) + }) + .collect(); + found.sort(); + found + .into_iter() + .map(|path| match std::fs::remove_file(&path) { + Ok(()) => format!( + "removed {}, left by an earlier run that was stopped while it copied the database", + path.display() + ), + Err(e) => format!( + "could not remove {} ({e}), left by an earlier run that was stopped while it \ + copied the database", + path.display() + ), + }) + .collect() +} + +/// Remove a copy that a run took and then did not need: it stopped before +/// it committed anything. Returns the line to print when the copy stays. +pub(crate) fn remove_copy(copy: &Path) -> Result<(), String> { + std::fs::remove_file(copy).map_err(|e| { + format!( + "the copy taken before this run, {}, could not be removed ({e}); nothing was \ + committed, so it is not needed", + copy.display() + ) + }) +} + +/// `.partial`, where the copy is written before it is renamed. +fn partial_path(copy: &Path) -> PathBuf { + let mut name = copy.as_os_str().to_owned(); + name.push(".partial"); + PathBuf::from(name) +} + +/// The files SQLite may keep beside a partial copy: the rollback journal +/// while the backup writes, and the WAL and its index while the copy is +/// switched to rollback-journal mode. +const PARTIAL_SIDE_FILES: [&str; 3] = ["-journal", "-wal", "-shm"]; + +/// Remove a partial copy and the files SQLite keeps beside it. +fn remove_partial(partial: &Path) { + let _ = std::fs::remove_file(partial); + for suffix in PARTIAL_SIDE_FILES { + let mut side = partial.as_os_str().to_owned(); + side.push(suffix); + let _ = std::fs::remove_file(PathBuf::from(side)); + } +} + +/// Sync the file at `path` to disk on a blocking thread, as the backup +/// runs. The file is opened for writing: on Windows `sync_all` is +/// `FlushFileBuffers`, which needs a handle with write access. +async fn sync_file(path: &Path) -> Result<(), MigrationError> { + sync(path, true).await +} + +/// Sync the directory `dir`, so that a rename in it is durable. A +/// directory is opened for reading, which Windows does not do with these +/// options; the caller ignores the error. +async fn sync_dir(dir: &Path) -> Result<(), MigrationError> { + sync(dir, false).await +} + +async fn sync(path: &Path, write: bool) -> Result<(), MigrationError> { + let owned = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + std::fs::OpenOptions::new() + .read(!write) + .write(write) + .open(&owned) + .and_then(|f| f.sync_all()) + }) + .await + .map_err(std::io::Error::other) + .and_then(|synced| synced) + .map_err(|e| MigrationError::store(format!("sync {}", path.display()), e)) +} + +/// Stop, with the figures, when `dir` has less than three times the +/// database's size free. +fn refuse_short_space(live: u64, free: u64, dir: &Path) -> Result<(), MigrationError> { + let needed = live.saturating_mul(SPACE_FACTOR); + if free < needed { + return Err(MigrationError::Refused(format!( + "migrating needs three times the database's size free, {needed} bytes for a \ + {live}-byte database: the copy taken before migrating, and the rewrite and WAL of \ + migration 1's backup_items rebuild. {} has {free} bytes free; free some space and \ + run the migration again", + dir.display() + ))); + } + Ok(()) +} + +/// `.pre-migration--to--`, with `at` in UTC. +pub(crate) fn copy_path(db: &Path, from: &str, to: u32, at: SystemTime) -> PathBuf { + let t = time::OffsetDateTime::from(at); + let stamp = format!( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ); + let mut name = db.as_os_str().to_owned(); + name.push(format!(".pre-migration-{from}-to-{to}-{stamp}")); + PathBuf::from(name) +} + +async fn live_size(pool: &SqlitePool) -> Result { + let pages: i64 = sqlx::query_scalar("PRAGMA page_count") + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("PRAGMA page_count", e))?; + let page_size: i64 = sqlx::query_scalar("PRAGMA page_size") + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("PRAGMA page_size", e))?; + Ok(u64::try_from(pages.saturating_mul(page_size)).unwrap_or(0)) +} + +/// Bytes available to this process on the filesystem holding `dir`. +#[cfg(unix)] +// The statvfs field types differ by platform. +#[allow(clippy::useless_conversion, clippy::unnecessary_cast)] +pub(crate) fn free_space(dir: &Path) -> Option { + use std::os::unix::ffi::OsStrExt; + let path = std::ffi::CString::new(dir.as_os_str().as_bytes()).ok()?; + // SAFETY: an all-zero statvfs is a valid value of a plain C struct. + let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; + // SAFETY: `path` is a NUL-terminated string and `stat` a valid struct, + // both live for the duration of the call. + let rc = unsafe { libc::statvfs(path.as_ptr(), &raw mut stat) }; + (rc == 0).then(|| (stat.f_bavail as u64).saturating_mul(stat.f_frsize as u64)) +} + +#[cfg(not(unix))] +pub(crate) fn free_space(_dir: &Path) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::{copy_path, copy_versions, free_space, refuse_short_space}; + use std::path::Path; + use std::time::{Duration, SystemTime}; + + /// A lock another connection holds past the wait stops the copy, and + /// the error says how long it waited: `sqlite3_backup_finish` returns + /// the last step's SQLITE_BUSY, which alone reads "database is locked". + #[tokio::test] + async fn a_lock_held_past_the_wait_stops_the_copy_and_says_so() { + let db = crate::migrations::tests::temp_db("copy-busy"); + let pool = crate::migrations::tests::plain_pool(&db).await; + sqlx::raw_sql( + "PRAGMA journal_mode = DELETE; CREATE TABLE t (x INTEGER); INSERT INTO t VALUES (1);", + ) + .execute(&pool) + .await + .expect("a database"); + let mut holder = pool.acquire().await.expect("a connection"); + sqlx::raw_sql("BEGIN EXCLUSIVE") + .execute(&mut *holder) + .await + .expect("an exclusive lock"); + let target = std::path::PathBuf::from(format!("{}.copy", db.display())); + super::create_partial(&target).expect("the target"); + let source = db.to_path_buf(); + let started = std::time::Instant::now(); + let err = tokio::task::spawn_blocking(move || { + super::backup::copy(&source, &target, Duration::from_secs(1)) + }) + .await + .expect("the copy's thread") + .expect_err("the lock is held"); + assert_eq!( + err, + "another connection held a lock on the database for 1 s" + ); + assert!( + started.elapsed() < Duration::from_secs(15), + "{:?}", + started.elapsed() + ); + sqlx::raw_sql("ROLLBACK") + .execute(&mut *holder) + .await + .expect("release"); + drop(holder); + pool.close().await; + } + + #[test] + fn a_complete_copy_name_gives_its_versions_and_no_other_name_does() { + let prefix = "x.sqlite.pre-migration-"; + let name = |rest: &str| format!("{prefix}{rest}"); + assert_eq!( + copy_versions(prefix, &name("legacy-to-4-20261029T061706Z")), + Some(("legacy", 4)) + ); + assert_eq!( + copy_versions(prefix, &name("3-to-4-20261029T061706Z")), + Some(("3", 4)) + ); + for other in [ + "3-to-4-20261029T061706Z.partial", + "3-to-4-20261029T061706Z.partial-journal", + "3-to-4-20261029T061706Z.partial-wal", + "3-to-4-20261029T061706Z.partial-shm", + "3-to-4-20261029T061706Z-wal", + "3-to-4-2026102", + "-to-4-20261029T061706Z", + "3-to-x-20261029T061706Z", + "3-4-20261029T061706Z", + ] { + assert_eq!(copy_versions(prefix, &name(other)), None, "{other}"); + } + assert_eq!( + copy_versions(prefix, "y.sqlite.pre-migration-3-to-4-20261029T061706Z"), + None + ); + } + + #[test] + fn a_short_filesystem_is_refused_with_the_figures() { + let err = refuse_short_space(4096, 12287, Path::new("/data")).expect_err("short"); + let message = err.to_string(); + assert!( + message.contains("three times the database's size"), + "{message}" + ); + assert!( + message.contains("12288 bytes for a 4096-byte database"), + "{message}" + ); + assert!(message.contains("/data has 12287 bytes free"), "{message}"); + assert_eq!(err.exit_code(), extenddb_storage::migrations::exit::ERROR); + refuse_short_space(4096, 12288, Path::new("/data")).expect("enough"); + } + + #[test] + fn the_free_space_probe_reads_the_filesystem() { + let free = free_space(&std::env::temp_dir()); + if cfg!(unix) { + assert!(free.is_some_and(|n| n > 0), "{free:?}"); + } + } + + #[test] + fn copy_names_carry_the_utc_time() { + // 2026-10-29T06:17:06Z + let at = SystemTime::UNIX_EPOCH + Duration::from_secs(1_793_254_626); + assert_eq!( + copy_path(Path::new("/var/db/x.sqlite"), "legacy", 4, at), + Path::new("/var/db/x.sqlite.pre-migration-legacy-to-4-20261029T061706Z") + ); + } +} diff --git a/crates/storage-sqlite/src/migrations/crash_tests.rs b/crates/storage-sqlite/src/migrations/crash_tests.rs new file mode 100644 index 000000000..4a27582b6 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/crash_tests.rs @@ -0,0 +1,242 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A migrator process killed inside a body. The test binary runs itself as +//! the child, selected by an environment variable. + +use std::path::PathBuf; +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + CatalogStatus, CodeBody, Done, MigrationBackend, MigrationError, MigrationId, Runner, Step, +}; + +use super::tests::{ + DEV_0_1_14, LATEST, RELEASE, catalog_store, count, load_dump, run, scalar, status, temp_db, + test_row, version_row, +}; +use super::{SqliteMigrations, SqliteStores}; + +/// Set to the database path in the child process of the kill test. +const CRASH_CHILD: &str = "EXTENDDB_MIGRATION_CRASH_CHILD"; + +/// Writes inside the runner's transaction, signals that it has, and waits +/// to be killed. +struct WriteAndWait; + +#[async_trait] +impl CodeBody for WriteAndWait { + fn fingerprint(&self) -> &'static str { + "test/write-and-wait" + } + async fn detect(&self, _: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + async fn run(&self, stores: &SqliteStores) -> Result { + sqlx::raw_sql( + "UPDATE tables SET attribute_definitions = '[]'; \ + INSERT INTO settings (key, value) VALUES ('killed_body', 'wrote');", + ) + .execute(&stores.pool) + .await + .map_err(|e| MigrationError::store("write", e))?; + let db = std::env::var(CRASH_CHILD).unwrap_or_default(); + std::fs::write(format!("{db}.ready"), b"").expect("ready file"); + tokio::time::sleep(Duration::from_secs(120)).await; + Ok(Done::default()) + } +} + +/// The child half of the kill test; does nothing in an ordinary run. +#[tokio::test] +async fn kill_test_child_process() { + let Ok(db) = std::env::var(CRASH_CHILD) else { + return; + }; + let migrations = SqliteMigrations::open(&db).await.expect("open"); + let store = catalog_store(std::path::Path::new(&db)); + let _ = migrations + .apply( + &store, + MigrationId::SortKeyDefinitions, + Step::Code(&WriteAndWait), + test_row(3), + Some(3), + ) + .await; +} + +/// The child process, killed when dropped, so a failed assertion does not +/// leave it sleeping. +struct Child(std::process::Child); + +impl Drop for Child { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// A migrator killed with SIGKILL inside a code body leaves the database as +/// the last committed migration left it, and a rerun completes. +#[cfg(unix)] +#[tokio::test] +async fn a_migrator_killed_inside_a_body_leaves_nothing_and_a_rerun_completes() { + let db = temp_db("killed"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let migrations = SqliteMigrations::open(&location).await.expect("open"); + Runner::new(&migrations, RELEASE) + .migrate_through(MigrationId::Baseline) + .await + .expect("migration 1"); + migrations.close().await; + let definitions = scalar(&db, "SELECT attribute_definitions FROM tables").await; + + let mut child = Child( + std::process::Command::new(std::env::current_exe().expect("test binary")) + .args([ + "--exact", + "migrations::crash_tests::kill_test_child_process", + "--nocapture", + ]) + .env(CRASH_CHILD, &location) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("child"), + ); + let ready = PathBuf::from(format!("{location}.ready")); + let started = std::time::Instant::now(); + while !ready.exists() { + assert!( + child.0.try_wait().expect("child state").is_none(), + "the child exited early" + ); + assert!( + started.elapsed() < Duration::from_secs(60), + "the child never wrote" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + child.0.kill().expect("SIGKILL"); + child.0.wait().expect("reaped"); + + assert_eq!( + scalar(&db, "SELECT attribute_definitions FROM tables").await, + definitions + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM settings WHERE key = 'killed_body'" + ) + .await, + 0 + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM extenddb_migrations").await, + 1 + ); + assert_eq!(version_row(&db).await.as_deref(), Some("1")); + run(&db).await.expect("rerun"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); +} + +/// Set to the copy's path in the child process of the copy kill test. +const COPY_CHILD: &str = "EXTENDDB_MIGRATION_COPY_CHILD"; + +/// Signals that the partial copy is written and synced, and waits to be +/// killed before the rename. +fn wait_before_rename(partial: &std::path::Path) { + std::fs::write(format!("{}.ready", partial.display()), b"").expect("ready file"); + std::thread::sleep(Duration::from_secs(120)); +} + +/// The child half of the copy kill test; does nothing in an ordinary run. +#[tokio::test] +async fn copy_kill_test_child_process() { + let Ok(copy) = std::env::var(COPY_CHILD) else { + return; + }; + let db = std::env::var(CRASH_CHILD).expect("database"); + let migrations = SqliteMigrations::open(&db).await.expect("open"); + let _ = super::copy::pre_migration_copy( + migrations.pool(), + std::path::Path::new(&db), + std::path::Path::new(©), + super::copy::Hooks { + before_rename: wait_before_rename, + ..super::copy::Hooks::RUN + }, + ) + .await; +} + +/// A migrator killed after it wrote its copy and before the rename leaves +/// no file under the copy's name, so an operator restoring "the copy" can +/// only find complete ones; the database is untouched. +#[cfg(unix)] +#[tokio::test] +async fn a_copy_interrupted_by_a_kill_leaves_nothing_under_the_copy_name() { + let db = temp_db("killed-copy"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + let mut child = Child( + std::process::Command::new(std::env::current_exe().expect("test binary")) + .args([ + "--exact", + "migrations::crash_tests::copy_kill_test_child_process", + "--nocapture", + ]) + .env(CRASH_CHILD, &location) + .env(COPY_CHILD, ©) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("child"), + ); + let ready = PathBuf::from(format!("{}.partial.ready", copy.display())); + let started = std::time::Instant::now(); + while !ready.exists() { + assert!( + child.0.try_wait().expect("child state").is_none(), + "the child exited early" + ); + assert!( + started.elapsed() < Duration::from_secs(60), + "the child never copied" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + child.0.kill().expect("SIGKILL"); + child.0.wait().expect("reaped"); + + assert!(!copy.exists(), "a file has the copy's name"); + let partial = PathBuf::from(format!("{}.partial", copy.display())); + assert!(partial.exists()); + assert_eq!(version_row(&db).await.as_deref(), Some("0.0.3")); + assert!(matches!( + status(&db).await, + CatalogStatus::Behind { version: None, .. } + )); + + // The next run, in a later second and so under another copy name, + // removes the stopped run's partial copy and its journal, and names it. + super::tests::next_second().await; + let report = super::tests::run(&db).await.expect("the next run"); + let journal = PathBuf::from(format!("{}.partial-journal", copy.display())); + assert!(!partial.exists(), "the stopped run's partial copy is left"); + assert!(!journal.exists(), "the stopped run's journal is left"); + let removed = format!("removed {}, left by an earlier run", partial.display()); + assert!( + report.notes.iter().any(|n| n.starts_with(&removed)), + "{:?}", + report.notes + ); +} diff --git a/crates/storage-sqlite/src/migrations/golden_tests.rs b/crates/storage-sqlite/src/migrations/golden_tests.rs new file mode 100644 index 000000000..a2e458ebb --- /dev/null +++ b/crates/storage-sqlite/src/migrations/golden_tests.rs @@ -0,0 +1,629 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Golden tests for SQLite migrations 2 to 4, each run on a released catalog +//! with the damage or the content its body handles, and migration 1's +//! accepted and converged pre-release shapes. + +use std::path::Path; + +use extenddb_storage::migrations::{CatalogStatus, MigrationError, exit}; + +use super::tests::{ + DEV_0_1_7, DEV_0_1_14, LATEST, copies, count, exec, load_dump, plain_pool, reconverge, run, + scalar, status, temp_db, version_row, +}; + +/// Rows of a query, one tab-separated line each: text as it is (JSON with +/// sorted keys, because the workspace build enables serde_json's insertion +/// order and a single-crate build does not), integers and reals in decimal, +/// blobs in hex, and NULL as `NULL`. +async fn dump(db: &Path, sql: &str) -> String { + let pool = plain_pool(db).await; + let rows = sqlx::query(sql).fetch_all(&pool).await.expect(sql); + pool.close().await; + let mut out = format!("-- {sql}\n"); + for row in rows { + use sqlx::Row; + let fields: Vec = (0..row.columns().len()) + .map(|i| { + if let Ok(text) = row.try_get::, _>(i) { + return text.map_or_else( + || "NULL".to_owned(), + |text| match serde_json::from_str::(&text) { + Ok(value) if value.is_object() || value.is_array() => canonical(&value), + _ => text, + }, + ); + } + if let Ok(n) = row.try_get::, _>(i) { + return n.map_or_else(|| "NULL".to_owned(), |n| n.to_string()); + } + if let Ok(x) = row.try_get::, _>(i) { + return x.map_or_else(|| "NULL".to_owned(), |x| x.to_string()); + } + let bytes = row.try_get::>, _>(i).expect("a blob"); + bytes.map_or_else( + || "NULL".to_owned(), + |b| b.iter().map(|byte| format!("{byte:02x}")).collect(), + ) + }) + .collect(); + out.push_str(&fields.join("\t")); + out.push('\n'); + } + out +} + +/// A JSON value with object keys in sorted order. +fn canonical(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort_unstable(); + let members: Vec = keys + .into_iter() + .map(|k| { + format!( + "{}:{}", + serde_json::Value::from(k.as_str()), + canonical(&map[k]) + ) + }) + .collect(); + format!("{{{}}}", members.join(",")) + } + serde_json::Value::Array(items) => { + format!( + "[{}]", + items.iter().map(canonical).collect::>().join(",") + ) + } + other => other.to_string(), + } +} + +/// Compare a body's result with its committed dump under +/// `tests/fixtures/golden/`. `EXTENDDB_BLESS_EXPECTED=1` rewrites the file. +fn golden(name: &str, actual: &str) { + let path = + Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("tests/fixtures/golden/{name}.txt")); + if super::tests::bless() { + std::fs::create_dir_all(path.parent().expect("dir")).expect("golden dir"); + std::fs::write(&path, actual).expect("write golden"); + } + let expected = std::fs::read_to_string(&path).unwrap_or_default(); + assert_eq!( + expected, + actual, + "{name} differs from {}; regenerate with EXTENDDB_BLESS_EXPECTED=1", + path.display() + ); +} + +#[tokio::test] +async fn the_golden_dump_writes_every_column_type() { + let db = temp_db("dump"); + exec( + &db, + "CREATE TABLE t (a TEXT, b INTEGER, c REAL, d BLOB, e TEXT)", + ) + .await; + exec(&db, "INSERT INTO t VALUES ('x', 7, 1.5, x'0aff', NULL)").await; + assert_eq!( + dump(&db, "SELECT a, b, c, d, e FROM t").await, + "-- SELECT a, b, c, d, e FROM t\nx\t7\t1.5\t0aff\tNULL\n" + ); +} + +/// The account and IAM user every dev-image fixture holds. +async fn principal(db: &Path) -> (String, String) { + let pool = plain_pool(db).await; + let row: (String, String) = + sqlx::query_as("SELECT account_id, user_name FROM iam_users LIMIT 1") + .fetch_one(&pool) + .await + .expect("an IAM user"); + pool.close().await; + row +} + +async fn put_policy(db: &Path, principal_type: &str, principal_name: &str, name: &str, doc: &str) { + let (account, _) = principal(db).await; + let pool = plain_pool(db).await; + sqlx::query( + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, \ + policy_document) VALUES (?, ?, ?, ?, ?)", + ) + .bind(&account) + .bind(principal_type) + .bind(principal_name) + .bind(name) + .bind(doc) + .execute(&pool) + .await + .expect("policy"); + pool.close().await; +} + +const TAGGED: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}"#; +const COLLIDING: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod","dynamodb:ResourceTag/env":"dev"}}}]}"#; + +/// The six places a policy is stored: user, group and role inline policies, +/// user and role permissions boundaries, and a session policy. +async fn tag_every_place(db: &Path) { + let (account, user) = principal(db).await; + let pool = plain_pool(db).await; + sqlx::query("INSERT INTO iam_groups (account_id, group_name, group_arn) VALUES (?, 'devs', ?)") + .bind(&account) + .bind(format!("arn:aws:iam::{account}:group/devs")) + .execute(&pool) + .await + .expect("group"); + sqlx::query( + "INSERT INTO iam_roles (account_id, role_name, role_arn, trust_policy) \ + VALUES (?, 'app', ?, '{}')", + ) + .bind(&account) + .bind(format!("arn:aws:iam::{account}:role/app")) + .execute(&pool) + .await + .expect("role"); + for (principal_type, name) in [("user", user.as_str()), ("role", "app")] { + sqlx::query( + "INSERT INTO iam_permissions_boundaries (account_id, principal_type, \ + principal_name, policy_document) VALUES (?, ?, ?, ?)", + ) + .bind(&account) + .bind(principal_type) + .bind(name) + .bind(TAGGED) + .execute(&pool) + .await + .expect("boundary"); + } + sqlx::query( + "INSERT INTO iam_sessions (session_token, access_key_id, secret_key_encrypted, \ + account_id, role_name, session_name, session_policy, expires_at) \ + VALUES ('token-1', 'ASIAFIXTURE0001', x'00', ?, 'app', 'session', ?, \ + '2999-01-01T00:00:00.000Z')", + ) + .bind(&account) + .bind(TAGGED) + .execute(&pool) + .await + .expect("session"); + pool.close().await; + put_policy(db, "user", &user, "tagged-user", TAGGED).await; + put_policy(db, "group", "devs", "tagged-group", TAGGED).await; + put_policy(db, "role", "app", "tagged-role", TAGGED).await; +} + +#[tokio::test] +async fn migration_2_rewrites_resource_tag_condition_keys_in_every_place() { + let db = temp_db("m2"); + load_dump(&db, DEV_0_1_14).await; + tag_every_place(&db).await; + run(&db).await.expect("migrate"); + for (place, sql) in [ + ( + "inline", + "SELECT COUNT(*) FROM iam_policies WHERE policy_document LIKE '%dynamodb:ResourceTag%'", + ), + ( + "boundary", + "SELECT COUNT(*) FROM iam_permissions_boundaries WHERE policy_document LIKE '%dynamodb:ResourceTag%'", + ), + ( + "session", + "SELECT COUNT(*) FROM iam_sessions WHERE session_policy LIKE '%dynamodb:ResourceTag%'", + ), + ] { + assert_eq!( + count(&db, sql).await, + 0, + "{place} policies keep the dynamodb: form" + ); + } + let document = scalar( + &db, + "SELECT policy_document FROM iam_policies WHERE policy_name = 'tagged-role'", + ) + .await + .expect("policy"); + let document: serde_json::Value = serde_json::from_str(&document).expect("json"); + assert_eq!( + document["Statement"][0]["Condition"], + serde_json::json!({"StringEquals": {"aws:ResourceTag/env": "prod"}}) + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 2") + .await + .expect("row"); + assert_eq!(notes.lines().count(), 6, "{notes}"); + let mut text = dump( + &db, + "SELECT account_id, principal_type, principal_name, policy_name, policy_document \ + FROM iam_policies ORDER BY principal_type, policy_name", + ) + .await; + text.push_str( + &dump( + &db, + "SELECT account_id, principal_type, principal_name, policy_document \ + FROM iam_permissions_boundaries ORDER BY principal_type", + ) + .await, + ); + text.push_str( + &dump( + &db, + "SELECT session_token, role_name, session_policy FROM iam_sessions \ + ORDER BY session_token", + ) + .await, + ); + text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 2").await); + golden("0002_resource_tag_condition_keys", &text); +} + +/// A collision is found by `detect` before anything is written: no copy, no +/// ledger, and the released version row untouched. +#[tokio::test] +async fn migration_2_collisions_stop_the_run_before_any_write() { + let db = temp_db("m2-collision"); + load_dump(&db, DEV_0_1_14).await; + let (_, user) = principal(&db).await; + put_policy(&db, "user", &user, "colliding", COLLIDING).await; + let found = status(&db).await; + let CatalogStatus::NeedsAttention { items } = &found else { + panic!("{found:?}"); + }; + assert!(items[0].contains("policy colliding"), "{items:?}"); + assert_eq!(found.exit_code(), exit::ATTENTION); + let err = run(&db).await.expect_err("refused"); + assert!( + matches!( + err, + MigrationError::Status(CatalogStatus::NeedsAttention { .. }) + ), + "{err:?}" + ); + assert_eq!(err.exit_code(), exit::ATTENTION); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM sqlite_master WHERE name = 'extenddb_migrations'" + ) + .await, + 0 + ); + assert_eq!(version_row(&db).await.as_deref(), Some("0.0.3")); + assert!(copies(&db).is_empty(), "no copy before a refusal"); +} + +/// The text of every stored policy, in a fixed order. +async fn every_policy_document(db: &Path) -> String { + let mut text = dump( + db, + "SELECT principal_type, principal_name, policy_name, policy_document \ + FROM iam_policies ORDER BY principal_type, principal_name, policy_name", + ) + .await; + text.push_str( + &dump( + db, + "SELECT principal_type, principal_name, policy_document \ + FROM iam_permissions_boundaries ORDER BY principal_type, principal_name", + ) + .await, + ); + text.push_str( + &dump( + db, + "SELECT session_token, session_policy FROM iam_sessions ORDER BY session_token", + ) + .await, + ); + text +} + +/// Record what a release before 1.0 leaves when it writes to a migrated +/// catalog, then reconverge: migration 2 is named as skipped, and the +/// catalog is current again. +async fn reconverge_after_an_older_release_wrote(db: &Path) { + exec( + db, + "INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied')", + ) + .await; + assert_eq!(status(db).await.label(), "WrittenByOlderRelease"); + let report = reconverge(db).await.expect("reconverge"); + assert!(report.applied.iter().all(|s| s.id != 2), "{report:?}"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), + "{report:?}" + ); + assert_eq!(status(db).await, CatalogStatus::Current { version: LATEST }); +} + +/// From 1.0 on a `dynamodb:ResourceTag` key written through the API has no +/// effect, as on the service. Migration 2 is not convergent, so a reconverge +/// leaves a policy written after it, in each of the six places, with the +/// text it was written with, and so with the decision it had. +#[tokio::test] +async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { + let db = temp_db("m2-reconverge"); + load_dump(&db, DEV_0_1_14).await; + run(&db).await.expect("upgrade"); + tag_every_place(&db).await; + let before = every_policy_document(&db).await; + assert!(before.contains("dynamodb:ResourceTag/env"), "{before}"); + reconverge_after_an_older_release_wrote(&db).await; + assert_eq!(every_policy_document(&db).await, before); +} + +/// A policy written after migration 2 with both forms on one key, which +/// the rewrite would refuse, does not stop a reconverge. +#[tokio::test] +async fn a_reconverge_is_not_stopped_by_a_policy_migration_2_would_refuse() { + let db = temp_db("m2-reconverge-c"); + load_dump(&db, DEV_0_1_14).await; + run(&db).await.expect("upgrade"); + let (_, user) = principal(&db).await; + put_policy(&db, "user", &user, "colliding", COLLIDING).await; + let before = every_policy_document(&db).await; + reconverge_after_an_older_release_wrote(&db).await; + assert_eq!(every_policy_document(&db).await, before); +} + +#[tokio::test] +async fn migration_3_restores_sort_key_definitions_in_tables_and_backups() { + let db = temp_db("m3"); + load_dump(&db, DEV_0_1_14).await; + // The pre-fix UpdateTable kept only the definitions its request named. + let damaged = r#"[{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]"#; + let key_schema = + r#"[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]"#; + let pool = plain_pool(&db).await; + let (table_id, account): (String, String) = + sqlx::query_as("SELECT table_id, account_id FROM tables") + .fetch_one(&pool) + .await + .expect("table"); + sqlx::query("UPDATE tables SET attribute_definitions = ?") + .bind(damaged) + .execute(&pool) + .await + .expect("damage"); + // b1: an item gives the type. b2: no item and no source table. b3: no + // item, and the source table's key column gives the type. + for (arn, source, with_item) in [ + ("arn:b1", table_id.as_str(), true), + ("arn:b2", "6f9619ff-8b86-4011-b42d-00c04fc964ff", false), + ("arn:b3", table_id.as_str(), false), + ] { + sqlx::query( + "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, key_schema, \ + attribute_definitions) VALUES (?, 'b', ?, 'cu_orders', ?, ?, ?)", + ) + .bind(arn) + .bind(source) + .bind(&account) + .bind(key_schema) + .bind(damaged) + .execute(&pool) + .await + .expect("backup"); + if with_item { + sqlx::query( + "INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES (?, 'a', NULL, ?)", + ) + .bind(arn) + .bind(r#"{"pk":{"S":"a"},"sk":{"S":"1"}}"#) + .execute(&pool) + .await + .expect("item"); + } + } + pool.close().await; + + run(&db).await.expect("migrate"); + let restored = scalar(&db, "SELECT attribute_definitions FROM tables") + .await + .expect("row"); + assert!( + restored.contains(r#"{"AttributeName":"sk","AttributeType":"S"}"#), + "{restored}" + ); + for arn in ["arn:b1", "arn:b3"] { + let repaired = scalar( + &db, + &format!("SELECT attribute_definitions FROM backups WHERE backup_arn = '{arn}'"), + ) + .await + .expect("backup"); + assert!( + repaired.contains(r#"{"AttributeName":"sk","AttributeType":"S"}"#), + "{arn}: {repaired}" + ); + } + let b2 = scalar( + &db, + "SELECT attribute_definitions FROM backups WHERE backup_arn = 'arn:b2'", + ) + .await + .expect("b2"); + assert_eq!( + b2, damaged, + "a backup with no items and no source table is left as it is" + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 3") + .await + .expect("row"); + let expected_notes = [ + "restored cu_orders: sk (S)", + "cu_orders: partition key definition(s) [pk] are absent", + "restored backup arn:b1 of cu_orders: sk (S)", + "backup arn:b2 of cu_orders: sort key(s) [sk] have no attribute definition", + "restored backup arn:b3 of cu_orders: sk (S)", + ]; + for expected in expected_notes { + assert!(notes.contains(expected), "{expected} not in {notes}"); + } + let mut text = dump( + &db, + "SELECT table_name, attribute_definitions FROM tables ORDER BY table_name", + ) + .await; + text.push_str( + &dump( + &db, + "SELECT backup_arn, attribute_definitions FROM backups ORDER BY backup_arn", + ) + .await, + ); + text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 3").await); + golden("0003_sort_key_definitions", &text); +} + +/// Pre-release shapes of `table_restores`: the extra default a pre-release +/// build gave `restore_date_time` is accepted with a note, and a table +/// without `source_table_arn` gains the column. +#[tokio::test] +async fn migration_1_accepts_and_converges_pre_release_table_restores() { + let db = temp_db("pre-release-default"); + load_dump(&db, DEV_0_1_14).await; + exec( + &db, + "CREATE TABLE table_restores (\ + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ + source_backup_arn TEXT NOT NULL, \ + restore_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), \ + source_table_arn TEXT)", + ) + .await; + run(&db).await.expect("accepted"); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1") + .await + .expect("row"); + assert!( + notes.contains("keeps the default a pre-release build gave it"), + "{notes}" + ); + + let db = temp_db("pre-release-no-arn"); + load_dump(&db, DEV_0_1_14).await; + exec( + &db, + "CREATE TABLE table_restores (\ + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ + source_backup_arn TEXT NOT NULL, \ + restore_date_time TEXT NOT NULL); \ + INSERT INTO table_restores SELECT table_id, 'arn:backup', '2026-01-01T00:00:00Z' \ + FROM tables;", + ) + .await; + run(&db).await.expect("converged"); + assert_eq!(status(&db).await.label(), "Current"); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM pragma_table_info('table_restores') \ + WHERE name = 'source_table_arn'" + ) + .await, + 1 + ); + assert_eq!( + scalar(&db, "SELECT source_backup_arn FROM table_restores") + .await + .as_deref(), + Some("arn:backup"), + "the row is kept" + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1") + .await + .expect("row"); + assert!( + notes.contains("added table_restores.source_table_arn"), + "{notes}" + ); +} + +#[tokio::test] +async fn migration_4_adds_the_base_key_index_to_index_tables_created_before_v0_1_8() { + let db = temp_db("m4"); + load_dump(&db, DEV_0_1_7).await; + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'idx_base_key_%'" + ) + .await, + 0 + ); + // A catalog row whose index table is gone is skipped and noted, and so + // is the index table of a table being deleted. + exec( + &db, + "INSERT INTO indexes (table_id, index_id, index_name, index_type, key_schema, projection) \ + SELECT table_id, '6f9619ff-8b86-4011-b42d-00c04fc964ff', 'gone', 'GSI', '[]', '{}' FROM tables; \ + CREATE TEMP TABLE going AS SELECT * FROM tables; \ + UPDATE going SET table_id = '0b7e9a52-3c1d-4f6e-9a2b-5d8c7e6f1a40', \ + table_name = 'going', table_status = 'DELETING'; \ + INSERT INTO tables SELECT * FROM going; \ + INSERT INTO indexes (table_id, index_id, index_name, index_type, key_schema, projection) \ + VALUES ('0b7e9a52-3c1d-4f6e-9a2b-5d8c7e6f1a40', '9d1f3b6a-2e4c-4a8b-b7d0-6c5e4f3a2b10', \ + 'by_gpk', 'GSI', '[]', '{}'); \ + CREATE TABLE \"_ddb_9d1f3b6a-2e4c-4a8b-b7d0-6c5e4f3a2b10\" (\ + pk TEXT NOT NULL, base_pk TEXT NOT NULL, base_sk_s TEXT NOT NULL, \ + PRIMARY KEY (pk, base_pk, base_sk_s));", + ) + .await; + run(&db).await.expect("migrate"); + let pool = plain_pool(&db).await; + let created: Vec<(String, String)> = sqlx::query_as( + "SELECT i.index_name, m.sql FROM indexes i \ + JOIN sqlite_master m ON m.name = 'idx_base_key_' || i.index_id ORDER BY i.index_name", + ) + .fetch_all(&pool) + .await + .expect("indexes"); + let ids: Vec<(String, String)> = sqlx::query_as( + "SELECT index_name, index_id FROM indexes WHERE table_id = \ + (SELECT table_id FROM tables WHERE table_name = 'cu_orders') ORDER BY index_name", + ) + .fetch_all(&pool) + .await + .expect("ids"); + pool.close().await; + assert_eq!(created.len(), 2, "{created:?}"); + for (name, sql) in &created { + let id = &ids.iter().find(|(n, _)| n == name).expect("id").1; + // The statement CreateTable writes for a new index table. + assert_eq!( + sql, + &format!("CREATE INDEX \"idx_base_key_{id}\" ON \"_ddb_{id}\" (base_pk, base_sk_s)") + ); + } + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 4") + .await + .expect("row"); + for expected in [ + "cu_orders index gone: skipped, its index table", + "cu_orders index by_gpk: created idx_base_key_", + "going index by_gpk: skipped, the table is being deleted", + ] { + assert!(notes.contains(expected), "{expected} not in {notes}"); + } + let mut text = dump( + &db, + "SELECT name, sql FROM sqlite_master WHERE name LIKE 'idx_base_key_%' ORDER BY name", + ) + .await; + text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 4").await); + golden("0004_index_table_base_key", &text); +} diff --git a/crates/storage-sqlite/src/migrations/m0001_baseline.rs b/crates/storage-sqlite/src/migrations/m0001_baseline.rs new file mode 100644 index 000000000..cf64cb103 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/m0001_baseline.rs @@ -0,0 +1,131 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 1, `baseline`, SQLite: create the v1.0 catalog schema in an +//! empty database and bring every released catalog to it. +//! +//! The body executes `migrations/0001_baseline.sql`, whose statements are all +//! safe to run again, and then the two steps a file cannot make conditional, +//! each only where it is needed. Running the body again on a catalog at a +//! later migration (`--reconverge`) therefore keeps every column and row a +//! later migration added. +//! +//! - `backup_items` is rebuilt with its `id` cursor when the table has no +//! `id` column. Every released catalog created it without one, and SQLite +//! cannot add an INTEGER PRIMARY KEY with ALTER TABLE, so the rebuild uses +//! the documented procedure: a new table, the rows copied with `rowid` as +//! `id` (numbering them in their stored order), the old table dropped and +//! the new one renamed. The runner has foreign keys off while a body runs, +//! so dropping the old table deletes no child rows. +//! - `table_restores.source_table_arn` is added when the column is missing, +//! as a pre-release build created the table. +//! +//! The rebuild leaves the old table's pages free inside the file until a +//! `VACUUM`, which the note says. +//! +//! The checksum test pins this file together with the SQL file (`SOURCE`) +//! once the migration is released. The module is excluded from rustfmt and +//! clippy so neither can require an edit, and keeps its own small helpers +//! instead of calling the crate's, which may change. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use sqlx::SqlitePool; + +use super::SqliteStores; + +/// The file this body executes first. +pub(crate) const FILE: &str = extenddb_storage::file_body!("migrations", "0001_baseline.sql"); + +/// This module and the file it executes, whose SHA-256 the checksum test +/// pins. The test `migration_1_pins_its_module_and_its_file` checks that it +/// ends with `FILE`. +pub(crate) const SOURCE: &str = concat!( + include_str!("m0001_baseline.rs"), + extenddb_storage::file_body!("migrations", "0001_baseline.sql") +); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0001_baseline/sqlite/1"; + +/// `backup_items` with its `id` cursor, as the file creates it in an empty +/// database, then the copy, the drop and the rename. +const REBUILD_BACKUP_ITEMS: &str = "\ +DROP TABLE IF EXISTS backup_items_new; +CREATE TABLE backup_items_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +INSERT INTO backup_items_new (id, backup_arn, pk, sk, item_data) + SELECT rowid, backup_arn, pk, sk, item_data FROM backup_items ORDER BY rowid; +DROP INDEX IF EXISTS idx_backup_items_arn; +DROP TABLE backup_items; +ALTER TABLE backup_items_new RENAME TO backup_items; +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +"; + +pub(crate) struct Baseline; + +#[async_trait] +impl CodeBody for Baseline { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + let pool = &stores.pool; + sqlx::raw_sql(FILE) + .execute(pool) + .await + .map_err(|e| store_error("migration 1 (0001_baseline.sql)", e))?; + let mut notes = Vec::new(); + if !has_column(pool, "backup_items", "id").await? { + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backup_items") + .fetch_one(pool) + .await + .map_err(|e| store_error("count backup_items", e))?; + sqlx::raw_sql(REBUILD_BACKUP_ITEMS) + .execute(pool) + .await + .map_err(|e| store_error("rebuild backup_items", e))?; + if rows > 0 { + notes.push(format!( + "rebuilt backup_items with an id column, copying {rows} row(s); the old \ + table's pages stay free inside the database file until a VACUUM" + )); + } + } + if !has_column(pool, "table_restores", "source_table_arn").await? { + sqlx::query("ALTER TABLE table_restores ADD COLUMN source_table_arn TEXT") + .execute(pool) + .await + .map_err(|e| store_error("add table_restores.source_table_arn", e))?; + notes.push( + "added table_restores.source_table_arn, which a pre-release build left out" + .to_owned(), + ); + } + Ok(Done { notes }) + } +} + +fn store_error(context: &str, e: sqlx::Error) -> MigrationError { + MigrationError::Store(format!("{context}: {e}")) +} + +async fn has_column(pool: &SqlitePool, table: &str, column: &str) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pragma_table_info(?) WHERE name = ?)") + .bind(table) + .bind(column) + .fetch_one(pool) + .await + .map_err(|e| store_error(&format!("read the columns of {table}"), e)) +} diff --git a/crates/storage-sqlite/src/migrations/m0003_sort_key_definitions.rs b/crates/storage-sqlite/src/migrations/m0003_sort_key_definitions.rs new file mode 100644 index 000000000..fe3546168 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/m0003_sort_key_definitions.rs @@ -0,0 +1,290 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 3, `sort_key_definitions`, SQLite: restore sort key attribute +//! definitions that the pre-fix `UpdateTable` deleted (#259), in table rows +//! and in the definitions stored with backups. +//! +//! Before the fix, `UpdateTable` replaced a table's stored attribute +//! definitions with the request's subset, so adding a GSI dropped the table's +//! own key definitions. A table's sort key type is read back from its data +//! table's PRIMARY KEY column (`sk_s`, `sk_n` or `sk_b`). A backup's is read +//! from the typed value in one of its items, or else from its source table. +//! A partition key definition the same write dropped cannot be recovered, +//! because the partition key column is always TEXT; it is reported as a note. +//! Nothing here stops the migration: a damaged definition it cannot repair is +//! a note, and the table or backup keeps failing loudly until a person fixes +//! it. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit, and keeps +//! its own small helpers (`table_exists`, the key column read) instead of +//! calling the crate's, which may change. + +use async_trait::async_trait; +use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType}; +use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use extenddb_storage::util::recover_sort_key_definitions; +use sqlx::SqlitePool; + +use super::SqliteStores; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0003_sort_key_definitions.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0003_sort_key_definitions/sqlite/1"; + +/// What a repair did: the definitions it restored, and what it could not. +#[derive(Default)] +struct Repair { + repaired: Vec, + needs_attention: Vec, +} + +pub(crate) struct SortKeyDefinitions; + +#[async_trait] +impl CodeBody for SortKeyDefinitions { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + let tables = repair_table_key_definitions(&stores.pool).await.map_err(op_error)?; + let backups = repair_backups(&stores.pool).await.map_err(op_error)?; + let mut notes = Vec::new(); + for report in [tables, backups] { + notes.extend(report.repaired.into_iter().map(|line| format!("restored {line}"))); + notes.extend(report.needs_attention); + } + Ok(Done { notes }) + } +} + +fn op_error(e: OpError) -> MigrationError { + match e { + OpError::Validation(m) + | OpError::AlreadyExists(m) + | OpError::NotFound(m) + | OpError::HasDependents(m) + | OpError::Internal(m) => MigrationError::Store(m), + OpError::Database { message, sqlstate } => MigrationError::Database { message, sqlstate }, + } +} + +/// Whether a table of this name exists. +async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(name) + .fetch_one(pool) + .await + .map_err(|e| OpError::Internal(format!("look up table {name}: {e}"))) +} + +/// The sort key elements of `key_schema` with no attribute definition. +fn missing_sort_keys<'a>(key_schema: &'a [KeySchemaElement], attr_defs: &[AttributeDefinition]) -> Vec<&'a KeySchemaElement> { + key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter(|ks| !attr_defs.iter().any(|ad| ad.attribute_name == ks.attribute_name)) + .collect() +} + +/// The PRIMARY KEY columns of the data table of `table_id`, in key order, or +/// `None` when the data table does not exist. +async fn primary_key_columns(pool: &SqlitePool, table_id: &str) -> OpResult>> { + // PRAGMA takes no bind parameters, so the id is checked as a UUID before + // it is interpolated. + if uuid::Uuid::parse_str(table_id).is_err() || !table_exists(pool, &format!("_ddb_{table_id}")).await? { + return Ok(None); + } + let mut key_columns: Vec<(i64, String)> = sqlx::query_as::<_, (i64, String, String, i64, Option, i64)>( + &format!("PRAGMA table_info(\"_ddb_{table_id}\")"), + ) + .fetch_all(pool) + .await + .map_err(|e| OpError::Internal(format!("read the columns of _ddb_{table_id}: {e}")))? + .into_iter() + .filter(|(.., pk_pos)| *pk_pos > 0) + .map(|(_, name, _, _, _, pk_pos)| (pk_pos, name)) + .collect(); + key_columns.sort_unstable(); + Ok(Some(key_columns.into_iter().map(|(_, name)| name).collect())) +} + +/// Repair the `tables` rows. The partition key note is reported on every run +/// while the definition stays absent. +async fn repair_table_key_definitions(pool: &SqlitePool) -> OpResult { + let mut report = Repair::default(); + if !table_exists(pool, "tables").await? { + return Ok(report); + } + let rows: Vec<(String, String, String, String, String)> = sqlx::query_as( + "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ + FROM tables ORDER BY table_name", + ) + .fetch_all(pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot read tables: {e}")))?; + + for (account_id, table_name, table_id, ks_json, ad_json) in rows { + let key_schema: Vec = match serde_json::from_str(&ks_json) { + Ok(v) => v, + Err(e) => { + report.needs_attention.push(format!("{table_name}: unreadable key_schema ({e})")); + continue; + } + }; + let attr_defs: Vec = serde_json::from_str(&ad_json).unwrap_or_default(); + + let pk_missing: Vec<&str> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .filter(|ks| !attr_defs.iter().any(|ad| ad.attribute_name == ks.attribute_name)) + .map(|ks| ks.attribute_name.as_str()) + .collect(); + if !pk_missing.is_empty() { + report.needs_attention.push(format!( + "{table_name}: partition key definition(s) [{}] are absent; reads and writes are \ + unaffected, but index key type validation cannot check them", + pk_missing.join(", ") + )); + } + + let missing = missing_sort_keys(&key_schema, &attr_defs); + if missing.is_empty() { + continue; + } + if uuid::Uuid::parse_str(&table_id).is_err() { + report.needs_attention.push(format!("{table_name}: table_id {table_id:?} is not a UUID")); + continue; + } + let columns = primary_key_columns(pool, &table_id).await?.unwrap_or_default(); + let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); + if recovered.is_empty() { + report.needs_attention.push(format!( + "{table_name}: sort key(s) [{}] have no attribute definition and no matching \ + PRIMARY KEY column was found to recover the type from", + names(&missing) + )); + continue; + } + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_string(&merged) + .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; + sqlx::query("UPDATE tables SET attribute_definitions = ?1 WHERE account_id = ?2 AND table_name = ?3") + .bind(&merged_json) + .bind(&account_id) + .bind(&table_name) + .execute(pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot repair {table_name}: {e}")))?; + for def in &recovered { + report.repaired.push(format!("{table_name}: {} ({:?})", def.attribute_name, def.attribute_type)); + } + } + Ok(report) +} + +fn names(elements: &[&KeySchemaElement]) -> String { + elements.iter().map(|ks| ks.attribute_name.as_str()).collect::>().join(", ") +} + +/// The type of `attribute` in one of the backup's items, from its typed +/// value (`{"S": ...}`, `{"N": ...}` or `{"B": ...}`). +async fn type_from_items(pool: &SqlitePool, backup_arn: &str, attribute: &str) -> OpResult> { + let item: Option = sqlx::query_scalar("SELECT item_data FROM backup_items WHERE backup_arn = ? LIMIT 1") + .bind(backup_arn) + .fetch_optional(pool) + .await + .map_err(|e| OpError::Internal(format!("read an item of backup {backup_arn}: {e}")))?; + let Some(item) = item else { + return Ok(None); + }; + let value: serde_json::Value = serde_json::from_str(&item).unwrap_or_default(); + let descriptor = value + .get(attribute) + .and_then(serde_json::Value::as_object) + .and_then(|typed| typed.keys().next().cloned()); + Ok(match descriptor.as_deref() { + Some("S") => Some(ScalarAttributeType::S), + Some("N") => Some(ScalarAttributeType::N), + Some("B") => Some(ScalarAttributeType::B), + _ => None, + }) +} + +/// Repair the definitions stored with backups, so a backup taken before the +/// fix restores like its table. RestoreTableFromBackup validates a stored +/// definition as CreateTable does, so a backup left unrepaired fails to +/// restore rather than bringing the damage back. +async fn repair_backups(pool: &SqlitePool) -> OpResult { + let mut report = Repair::default(); + if !table_exists(pool, "backups").await? { + return Ok(report); + } + let rows: Vec<(String, String, String, String, String)> = sqlx::query_as( + "SELECT backup_arn, table_name, table_id, key_schema, attribute_definitions \ + FROM backups ORDER BY backup_arn", + ) + .fetch_all(pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot read backups: {e}")))?; + + for (backup_arn, table_name, table_id, ks_json, ad_json) in rows { + let label = format!("backup {backup_arn} of {table_name}"); + let Ok(key_schema) = serde_json::from_str::>(&ks_json) else { + report.needs_attention.push(format!("{label}: unreadable key_schema")); + continue; + }; + let attr_defs: Vec = serde_json::from_str(&ad_json).unwrap_or_default(); + let missing = missing_sort_keys(&key_schema, &attr_defs); + if missing.is_empty() { + continue; + } + let mut recovered = Vec::new(); + for element in &missing { + if let Some(attribute_type) = type_from_items(pool, &backup_arn, &element.attribute_name).await? { + recovered.push(AttributeDefinition { + attribute_name: element.attribute_name.clone(), + attribute_type, + }); + } + } + if recovered.len() < missing.len() { + let columns = primary_key_columns(pool, &table_id).await?.unwrap_or_default(); + let known: Vec = attr_defs.iter().chain(recovered.iter()).cloned().collect(); + recovered.extend(recover_sort_key_definitions(&key_schema, &known, &columns)); + } + if recovered.len() < missing.len() { + report.needs_attention.push(format!( + "{label}: sort key(s) [{}] have no attribute definition, and neither its items nor \ + its source table give the type; restoring it fails until the definition is added", + names(&missing) + )); + continue; + } + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_string(&merged) + .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; + sqlx::query("UPDATE backups SET attribute_definitions = ? WHERE backup_arn = ?") + .bind(&merged_json) + .bind(&backup_arn) + .execute(pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot repair {label}: {e}")))?; + for def in &recovered { + report.repaired.push(format!("{label}: {} ({:?})", def.attribute_name, def.attribute_type)); + } + } + Ok(report) +} diff --git a/crates/storage-sqlite/src/migrations/m0004_index_table_base_key.rs b/crates/storage-sqlite/src/migrations/m0004_index_table_base_key.rs new file mode 100644 index 000000000..114aa9ace --- /dev/null +++ b/crates/storage-sqlite/src/migrations/m0004_index_table_base_key.rs @@ -0,0 +1,138 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 4, `index_table_base_key`, SQLite: give every GSI and LSI index +//! table its `idx_base_key_` index. +//! +//! Index propagation deletes an item's old index row by its base table key, +//! with no predicate on the index key, and both the index table's PRIMARY KEY +//! and its ordering index lead with the index key, so without this index +//! every propagating write scans the whole index table. CreateTable and +//! UpdateTable have created it with each index table since v0.1.8; index +//! tables created by v0.1.3 to v0.1.7 lack it. +//! +//! Tables are listed from the catalog rows, never by name in the file. The +//! index's columns are the index table's PRIMARY KEY columns after `pk` +//! (`base_pk` and the base sort key columns), read from the table itself, so +//! a damaged attribute definition cannot choose the wrong column. A catalog +//! row whose index table is missing, or whose table is being deleted, is +//! skipped and noted. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit, and keeps +//! its own small helpers (`object_exists`, the key column read) instead of +//! calling the crate's, which may change. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use sqlx::SqlitePool; + +use super::SqliteStores; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0004_index_table_base_key.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0004_index_table_base_key/sqlite/1"; + +pub(crate) struct IndexTableBaseKey; + +#[async_trait] +impl CodeBody for IndexTableBaseKey { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + add_base_key_indexes(&stores.pool).await.map(|notes| Done { notes }) + } +} + +fn store_error(context: &str, e: sqlx::Error) -> MigrationError { + MigrationError::Store(format!("{context}: {e}")) +} + +async fn object_exists(pool: &SqlitePool, kind: &str, name: &str) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = ? AND name = ?)") + .bind(kind) + .bind(name) + .fetch_one(pool) + .await + .map_err(|e| store_error(&format!("look up {kind} {name}"), e)) +} + +/// Create the missing indexes; returns one note per index created and per +/// catalog row skipped. +async fn add_base_key_indexes(pool: &SqlitePool) -> Result, MigrationError> { + let mut notes = Vec::new(); + if !object_exists(pool, "table", "indexes").await? { + return Ok(notes); + } + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT t.table_name, t.table_status, i.index_name, i.index_id \ + FROM indexes i JOIN tables t ON t.table_id = i.table_id \ + WHERE i.index_type IN ('GSI', 'LSI') \ + ORDER BY t.table_name, i.index_name", + ) + .fetch_all(pool) + .await + .map_err(|e| store_error("list secondary indexes", e))?; + + for (table_name, table_status, index_name, index_id) in rows { + let label = format!("{table_name} index {index_name}"); + // The id is interpolated into identifiers, so it must be a UUID. + if uuid::Uuid::parse_str(&index_id).is_err() { + notes.push(format!("{label}: skipped, index_id {index_id:?} is not a UUID")); + continue; + } + if table_status == "DELETING" { + notes.push(format!("{label}: skipped, the table is being deleted")); + continue; + } + let index_table = format!("_ddb_{index_id}"); + if !object_exists(pool, "table", &index_table).await? { + notes.push(format!("{label}: skipped, its index table {index_table} does not exist")); + continue; + } + let index = format!("idx_base_key_{index_id}"); + if object_exists(pool, "index", &index).await? { + continue; + } + let mut key_columns: Vec<(i64, String)> = sqlx::query_as::<_, (i64, String, String, i64, Option, i64)>( + &format!("PRAGMA table_info(\"{index_table}\")"), + ) + .fetch_all(pool) + .await + .map_err(|e| store_error(&format!("read the columns of {index_table}"), e))? + .into_iter() + .filter(|(.., pk_pos)| *pk_pos > 0) + .map(|(_, name, _, _, _, pk_pos)| (pk_pos, name)) + .collect(); + key_columns.sort_unstable(); + let base_columns: Vec = key_columns + .into_iter() + .map(|(_, name)| name) + .filter(|name| name.starts_with("base_")) + .collect(); + if base_columns.first().map(String::as_str) != Some("base_pk") { + notes.push(format!( + "{label}: skipped, the PRIMARY KEY of {index_table} has no base_pk column" + )); + continue; + } + sqlx::query(&format!( + "CREATE INDEX IF NOT EXISTS \"{index}\" ON \"{index_table}\" ({})", + base_columns.join(", ") + )) + .execute(pool) + .await + .map_err(|e| store_error(&format!("create {index}"), e))?; + notes.push(format!("{label}: created {index} on ({})", base_columns.join(", "))); + } + Ok(notes) +} diff --git a/crates/storage-sqlite/src/migrations/mod.rs b/crates/storage-sqlite/src/migrations/mod.rs new file mode 100644 index 000000000..63252433f --- /dev/null +++ b/crates/storage-sqlite/src/migrations/mod.rs @@ -0,0 +1,345 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The SQLite backend's catalog migrations. +//! +//! The database file is the backend's one store, [`StoreKind::Catalog`]. +//! [`migration`] is the table: an exhaustive `match` from every +//! [`MigrationId`] to this backend's body. [`RELEASED`] pins the released +//! bodies; the release PR that raises `RELEASED_THROUGH` pastes the lines +//! the checksum test prints. +//! +//! Entry points: [`catalog_status`] reads, [`migrate`] takes the database's +//! lock and applies, [`migrate_holding`] applies for a caller that already +//! holds the lock (the server's factory), and [`SqliteMigrations::on_pool`] +//! runs on an in-memory database's single connection. + +mod backend; +mod commands; +mod copy; +mod open; +// The migration bodies are frozen once released: neither rustfmt nor a later +// clippy lint may require an edit that changes their checksums. +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0001_baseline; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0003_sort_key_definitions; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0004_index_table_base_key; +mod postcheck; + +use std::time::Duration; + +use extenddb_storage::migrations::m0002_resource_tag_condition_keys::{ + self, ResourceTagConditionKeys, +}; +use extenddb_storage::migrations::{ + CatalogStatus, Kind, Migration, MigrationBackend, MigrationError, MigrationId, Released, + RunReport, Runner, StoreBody, StoreKind, +}; + +pub use backend::{SqliteMigrations, SqliteStores}; +pub use commands::SqliteCatalogMigrations; + +/// The definitions migration 1 creates in an empty database, which the +/// post-check compares the database with the first time migration 1 is +/// applied to it. +const BASELINE_EXPECTED: &str = include_str!("../../migrations/0001_baseline.expected.json"); + +/// Checksums of the released bodies (none until v1.0). +pub(crate) const RELEASED: &[Released] = &[]; + +/// This backend's entry for every id. Public so `migrate --list` can print +/// each migration's kind without opening the database. +#[must_use] +pub fn migration(id: MigrationId) -> Migration { + match id { + MigrationId::Baseline => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0001_baseline::Baseline, + m0001_baseline::SOURCE, + )], + }, + MigrationId::ResourceTagConditionKeys => Migration::Applies { + kind: Kind::Breaking, + // A `dynamodb:ResourceTag` key written through the API after the + // rewrite has no effect, as on the service; running the rewrite + // again would give it one. + convergent: false, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &ResourceTagConditionKeys, + m0002_resource_tag_condition_keys::SOURCE, + )], + }, + MigrationId::SortKeyDefinitions => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0003_sort_key_definitions::SortKeyDefinitions, + m0003_sort_key_definitions::SOURCE, + )], + }, + MigrationId::IndexTableBaseKey => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0004_index_table_base_key::IndexTableBaseKey, + m0004_index_table_base_key::SOURCE, + )], + }, + } +} + +/// The longest wait a lock timeout can ask for that `Instant` can add. +const MAX_LOCK_WAIT: Duration = Duration::from_secs(86_400); + +/// The database file's lock, held for a migration. Dropping it releases it. +#[derive(Debug)] +pub struct MigrationLock { + _lock: crate::serve_lock::ServeLock, +} + +/// Take the lock `extenddb serve` holds on the database file, retrying every +/// 250 ms while another process holds it, for up to `timeout`. `Ok(None)` for +/// an in-memory database, which no other process can open. +/// +/// # Errors +/// +/// [`MigrationError::Locked`] when the lock is still held at the timeout +/// (exit 6), or a store error when the lock file cannot be opened. +pub async fn lock_for_migration( + location: &str, + timeout: Duration, +) -> Result, MigrationError> { + let Some(file) = crate::serve_lock::database_file(location).map_err(MigrationError::Store)? + else { + return Ok(None); + }; + // The lock file goes beside the database, in a directory that exists: + // `init` creates both before it takes the lock. + let now = tokio::time::Instant::now(); + let deadline = now.checked_add(timeout).unwrap_or(now + MAX_LOCK_WAIT); + loop { + match crate::serve_lock::ServeLock::try_acquire(&file) { + Ok(lock) => return Ok(Some(MigrationLock { _lock: lock })), + Err(crate::serve_lock::LockFailure::Failed(m)) => return Err(MigrationError::Store(m)), + Err(crate::serve_lock::LockFailure::Held(m)) => { + if tokio::time::Instant::now() >= deadline { + return Err(MigrationError::Locked(format!( + "{m}; waited {} s", + timeout.as_secs() + ))); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + } +} + +/// The catalog status of the database at `location`. Read-only: a file that +/// does not exist is `Uninitialized` and is not created, and nothing is +/// written. Takes no lock, so it runs beside a server. +/// +/// # Errors +/// +/// The database cannot be opened or its ledger read. +pub async fn catalog_status(location: &str) -> Result { + let Some(file) = crate::serve_lock::database_file(location).map_err(MigrationError::Store)? + else { + return Err(in_memory_refusal()); + }; + if !file.exists() { + return Ok(CatalogStatus::Uninitialized); + } + let migrations = SqliteMigrations::open_for_status(location).await?; + // A read records nothing, so no release is named. + let status = Runner::new(&migrations, "").status().await; + migrations.close().await; + status +} + +/// Apply every pending migration to the database at `location`: take the +/// lock (waiting up to `lock_timeout`), open the runner's connection, run, +/// and close the connection and release the lock on every exit path. +/// +/// # Errors +/// +/// As [`Runner::migrate`] and [`lock_for_migration`]. +pub async fn migrate( + location: &str, + release: &str, + lock_timeout: Duration, +) -> Result { + if crate::serve_lock::database_file(location) + .map_err(MigrationError::Store)? + .is_none() + { + return Err(in_memory_refusal()); + } + let _lock = lock_for_migration(location, lock_timeout).await?; + run_on_file(location, release).await +} + +/// [`migrate`] for a caller that already holds the database's lock, which a +/// second `flock` from this process could never take: the server's factory, +/// which takes the lock before it opens the engine. `held` must be the lock +/// of `location`'s file. +/// +/// # Errors +/// +/// [`MigrationError::Refused`] when `held` is another file's lock or the +/// location is in memory, or as [`Runner::migrate`]. +pub(crate) async fn migrate_holding( + location: &str, + release: &str, + held: &crate::serve_lock::ServeLock, +) -> Result { + let file = crate::serve_lock::database_file(location) + .map_err(MigrationError::Store)? + .ok_or_else(in_memory_refusal)?; + let expected = crate::serve_lock::ServeLock::lock_path(&file); + if held.path() != expected { + return Err(MigrationError::Refused(format!( + "the lock held is {}, and the database at {location} is locked by {}", + held.path().display(), + expected.display() + ))); + } + run_on_file(location, release).await +} + +/// Decide whether a server may open the file database at `location`, whose +/// lock the server holds: `Current` and `AheadCompatible` serve (the second +/// with a warning naming the migrations above this binary's), and in a dev +/// build `Behind` and an `Uninitialized` file with no tables (missing or +/// empty) are migrated first. An `Uninitialized` file that holds tables is +/// another program's database, and is refused as the commands refuse it, +/// naming `extenddb init`, before anything writes to it. Every other status +/// refuses with [`MigrationError::Status`], in a dev build too. +/// +/// # Errors +/// +/// The refusal, or as [`migrate_holding`]. +pub(crate) async fn prepare_for_serve( + location: &str, + release: &str, + held: &crate::serve_lock::ServeLock, + dev: bool, +) -> Result<(), MigrationError> { + match catalog_status(location).await? { + CatalogStatus::Current { .. } => Ok(()), + status @ CatalogStatus::AheadCompatible { .. } => { + tracing::warn!( + "{status}; serving it, because a binary may serve migrations it does not know \ + when every one of them is additive" + ); + Ok(()) + } + CatalogStatus::Uninitialized if dev && !holds_tables(location).await? => { + log_report(&migrate_holding(location, release, held).await?); + Ok(()) + } + CatalogStatus::Behind { .. } if dev => { + log_report(&migrate_holding(location, release, held).await?); + Ok(()) + } + status => Err(MigrationError::Status(status)), + } +} + +/// Whether the file database at `location` exists and holds a table, read +/// on a read-only connection. +async fn holds_tables(location: &str) -> Result { + let exists = crate::serve_lock::database_file(location) + .map_err(MigrationError::Store)? + .is_some_and(|file| file.exists()); + if !exists { + return Ok(false); + } + let migrations = SqliteMigrations::open_for_status(location).await?; + let empty = migrations.catalog_is_empty().await; + migrations.close().await; + Ok(!empty?) +} + +/// Log what a run the server started did: the copy's path, then each step +/// with its notes. +pub(crate) fn log_report(report: &RunReport) { + for note in &report.notes { + tracing::info!("{note}"); + } + for step in &report.applied { + tracing::info!("applied catalog migration {} {}", step.id, step.name); + for note in &step.notes { + tracing::info!(" {note}"); + } + } +} + +/// Open the runner's connection to a file database, run, and close it. +async fn run_on_file(location: &str, release: &str) -> Result { + let migrations = SqliteMigrations::open(location).await?; + let result = Runner::new(&migrations, release).migrate().await; + let result = migrations.discard_unused_copy(result); + migrations.close().await; + result +} + +/// Bring a test database to the latest migration: on the engine's own +/// connection for an in-memory database, else through [`migrate`] on its +/// file. Replaces the schema setup the engine's unit tests did before the +/// runner existed. +#[cfg(test)] +pub(crate) async fn apply_for_tests(pool: &sqlx::SqlitePool) { + if pool.options().get_max_connections() == 1 { + let migrations = + SqliteMigrations::on_pool(pool.clone(), ":memory:").expect("one connection"); + Runner::new(&migrations, "test") + .migrate() + .await + .expect("migrate the test database"); + } else { + let file = pool + .connect_options() + .get_filename() + .to_string_lossy() + .into_owned(); + migrate(&file, "test", Duration::ZERO) + .await + .expect("migrate the test database"); + } +} + +/// A new connection to an in-memory location opens a new, empty database, +/// so the file-based entry points refuse one. +fn in_memory_refusal() -> MigrationError { + MigrationError::Refused( + "an in-memory SQLite database exists only inside a running server, which creates its \ + catalog when it starts; init, migrate and verify have no database to work on" + .to_owned(), + ) +} + +#[cfg(test)] +mod crash_tests; +#[cfg(test)] +mod golden_tests; +#[cfg(test)] +mod open_tests; +#[cfg(test)] +mod refusal_tests; +#[cfg(test)] +mod released_tests; +#[cfg(test)] +mod serve_tests; +#[cfg(test)] +mod tests; diff --git a/crates/storage-sqlite/src/migrations/open.rs b/crates/storage-sqlite/src/migrations/open.rs new file mode 100644 index 000000000..7e03a46d8 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/open.rs @@ -0,0 +1,50 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Opening the database file for the runner: the file's mode when the runner +//! creates it, and the error for a database that cannot be opened. + +use std::path::Path; + +use extenddb_storage::migrations::MigrationError; + +/// SQLite's `SQLITE_READONLY_DIRECTORY`: the database cannot be opened +/// because the process may not write to its directory. +const READONLY_DIRECTORY: &str = "1544"; + +/// The error for a database that could not be opened. A WAL database whose +/// directory this process may not write fails with `SQLITE_READONLY_DIRECTORY` +/// even for a reader, which has to create `-shm` there, and the message +/// says so. +pub(super) fn open_error(location: &str, e: &sqlx::Error) -> MigrationError { + let code = e.as_database_error().and_then(|d| d.code()); + if code.as_deref() == Some(READONLY_DIRECTORY) { + return MigrationError::Store(format!( + "open SQLite database {location}: the database is in WAL mode, and reading it needs \ + write access to its directory, where SQLite creates {location}-shm while a reader \ + has it open ({e})" + )); + } + MigrationError::store(format!("open SQLite database {location}"), e) +} + +/// Create the database file with mode 0600 when it does not exist, before +/// SQLite opens it: the database holds the encryption key once `init` writes +/// it, and SQLite gives its `-wal` and `-shm` files the database file's mode. +pub(super) fn create_private(path: &Path) -> Result<(), MigrationError> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(path) { + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), + Err(e) => Err(MigrationError::store( + format!("create the database file {}", path.display()), + e, + )), + } +} diff --git a/crates/storage-sqlite/src/migrations/open_tests.rs b/crates/storage-sqlite/src/migrations/open_tests.rs new file mode 100644 index 000000000..b5db709d7 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/open_tests.rs @@ -0,0 +1,209 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Opening the runner's and the status's connections: the status of a WAL +//! catalog in a directory the process cannot write, a database the runner +//! cannot write, and sqlx's slow-statement warnings, which the operator +//! lines replace. + +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::CatalogStatus; +use sqlx::{ConnectOptions, Connection}; + +use super::SqliteMigrations; +use super::tests::{DEV_0_1_14, LATEST, RELEASE, exec, load_dump, run, status, temp_db}; + +/// The bytes of the database and of its `-wal` and `-shm` files. +fn files(db: &Path) -> Vec>> { + ["", "-wal", "-shm"] + .iter() + .map(|suffix| std::fs::read(format!("{}{suffix}", db.display())).ok()) + .collect() +} + +fn chmod(path: &Path, mode: u32) { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).expect("chmod"); +} + +/// A process that can write a 0500 directory (root) cannot test this. +fn writes_anyway(dir: &Path) -> bool { + std::fs::write(dir.join("probe"), b"").is_ok() +} + +#[tokio::test] +async fn the_status_of_a_wal_catalog_in_a_read_only_directory_changes_nothing() { + let db = temp_db("ro-dir"); + run(&db).await.expect("migrate"); + // A writer keeps the catalog in WAL with its -wal and -shm files, as a + // running server does. + let writer = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect(&format!("sqlite://{}", db.display())) + .await + .expect("writer"); + sqlx::query("UPDATE settings SET value = value WHERE key = 'catalog_version'") + .execute(&writer) + .await + .expect("write"); + let dir = db.parent().expect("dir").to_owned(); + let before = files(&db); + assert!( + before.iter().all(Option::is_some), + "the catalog is not in WAL" + ); + chmod(&dir, 0o500); + if writes_anyway(&dir) { + chmod(&dir, 0o700); + writer.close().await; + eprintln!("SKIP: this process writes a read-only directory"); + return; + } + let started = Instant::now(); + let read = super::catalog_status(&db.to_string_lossy()).await; + let elapsed = started.elapsed(); + let after = files(&db); + chmod(&dir, 0o700); + writer.close().await; + assert_eq!( + read.expect("status"), + CatalogStatus::Current { version: LATEST } + ); + assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}"); + assert!( + before == after, + "reading the status changed the database's files" + ); +} + +/// With no process holding the WAL catalog open there is no `-shm` file, +/// and a reader has to create one in the directory: the status fails at +/// once, and the message says why, not only that the database is read-only. +#[tokio::test] +async fn a_closed_wal_catalog_in_a_read_only_directory_names_the_shm_file() { + let db = temp_db("ro-dir-closed"); + run(&db).await.expect("migrate"); + let dir = db.parent().expect("dir").to_owned(); + assert!( + files(&db)[1..].iter().all(Option::is_none), + "a -wal or -shm file is left open" + ); + chmod(&dir, 0o500); + if writes_anyway(&dir) { + chmod(&dir, 0o700); + eprintln!("SKIP: this process writes a read-only directory"); + return; + } + let started = Instant::now(); + let read = super::catalog_status(&db.to_string_lossy()).await; + let elapsed = started.elapsed(); + chmod(&dir, 0o700); + let message = read + .expect_err("a WAL reader needs to create -shm") + .to_string(); + assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}"); + assert!( + message.contains("WAL mode") && message.contains("-shm"), + "{message}" + ); +} + +#[tokio::test] +async fn a_database_the_runner_cannot_write_fails_at_once_with_its_cause() { + // A rollback-journal database, as a 0.1.x release left it: the runner's + // connection switches it to WAL, which writes the file. + let db = temp_db("ro-file"); + exec( + &db, + "CREATE TABLE t (x INTEGER); PRAGMA journal_mode = DELETE", + ) + .await; + chmod(&db, 0o400); + if std::fs::OpenOptions::new().write(true).open(&*db).is_ok() { + chmod(&db, 0o600); + eprintln!("SKIP: this process writes a 0400 file"); + return; + } + let started = Instant::now(); + let opened = SqliteMigrations::open(&db.to_string_lossy()).await; + let elapsed = started.elapsed(); + chmod(&db, 0o600); + let Err(err) = opened else { + panic!("the runner opened a database it cannot write"); + }; + assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}"); + let message = err.to_string(); + assert!( + message.contains("readonly database") && !message.contains("pool timed out"), + "{message}" + ); +} + +/// Set by the parent of the slow-statement test for its child. +const SLOW_CHILD: &str = "EXTENDDB_TEST_SLOW_STATEMENT_CHILD"; + +/// The child half of the slow-statement test; does nothing in an ordinary +/// run. Every event goes to stderr. A connection that logs every statement +/// as slow runs one, so the parent sees that the capture works, then a run +/// migrates the catalog at the path given. +#[tokio::test] +async fn slow_statement_test_child_process() { + let Ok(db) = std::env::var(SLOW_CHILD) else { + return; + }; + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_max_level(tracing::Level::TRACE) + .with_ansi(false) + .init(); + let mut probe = sqlx::sqlite::SqliteConnectOptions::new() + .filename(format!("{db}.probe")) + .create_if_missing(true) + .log_slow_statements(log::LevelFilter::Warn, Duration::ZERO) + .connect() + .await + .expect("probe"); + sqlx::query("SELECT 1") + .execute(&mut probe) + .await + .expect("probe"); + let _ = probe.close().await; + eprintln!("=== run"); + let report = super::migrate(&db, RELEASE, Duration::from_secs(10)) + .await + .expect("migrate"); + eprintln!("=== applied {}", report.applied.len()); +} + +/// In a test build the runner's connections count every statement as slow +/// (`backend.rs`), so a run with statement logging left on would print a +/// warning for every statement, the copy and the rebuilds included. +#[tokio::test] +async fn a_run_prints_no_slow_statement_warning() { + let db = temp_db("slow"); + load_dump(&db, DEV_0_1_14).await; + let output = std::process::Command::new(std::env::current_exe().expect("test binary")) + .args([ + "--exact", + "migrations::open_tests::slow_statement_test_child_process", + "--nocapture", + ]) + .env(SLOW_CHILD, db.to_string_lossy().as_ref()) + .output() + .expect("child"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stderr}"); + let (probe, run) = stderr.split_once("=== run").expect("the child ran"); + assert!( + probe.contains("slow statement"), + "the capture saw nothing: {probe}" + ); + assert!(run.contains("=== applied"), "{run}"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert!(!run.contains("slow statement"), "{run}"); +} diff --git a/crates/storage-sqlite/src/migrations/postcheck.rs b/crates/storage-sqlite/src/migrations/postcheck.rs new file mode 100644 index 000000000..a89b8037f --- /dev/null +++ b/crates/storage-sqlite/src/migrations/postcheck.rs @@ -0,0 +1,278 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The check after migration 1: the database's normalised definitions are +//! compared with `migrations/0001_baseline.expected.json`, which is generated +//! by running migration 1 on an empty database. +//! +//! A table's definition is the set of its column definitions and constraints, +//! each normalised (comments removed, whitespace collapsed, plain identifiers +//! unquoted), so column order does not count. An index's is its normalised +//! `CREATE INDEX` statement. Only objects the expected file names are +//! checked: per-table objects that no migration creates (`_ddb_` data and +//! index tables and their indexes) are not in it. + +use std::collections::BTreeMap; + +use extenddb_storage::migrations::{MigrationError, split_comments}; +use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; + +/// One object in the expected file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Expected { + #[serde(rename = "type")] + pub kind: String, + pub name: String, + pub table: String, + pub definition: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ExpectedFile { + objects: Vec, +} + +/// A difference that a pre-release build left and that is accepted: +/// `(table, expected part, accepted part, note recorded in the ledger)`. +const ACCEPTED: &[(&str, &str, &str, &str)] = &[( + "table_restores", + "restore_date_time TEXT NOT NULL", + "restore_date_time TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + "table_restores.restore_date_time keeps the default a pre-release build gave it; every row \ + this build writes supplies the value", +)]; + +/// Normalise one `sqlite_schema` statement: comments removed, whitespace +/// collapsed to single spaces and dropped around `(`, `)` and `,`, and +/// double quotes removed from identifiers that need none. String literals +/// are kept as written. +pub(crate) fn normalize(sql: &str) -> String { + let (code, _) = split_comments(sql, false); + let mut out = String::with_capacity(code.len()); + let mut space = false; + let mut chars = code.chars().peekable(); + let push = |out: &mut String, piece: &str, space: &mut bool| { + let punct = matches!(piece, "(" | ")" | ","); + let after_punct = out.ends_with(['(', ')', ',']); + if *space && !punct && !after_punct && !out.is_empty() { + out.push(' '); + } + *space = false; + out.push_str(piece); + }; + while let Some(c) = chars.next() { + match c { + c if c.is_whitespace() => space = true, + '\'' => { + let mut literal = String::from('\''); + while let Some(d) = chars.next() { + literal.push(d); + if d == '\'' { + if chars.peek() == Some(&'\'') { + literal.push('\''); + chars.next(); + } else { + break; + } + } + } + push(&mut out, &literal, &mut space); + } + '"' => { + let mut ident = String::new(); + while let Some(d) = chars.next() { + if d == '"' { + if chars.peek() == Some(&'"') { + ident.push('"'); + chars.next(); + } else { + break; + } + } else { + ident.push(d); + } + } + let plain = !ident.is_empty() + && ident.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if plain { + push(&mut out, &ident, &mut space); + } else { + push( + &mut out, + &format!("\"{}\"", ident.replace('"', "\"\"")), + &mut space, + ); + } + } + _ => push(&mut out, &c.to_string(), &mut space), + } + } + out +} + +/// A normalised table statement's column definitions and constraints, in +/// sorted order. +pub(crate) fn table_parts(normalized: &str) -> Vec { + let (Some(open), Some(close)) = (normalized.find('('), normalized.rfind(')')) else { + return vec![normalized.to_owned()]; + }; + let mut parts = Vec::new(); + let mut depth = 0usize; + let mut quoted = false; + let mut current = String::new(); + for c in normalized[open + 1..close].chars() { + match c { + '\'' => quoted = !quoted, + '(' if !quoted => depth += 1, + ')' if !quoted => depth = depth.saturating_sub(1), + ',' if !quoted && depth == 0 => { + parts.push(std::mem::take(&mut current)); + continue; + } + _ => {} + } + current.push(c); + } + parts.push(current); + let tail = normalized[close + 1..].trim(); + if !tail.is_empty() { + parts.push(tail.to_owned()); + } + parts.sort_unstable(); + parts +} + +/// Every object of the database, except SQLite's own and the ledger, by +/// type and name. +pub(crate) async fn definitions( + pool: &SqlitePool, +) -> Result, MigrationError> { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT type, name, tbl_name, sql FROM sqlite_master \ + WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' AND tbl_name <> 'extenddb_migrations'", + ) + .fetch_all(pool) + .await + .map_err(|e| MigrationError::store("read the schema", e))?; + Ok(rows + .into_iter() + .map(|(kind, name, table, sql)| { + let normalized = normalize(&sql); + let definition = if kind == "table" { + table_parts(&normalized) + } else { + vec![normalized] + }; + ( + (kind.clone(), name.clone()), + Expected { + kind, + name, + table, + definition, + }, + ) + }) + .collect()) +} + +/// The expected file's contents for the database as it is, for the test +/// that regenerates it from an empty database. +#[cfg(test)] +pub(crate) async fn expected_file(pool: &SqlitePool) -> Result { + let objects = definitions(pool).await?.into_values().collect(); + let mut text = serde_json::to_string_pretty(&ExpectedFile { objects }) + .map_err(|e| MigrationError::store("serialize the expected definitions", e))?; + text.push('\n'); + Ok(text) +} + +/// Compare the database with the expected definitions. Returns the notes for +/// accepted differences. +/// +/// # Errors +/// +/// An object the expected file names is missing or differs; the error names +/// it and the parts that differ. +pub(crate) async fn check( + pool: &SqlitePool, + expected: &str, +) -> Result, MigrationError> { + let expected: ExpectedFile = serde_json::from_str(expected) + .map_err(|e| MigrationError::store("parse 0001_baseline.expected.json", e))?; + let actual = definitions(pool).await?; + let mut notes = Vec::new(); + let mut problems = Vec::new(); + for object in expected.objects { + let key = (object.kind.clone(), object.name.clone()); + let Some(found) = actual.get(&key) else { + problems.push(format!("{} {} is missing", object.kind, object.name)); + continue; + }; + let mut missing: Vec<&String> = object + .definition + .iter() + .filter(|p| !found.definition.contains(p)) + .collect(); + let mut extra: Vec<&String> = found + .definition + .iter() + .filter(|p| !object.definition.contains(p)) + .collect(); + for (table, expected_part, accepted_part, note) in ACCEPTED { + if object.name == *table + && missing.iter().any(|p| p.as_str() == *expected_part) + && extra.iter().any(|p| p.as_str() == *accepted_part) + { + missing.retain(|p| p.as_str() != *expected_part); + extra.retain(|p| p.as_str() != *accepted_part); + notes.push((*note).to_owned()); + } + } + if !missing.is_empty() || !extra.is_empty() { + problems.push(format!( + "{} {} differs: expected [{}], found [{}]", + object.kind, + object.name, + missing + .iter() + .map(|s| s.as_str()) + .collect::>() + .join("; "), + extra + .iter() + .map(|s| s.as_str()) + .collect::>() + .join("; ") + )); + } + } + if problems.is_empty() { + Ok(notes) + } else { + Err(MigrationError::Store(format!( + "after migration 1 the database does not have the expected schema, so nothing was \ + committed: {}", + problems.join("; ") + ))) + } +} + +#[cfg(test)] +mod tests { + use super::{normalize, table_parts}; + + #[test] + fn normalisation_ignores_layout_comments_and_plain_quotes() { + let a = "CREATE TABLE \"t\" (\n a TEXT NOT NULL, -- note, with a comma\n b INTEGER DEFAULT (strftime('%s', 'now')),\n PRIMARY KEY (a)\n)"; + let b = "CREATE TABLE t(b INTEGER DEFAULT(strftime('%s','now')),a TEXT NOT NULL,PRIMARY KEY(a))"; + assert_eq!(table_parts(&normalize(a)), table_parts(&normalize(b))); + // A literal keeps its spaces, and a quoted name that needs quotes keeps them. + assert_eq!(normalize("SELECT 'a b'"), "SELECT 'a b'"); + assert_eq!( + normalize("CREATE INDEX \"i-1\" ON \"_ddb_x-y\" (c)"), + "CREATE INDEX \"i-1\" ON \"_ddb_x-y\"(c)" + ); + } +} diff --git a/crates/storage-sqlite/src/migrations/refusal_tests.rs b/crates/storage-sqlite/src/migrations/refusal_tests.rs new file mode 100644 index 000000000..a832ae224 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/refusal_tests.rs @@ -0,0 +1,1027 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What the SQLite runner refuses, and that a refusal or a failed body +//! commits nothing: foreign key violations, a failing file body, the +//! post-check, a ledger it cannot read, the copy's name and the free space. + +use extenddb_storage::migrations::{ + CatalogStatus, MigrationBackend, MigrationError, MigrationId, Runner, Step, exit, +}; + +use super::SqliteMigrations; +use super::tests::{ + DEV_0_1_14, LATEST, Orphan, RELEASE, catalog_store, count, exec, load_dump, plain_pool, run, + scalar, status, temp_db, test_row, version_row, +}; + +async fn schema(db: &std::path::Path) -> Vec<(String, String, Option)> { + let pool = plain_pool(db).await; + let rows = sqlx::query_as("SELECT type, name, sql FROM sqlite_master ORDER BY type, name") + .fetch_all(&pool) + .await + .expect("schema"); + pool.close().await; + rows +} + +#[tokio::test] +async fn a_foreign_key_violation_a_body_adds_rolls_the_body_back() { + let db = temp_db("fk"); + run(&db).await.expect("fresh"); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let err = migrations + .apply( + &catalog_store(&db), + MigrationId::IndexTableBaseKey, + Step::Code(&Orphan), + test_row(LATEST + 5), + None, + ) + .await + .expect_err("orphan row"); + assert!(err.to_string().contains("violate a foreign key"), "{err}"); + migrations.close().await; + assert_eq!(count(&db, "SELECT COUNT(*) FROM indexes").await, 0); + assert_eq!( + count( + &db, + &format!( + "SELECT COUNT(*) FROM extenddb_migrations WHERE id = {}", + LATEST + 5 + ) + ) + .await, + 0 + ); +} + +/// Rows that violated a foreign key before the run (a parent deleted with +/// foreign keys off, which the `sqlite3` shell does by default) are noted in +/// every body's row and do not stop it; a violation a body adds still does. +#[tokio::test] +async fn foreign_key_violations_that_predate_the_run_are_noted() { + let db = temp_db("fk-before"); + load_dump(&db, DEV_0_1_14).await; + exec(&db, "PRAGMA foreign_keys = OFF; DELETE FROM tables;").await; + run(&db).await.expect("migrate past the earlier violations"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1") + .await + .expect("row"); + assert!( + notes.contains("violated a foreign key before migration 1 baseline ran"), + "{notes}" + ); + assert!(notes.contains("indexes (2)"), "{notes}"); + assert!(notes.contains("stream_shards (4)"), "{notes}"); + + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let err = migrations + .apply( + &catalog_store(&db), + MigrationId::IndexTableBaseKey, + Step::Code(&Orphan), + test_row(LATEST + 5), + None, + ) + .await + .expect_err("a new orphan"); + migrations.close().await; + assert!(err.to_string().contains("left 1 row(s)"), "{err}"); +} + +/// A file body's statements run inside the runner's transaction: when its +/// last statement fails, the first one's write is gone too. +#[tokio::test] +async fn a_file_body_that_fails_part_way_commits_nothing() { + let db = temp_db("file-body"); + run(&db).await.expect("fresh"); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let err = migrations + .apply( + &catalog_store(&db), + MigrationId::IndexTableBaseKey, + Step::File { + name: "0004_index_table_base_key.sql", + text: "INSERT INTO settings (key, value) VALUES ('file_body', 'wrote');\n\ + INSERT INTO no_such_table VALUES (1);", + }, + test_row(LATEST + 5), + Some(LATEST + 5), + ) + .await + .expect_err("the second statement fails"); + migrations.close().await; + assert!(err.to_string().contains("no such table"), "{err}"); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM settings WHERE key = 'file_body'").await, + 0 + ); + assert_eq!(version_row(&db).await, Some(LATEST.to_string())); +} + +#[tokio::test] +async fn a_body_that_fails_part_way_leaves_nothing_and_a_rerun_completes() { + let db = temp_db("crash"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let migrations = SqliteMigrations::open(&location).await.expect("open"); + Runner::new(&migrations, RELEASE) + .migrate_through(MigrationId::Baseline) + .await + .expect("migration 1"); + // Migration 3's step with a body that writes and then fails. + let err = migrations + .apply( + &catalog_store(&db), + MigrationId::SortKeyDefinitions, + Step::Code(&Orphan), + test_row(3), + Some(3), + ) + .await + .expect_err("fails at the foreign key check"); + assert_eq!(err.exit_code(), exit::ERROR); + migrations.close().await; + assert_eq!( + version_row(&db).await.as_deref(), + Some("1"), + "the version row is the one committed with migration 1" + ); + let CatalogStatus::Behind { version, .. } = status(&db).await else { + panic!("expected Behind"); + }; + assert_eq!(version, Some(1)); + let report = run(&db).await.expect("rerun"); + assert_eq!(report.from, Some(1)); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); +} + +/// Shapes the post-check refuses after migration 1: an extra column, and an +/// index on other columns. Each refusal commits nothing: the schema, the +/// version row and the marker row that keeps released servers off the +/// catalog are as they were, and there is no ledger. +#[tokio::test] +async fn the_post_check_refuses_extra_parts_and_commits_nothing() { + for (case, change, expected) in [ + ( + "extra-column", + "CREATE TABLE table_restores (\ + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ + source_backup_arn TEXT NOT NULL, source_table_arn TEXT, \ + restore_date_time TEXT NOT NULL, restored_by TEXT NOT NULL)", + "found [restored_by TEXT NOT NULL]", + ), + ( + "other-index", + "DROP INDEX idx_backups_table; \ + CREATE INDEX idx_backups_table ON backups (table_name);", + "index idx_backups_table differs", + ), + ] { + let db = temp_db(case); + load_dump(&db, DEV_0_1_14).await; + exec(&db, change).await; + let before = schema(&db).await; + let err = run(&db).await.expect_err(case); + let message = err.to_string(); + assert!( + message.contains("nothing was committed"), + "{case}: {message}" + ); + assert!(message.contains(expected), "{case}: {message}"); + assert_eq!(schema(&db).await, before, "{case}"); + assert_eq!(version_row(&db).await.as_deref(), Some("0.0.3"), "{case}"); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM settings WHERE key = 'resource_tag_migration'" + ) + .await, + 1, + "{case}: the marker is still there" + ); + } +} + +/// A ledger this binary cannot read is an error, never "not applied". +#[tokio::test] +async fn a_ledger_that_cannot_be_read_is_an_error() { + let db = temp_db("bad-ledger"); + run(&db).await.expect("fresh"); + exec(&db, "ALTER TABLE extenddb_migrations DROP COLUMN notes").await; + let found = super::catalog_status(&db.to_string_lossy()).await; + assert!( + matches!(found, Err(MigrationError::Store(ref m)) if m.contains("read extenddb_migrations")), + "{found:?}" + ); + assert!(run(&db).await.is_err()); +} + +/// Reading the status writes nothing: a rollback-journal database keeps its +/// journal mode and its bytes. +#[tokio::test] +async fn the_status_writes_nothing() { + let db = temp_db("read-only"); + run(&db).await.expect("fresh"); + exec(&db, "PRAGMA journal_mode = DELETE").await; + let bytes = std::fs::read(&*db).expect("database"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert_eq!(std::fs::read(&*db).expect("database"), bytes); + assert_eq!( + scalar(&db, "PRAGMA journal_mode").await.as_deref(), + Some("delete") + ); +} + +/// The list of policies a pending migration 2 rewrites, which the preview +/// and `verify` print, is read on a read-only connection as the status is: +/// a rollback-journal database keeps its journal mode and its bytes. A +/// connection that opens the database for writing sets WAL mode, and when +/// it is the last one to close it checkpoints the WAL into the file. +#[tokio::test] +async fn listing_the_policies_migration_2_rewrites_writes_nothing() { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::SqliteCatalogMigrations; + + let db = temp_db("pending-read-only"); + load_dump(&db, DEV_0_1_14).await; + let pool = plain_pool(&db).await; + let (account, user): (String, String) = + sqlx::query_as("SELECT account_id, user_name FROM iam_users LIMIT 1") + .fetch_one(&pool) + .await + .expect("an IAM user"); + sqlx::query( + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, \ + policy_document) VALUES (?, 'user', ?, 'tagged', ?)", + ) + .bind(&account) + .bind(&user) + .bind( + r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}"#, + ) + .execute(&pool) + .await + .expect("policy"); + pool.close().await; + exec(&db, "PRAGMA journal_mode = DELETE").await; + let bytes = std::fs::read(&*db).expect("database"); + // As in the binary: the trait's default would open the installed + // backend's management store. + let _ = extenddb_storage::set_backend(crate::backend()); + + let target = Target::at(db.to_string_lossy()); + let status = SqliteCatalogMigrations + .status(&target) + .await + .expect("status"); + let lines = SqliteCatalogMigrations + .pending_changes(&target, &status) + .await + .expect("the policies migration 2 rewrites"); + assert_eq!( + lines[0], + "Migration 2 rewrites 1 stored policy from dynamodb:ResourceTag to aws:ResourceTag:" + ); + assert!( + lines[1].contains(&format!("account {account} ")) && lines[1].contains("policy tagged"), + "{lines:?}" + ); + assert!( + std::fs::read(&*db).expect("database") == bytes, + "listing the policies changed the database file" + ); + assert_eq!( + scalar(&db, "PRAGMA journal_mode").await.as_deref(), + Some("delete") + ); +} + +/// A file at the copy's name, whatever it holds, is never kept as the copy +/// or replaced: the run stops, and the file and the database are untouched. +#[tokio::test] +async fn a_file_at_the_copy_name_stops_the_run() { + let db = temp_db("copy-exists"); + load_dump(&db, DEV_0_1_14).await; + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let at = std::time::SystemTime::now(); + let copy = super::copy::copy_path(&db, "legacy", LATEST, at); + let full = super::copy::copy_path(&db, "earlier", LATEST, at); + super::copy::pre_migration_copy(migrations.pool(), &db, &full, super::copy::Hooks::RUN) + .await + .expect("a full copy"); + let no_tables = db.with_file_name("no-tables.sqlite"); + exec(&no_tables, "PRAGMA user_version = 1").await; + let first_page = std::fs::read(&full).expect("copy")[..4096].to_vec(); + for content in [ + first_page, + Vec::new(), + std::fs::read(&no_tables).expect("database"), + std::fs::read(&full).expect("copy"), + ] { + std::fs::write(©, &content).expect("plant"); + let err = + super::copy::pre_migration_copy(migrations.pool(), &db, ©, super::copy::Hooks::RUN) + .await + .expect_err("refused"); + assert!(err.to_string().contains("already exists"), "{err}"); + assert_eq!(std::fs::read(©).expect("kept"), content); + } + migrations.close().await; + assert_eq!(version_row(&db).await.as_deref(), Some("0.0.3")); +} + +/// A copy that fails while it writes removes its partial file: the likeliest +/// cause is a full disk, where that file is as large as the database. A +/// directory in place of the journal SQLite writes beside the partial file +/// makes the copy fail after the partial file exists. +#[tokio::test] +async fn a_failed_copy_leaves_no_partial_file() { + let db = temp_db("copy-fails"); + load_dump(&db, DEV_0_1_14).await; + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + let partial = std::path::PathBuf::from(format!("{}.partial", copy.display())); + let journal = std::path::PathBuf::from(format!("{}.partial-journal", copy.display())); + std::fs::create_dir(&journal).expect("a directory at the journal's path"); + let err = + super::copy::pre_migration_copy(migrations.pool(), &db, ©, super::copy::Hooks::RUN) + .await + .expect_err("the copy fails"); + migrations.close().await; + assert!(err.to_string().contains("copy the database"), "{err}"); + assert!( + !partial.exists(), + "the failed copy left {}", + partial.display() + ); + assert!(!copy.exists()); +} + +/// A write that is only in the WAL, which no checkpoint has moved into the +/// main file, is in the copy: the backup reads the committed frames of the +/// WAL with the main file. The copy is owner-only and in rollback-journal +/// mode, as `VACUUM INTO` left it, so it can be read without a `-shm` file +/// beside it, and SQLite leaves no other file beside it. +#[cfg(unix)] +#[tokio::test] +async fn a_copy_holds_a_write_that_is_only_in_the_wal() { + use sqlx::{ConnectOptions as _, Connection as _}; + use std::os::unix::fs::PermissionsExt; + let db = temp_db("copy-wal"); + load_dump(&db, DEV_0_1_14).await; + // The writer stays open and never checkpoints, so the write stays in + // the WAL alone. + let writer = super::tests::plain_pool(&db).await; + let mut held = writer.acquire().await.expect("connection"); + sqlx::raw_sql( + "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; \ + INSERT INTO settings (key, value) VALUES ('in_the_wal', '1');", + ) + .execute(&mut *held) + .await + .expect("a write left in the WAL"); + let mut main_file = sqlx::sqlite::SqliteConnectOptions::new() + .filename(&*db) + .read_only(true) + .immutable(true) + .connect() + .await + .expect("the main file alone"); + let (in_main,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM settings WHERE key = 'in_the_wal'") + .fetch_one(&mut main_file) + .await + .expect("count"); + main_file.close().await.expect("close"); + assert_eq!(in_main, 0, "the write is in the main file already"); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + let lines = + super::copy::pre_migration_copy(migrations.pool(), &db, ©, super::copy::Hooks::RUN) + .await + .expect("the copy"); + let line = lines.last().expect("the copy's line"); + assert!(line.contains(©.display().to_string()), "{line}"); + let mode = std::fs::metadata(©).expect("copy").permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let header = std::fs::read(©).expect("copy"); + assert_eq!( + (header[18], header[19]), + (1, 1), + "the copy is not in rollback-journal mode" + ); + for suffix in ["-wal", "-shm", "-journal"] { + let side = format!("{}{suffix}", copy.display()); + assert!(!std::path::Path::new(&side).exists(), "{side} is left"); + } + drop(held); + writer.close().await; + migrations.close().await; + assert_eq!( + count( + ©, + "SELECT COUNT(*) FROM settings WHERE key = 'in_the_wal'" + ) + .await, + 1 + ); +} + +/// The copy opens no file descriptor of the database outside SQLite. +/// Closing one would release every POSIX lock this process holds on the +/// file, the shared lock of the runner's connection included, and another +/// process that then closed its connection would take itself for the last +/// one and delete the WAL, with the commits the run makes after it. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn the_copy_leaves_the_runners_lock_on_the_database() { + let db = temp_db("copy-lock"); + load_dump(&db, DEV_0_1_14).await; + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + // A connection in WAL mode keeps the shared lock its first read takes. + sqlx::query("SELECT COUNT(*) FROM sqlite_master") + .execute(migrations.pool()) + .await + .expect("read"); + let held = posix_locks(&db); + assert!(!held.is_empty(), "the runner's connection holds no lock"); + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + super::copy::pre_migration_copy(migrations.pool(), &db, ©, super::copy::Hooks::RUN) + .await + .expect("the copy"); + assert_eq!( + posix_locks(&db), + held, + "the copy released the runner's lock" + ); + migrations.close().await; +} + +/// A hard link and a symbolic link to the database under copies' names are +/// not copies: the search for the earlier copies a run replaces opens +/// neither, so this process keeps its locks on the database, and removes +/// neither. A restore with `ln` can leave such a link. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn a_link_to_the_database_under_a_copys_name_is_not_opened() { + let db = temp_db("copy-link"); + load_dump(&db, DEV_0_1_14).await; + let at = |second: u64| { + std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_767_225_600 + second) + }; + let hard = super::copy::copy_path(&db, "legacy", LATEST, at(0)); + std::fs::hard_link(&*db, &hard).expect("a hard link"); + let soft = super::copy::copy_path(&db, "legacy", LATEST, at(1)); + std::os::unix::fs::symlink(&*db, &soft).expect("a symbolic link"); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + sqlx::query("SELECT COUNT(*) FROM sqlite_master") + .execute(migrations.pool()) + .await + .expect("read"); + let held = posix_locks(&db); + assert!(!held.is_empty(), "the runner's connection holds no lock"); + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + super::copy::pre_migration_copy(migrations.pool(), &db, ©, super::copy::Hooks::RUN) + .await + .expect("the copy"); + assert_eq!( + std::fs::metadata(&hard).expect("the link").len(), + std::fs::metadata(©).expect("the copy").len(), + "the link is no candidate by its size alone" + ); + let lines = super::copy::remove_superseded(&db, ©).await; + assert_eq!( + posix_locks(&db), + held, + "the search released the runner's lock" + ); + assert!(lines.is_empty(), "{lines:?}"); + assert!(hard.exists(), "the hard link was removed"); + assert!( + soft.symlink_metadata().is_ok(), + "the symbolic link was removed" + ); + migrations.close().await; +} + +/// This process's POSIX locks on the file at `path`, from /proc/locks: one +/// ` -` entry each. +#[cfg(target_os = "linux")] +fn posix_locks(path: &std::path::Path) -> Vec { + use std::os::unix::fs::MetadataExt; + let inode = std::fs::metadata(path).expect("the file").ino().to_string(); + let pid = std::process::id().to_string(); + std::fs::read_to_string("/proc/locks") + .expect("/proc/locks") + .lines() + .filter_map(|line| { + // `1: POSIX ADVISORY READ 4242 08:01:131074 1073741826 1073742335` + let fields: Vec<&str> = line.split_whitespace().collect(); + let ours = fields.len() >= 8 + && fields[1] == "POSIX" + && fields[4] == pid + && fields[5].rsplit(':').next() == Some(inode.as_str()); + ours.then(|| format!("{} {}-{}", fields[3], fields[6], fields[7])) + }) + .collect() +} + +/// The free space the probe reports is compared with three times the +/// database's size before the copy is taken, after the partial copies of +/// stopped runs are removed, since they may be what fills the disk. +#[tokio::test] +async fn a_short_filesystem_stops_the_run_before_the_copy() { + let db = temp_db("short"); + load_dump(&db, DEV_0_1_14).await; + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let copy = super::copy::copy_path(&db, "legacy", LATEST, std::time::SystemTime::now()); + let stopped = std::path::PathBuf::from(format!( + "{}.pre-migration-legacy-to-4-20260101T000000Z.partial", + db.display() + )); + std::fs::write(&stopped, b"a stopped run's copy").expect("plant"); + let err = super::copy::pre_migration_copy( + migrations.pool(), + &db, + ©, + super::copy::Hooks { + free_space: |_| Some(4096), + ..super::copy::Hooks::RUN + }, + ) + .await + .expect_err("short"); + migrations.close().await; + assert!( + err.to_string().contains("three times the database's size"), + "{err}" + ); + assert!(!copy.exists()); + assert!(!stopped.exists(), "the space check came before the removal"); +} + +/// A run with nothing to apply takes no copy, and still removes the partial +/// copies stopped runs left, with the files SQLite keeps beside them, and +/// names them. +#[tokio::test] +async fn a_run_with_nothing_to_apply_removes_stopped_partial_copies() { + let db = temp_db("partials-current"); + run(&db).await.expect("fresh"); + let stem = format!("{}.pre-migration-3-to-4-20260101T000000Z", db.display()); + let partials = [ + format!("{stem}.partial"), + format!("{stem}.partial-journal"), + format!("{stem}.partial-wal"), + format!("{stem}.partial-shm"), + ]; + for p in &partials { + std::fs::write(p, b"left by a stopped run").expect("plant"); + } + let report = run(&db).await.expect("nothing to apply"); + assert!(report.applied.is_empty(), "{report:?}"); + for p in &partials { + assert!(!std::path::Path::new(p).exists(), "{p} is left"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with(&format!("removed {p},"))), + "{:?}", + report.notes + ); + } +} + +/// A run killed after its copy was complete, with nothing committed, leaves +/// the copy. The next run's copy has the same bytes, so it replaces it +/// whatever its ``, and the run names it. A copy with other bytes, one a +/// process opened (a `-wal` beside it) and one with another `` stay, +/// and a reconverge after the run, whose copy holds the migrated database, +/// replaces none. +#[tokio::test] +async fn a_copy_of_the_same_contents_replaces_an_earlier_one() { + let db = temp_db("superseded"); + load_dump(&db, DEV_0_1_14).await; + let at = |second: u64| { + std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_767_225_600 + second) + }; + let stopped = super::copy::copy_path(&db, "legacy", LATEST, at(0)); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + super::copy::pre_migration_copy(migrations.pool(), &db, &stopped, super::copy::Hooks::RUN) + .await + .expect("a stopped run's copy"); + migrations.close().await; + let bytes = std::fs::read(&stopped).expect("copy"); + let plant = |path: &std::path::Path, content: &[u8]| { + std::fs::write(path, content).expect("plant"); + }; + let older_binary = super::copy::copy_path(&db, "legacy", 3, at(1)); + plant(&older_binary, &bytes); + let mut changed = bytes.clone(); + *changed.last_mut().expect("a page") ^= 1; + let other_bytes = super::copy::copy_path(&db, "legacy", LATEST, at(2)); + plant(&other_bytes, &changed); + let opened = super::copy::copy_path(&db, "legacy", LATEST, at(3)); + plant(&opened, &bytes); + plant( + std::path::Path::new(&format!("{}-wal", opened.display())), + b"", + ); + let other_version = super::copy::copy_path(&db, "2", LATEST, at(4)); + plant(&other_version, &bytes); + + let report = run(&db).await.expect("migrate"); + for path in [&stopped, &older_binary] { + assert!(!path.exists(), "{} is left", path.display()); + let removed = format!( + "removed {}, a copy an earlier run took of the same database contents", + path.display() + ); + assert!( + report.notes.iter().any(|n| n.starts_with(&removed)), + "{:?}", + report.notes + ); + } + for path in [&other_bytes, &opened, &other_version] { + assert!(path.exists(), "{} was removed", path.display()); + } + + super::tests::next_second().await; + let report = super::tests::reconverge(&db).await.expect("reconverge"); + assert!( + report.notes.iter().all(|n| !n.starts_with("removed")), + "{:?}", + report.notes + ); + for path in [&other_bytes, &opened, &other_version] { + assert!(path.exists(), "{} was removed", path.display()); + } +} + +/// A run that stops before its first commit left the database as it was, +/// so it removes its own copy, and the earlier copy of the same contents +/// its copy replaced, and names that one with its error. +#[tokio::test] +async fn a_run_that_commits_nothing_names_the_earlier_copy_it_replaced() { + let db = temp_db("superseded-stopped"); + load_dump(&db, DEV_0_1_14).await; + exec(&db, "ALTER TABLE tables ADD COLUMN not_in_any_release TEXT").await; + let earlier = super::copy::copy_path( + &db, + "legacy", + LATEST, + std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_767_225_600), + ); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + super::copy::pre_migration_copy(migrations.pool(), &db, &earlier, super::copy::Hooks::RUN) + .await + .expect("a killed run's copy"); + migrations.close().await; + let err = run(&db).await.expect_err("the post-check refuses"); + let MigrationError::Stopped { error, report } = &err else { + panic!("expected the removal named with the error: {err:?}"); + }; + assert!(report.applied.is_empty(), "{report:?}"); + assert!( + error.to_string().contains("nothing was committed"), + "{error}" + ); + let removed = format!( + "removed {}, a copy an earlier run took of the same database contents", + earlier.display() + ); + assert!( + report.notes.iter().any(|n| n.starts_with(&removed)), + "{:?}", + report.notes + ); + assert_eq!( + super::tests::copies(&db), + Vec::::new(), + "a copy of the unchanged database is left" + ); +} + +/// A run killed after its copy is complete and before its first commit +/// leaves its copy, and the next run's copy replaces it as soon as it is +/// complete: runs killed there one after the other, as in a crash loop, +/// leave one copy. +#[tokio::test] +async fn runs_killed_before_their_first_commit_leave_one_copy() { + let db = temp_db("killed-loop"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + for _ in 0..3 { + let migrations = SqliteMigrations::open(&location).await.expect("open"); + migrations + .before_first_write(None, LATEST) + .await + .expect("copy"); + // Killed here: nothing reports the run or removes its copy. + migrations.close().await; + let copies = super::tests::copies(&db); + assert_eq!(copies.len(), 1, "{copies:?}"); + super::tests::next_second().await; + } +} + +/// A run refused by migration 1's post-check committed nothing, so it +/// removes the copy it took: two failed runs leave no copy, and each +/// reports the post-check alone. +#[tokio::test] +async fn a_run_that_commits_nothing_removes_its_copy() { + let db = temp_db("failed-first-step"); + load_dump(&db, DEV_0_1_14).await; + exec(&db, "ALTER TABLE tables ADD COLUMN not_in_any_release TEXT").await; + for _ in 0..2 { + let err = run(&db).await.expect_err("the post-check refuses"); + assert!(!matches!(err, MigrationError::Stopped { .. }), "{err:?}"); + assert!(err.to_string().contains("nothing was committed"), "{err}"); + assert_eq!(super::tests::copies(&db), Vec::::new()); + super::tests::next_second().await; + } + assert_eq!(version_row(&db).await.as_deref(), Some("0.0.3")); +} + +/// No SQLite migration is declared additive until CI runs the previous +/// release against a catalog this binary migrated; the PR that adds that +/// job deletes this test. +#[test] +fn no_migration_is_additive_before_the_old_binary_job() { + extenddb_storage::migrations::refuse_additive("sqlite", super::migration) + .unwrap_or_else(|e| panic!("{e}")); +} + +/// `migrate --yes` on a path with no database refuses it as `Uninitialized` +/// (exit 1, naming `extenddb init`) and creates no file and no directory; +/// on a file that is not a catalog it refuses under the lock and writes +/// nothing. +#[tokio::test] +async fn migrate_yes_refuses_a_store_that_is_not_a_catalog() { + use extenddb_storage::migrations::{CatalogMigrations, RunKind, Target, run_locked}; + + use super::SqliteCatalogMigrations; + + let db = temp_db("not-a-catalog"); + let missing_dir = db.with_file_name("missing-dir"); + let missing = Target::at(missing_dir.join("extenddb.sqlite").to_string_lossy()); + let err = SqliteCatalogMigrations + .lock(&missing, std::time::Duration::ZERO) + .await + .expect_err("no database"); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + assert_eq!(err.exit_code(), exit::ERROR); + assert!(!missing_dir.exists(), "the lock created a directory"); + + std::fs::write(&*db, b"").expect("empty file"); + let empty = Target::at(db.to_string_lossy()); + for kind in [RunKind::Migrate, RunKind::Reconverge] { + let lock = SqliteCatalogMigrations + .lock(&empty, std::time::Duration::ZERO) + .await + .expect("lock"); + let err = run_locked(&SqliteCatalogMigrations, &empty, &lock, RELEASE, kind) + .await + .expect_err("not a catalog"); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + } + assert_eq!(std::fs::metadata(&*db).expect("file").len(), 0); + assert!(!db.with_file_name("db.sqlite-wal").exists()); +} + +/// A run that committed a step before it stopped keeps its copy and names +/// it in the error; one that committed nothing removes it. +#[tokio::test] +async fn a_stopped_run_keeps_its_copy_only_after_a_commit() { + use extenddb_storage::migrations::{AppliedStep, RunReport}; + + let db = temp_db("stopped-copy"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let stopped = |notes: Vec, applied: Vec| { + Err(MigrationError::Stopped { + error: Box::new(MigrationError::Store("migration 3 failed".to_owned())), + report: RunReport { + from: None, + to: Some(2), + applied, + notes, + }, + }) + }; + let committed = vec![AppliedStep { + id: 1, + name: "baseline", + store_name: location.clone(), + notes: Vec::new(), + }]; + + let migrations = SqliteMigrations::open(&location).await.expect("open"); + let notes = migrations + .before_first_write(None, LATEST) + .await + .expect("copy"); + let kept = migrations.discard_unused_copy(stopped(notes.clone(), committed)); + let Err(MigrationError::Stopped { report, .. }) = kept else { + panic!("expected Stopped: {kept:?}"); + }; + assert_eq!(report.notes, notes); + let [copy] = super::tests::copies(&db).try_into().expect("one copy"); + assert!(notes[0].contains(©.display().to_string()), "{notes:?}"); + migrations.close().await; + std::fs::remove_file(©).expect("remove the kept copy"); + + super::tests::next_second().await; + let migrations = SqliteMigrations::open(&location).await.expect("open"); + let notes = migrations + .before_first_write(None, LATEST) + .await + .expect("copy"); + assert_eq!(super::tests::copies(&db).len(), 1); + let removed = migrations.discard_unused_copy(stopped(notes, Vec::new())); + assert_eq!( + removed, + Err(MigrationError::Store("migration 3 failed".to_owned())) + ); + assert_eq!(super::tests::copies(&db), Vec::::new()); + migrations.close().await; +} + +/// A step whose `COMMIT` succeeded is reported as applied even when the +/// `PRAGMA foreign_keys = ON` after it fails: the run stops with that error +/// at its next step and keeps its copy, since the database has changed. +#[tokio::test] +async fn a_failed_pragma_after_a_commit_keeps_the_copy() { + use extenddb_storage::migrations::Runner; + + let db = temp_db("pragma-after-commit"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let migrations = SqliteMigrations::open(&location).await.expect("open"); + migrations + .fail_restore + .store(true, std::sync::atomic::Ordering::SeqCst); + let result = Runner::new(&migrations, RELEASE).migrate().await; + let result = migrations.discard_unused_copy(result); + migrations.close().await; + let Err(MigrationError::Stopped { error, report }) = result else { + panic!("expected Stopped: {result:?}"); + }; + assert!( + error + .to_string() + .contains("was committed, and then PRAGMA foreign_keys = ON"), + "{error}" + ); + let applied: Vec = report.applied.iter().map(|s| s.id).collect(); + assert_eq!(applied, [1]); + assert_eq!( + super::tests::copies(&db).len(), + 1, + "the copy of a changed database was removed" + ); +} + +/// A body and its ledger row commit together: when the row's insert fails +/// after the body ran, none of the body's changes are left, and the id is +/// still pending. Migration 2's body rewrites the policy this test stores. +#[tokio::test] +async fn a_failed_ledger_insert_leaves_nothing_of_the_body() { + let db = temp_db("ledger-insert"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let migrations = SqliteMigrations::open(&location).await.expect("open"); + Runner::new(&migrations, RELEASE) + .migrate_through(MigrationId::Baseline) + .await + .expect("migration 1"); + migrations.close().await; + let pool = plain_pool(&db).await; + let (account, user): (String, String) = + sqlx::query_as("SELECT account_id, user_name FROM iam_users LIMIT 1") + .fetch_one(&pool) + .await + .expect("an IAM user"); + sqlx::query( + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, \ + policy_document) VALUES (?, 'user', ?, 'tagged', ?)", + ) + .bind(&account) + .bind(&user) + .bind( + r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}"#, + ) + .execute(&pool) + .await + .expect("policy"); + pool.close().await; + let document = "SELECT policy_document FROM iam_policies WHERE policy_name = 'tagged'"; + let before = scalar(&db, document).await; + + let migrations = SqliteMigrations::open(&location).await.expect("open"); + migrations + .fail_insert_row + .store(true, std::sync::atomic::Ordering::SeqCst); + let result = Runner::new(&migrations, RELEASE).migrate().await; + let result = migrations.discard_unused_copy(result); + migrations.close().await; + let err = result.expect_err("the insert fails"); + assert!(err.to_string().contains("record migration 2"), "{err}"); + assert_eq!( + scalar(&db, document).await, + before, + "the body's rewrite is left" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM extenddb_migrations").await, + 1 + ); + let CatalogStatus::Behind { pending, .. } = status(&db).await else { + panic!("migration 2 is pending"); + }; + assert_eq!( + pending[0].id, + MigrationId::ResourceTagConditionKeys.number() + ); +} + +/// `forget` deletes the ledger row and writes the version row in one +/// transaction: when the version row cannot be written, the ledger row is +/// still there. +#[tokio::test] +async fn a_forget_whose_version_row_fails_keeps_the_ledger_row() { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::SqliteCatalogMigrations; + + let db = temp_db("forget-one-transaction"); + run(&db).await.expect("fresh"); + exec( + &db, + "CREATE TRIGGER no_version BEFORE UPDATE ON settings \ + WHEN NEW.key = 'catalog_version' \ + BEGIN SELECT RAISE(ABORT, 'the version row cannot be written'); END;", + ) + .await; + let target = Target::at(db.to_string_lossy()); + let lock = SqliteCatalogMigrations + .lock(&target, std::time::Duration::ZERO) + .await + .expect("lock"); + let err = SqliteCatalogMigrations + .forget(&lock, LATEST) + .await + .expect_err("the version row fails"); + drop(lock); + assert!( + err.to_string() + .contains("the version row cannot be written"), + "{err}" + ); + assert_eq!( + count( + &db, + &format!("SELECT COUNT(*) FROM extenddb_migrations WHERE id = {LATEST}") + ) + .await, + 1 + ); + assert_eq!(version_row(&db).await, Some(LATEST.to_string())); +} diff --git a/crates/storage-sqlite/src/migrations/released_tests.rs b/crates/storage-sqlite/src/migrations/released_tests.rs new file mode 100644 index 000000000..43244a10b --- /dev/null +++ b/crates/storage-sqlite/src/migrations/released_tests.rs @@ -0,0 +1,576 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Every released SQLite catalog, upgraded and compared with a fresh install +//! seeded the same way: catalog objects, per-table objects with ids mapped to +//! names, settings rows, the rows kept, and backup items row by row. + +use std::collections::BTreeMap; +use std::path::Path; + +use extenddb_storage::MetadataEngine; +use extenddb_storage::migrations::{CatalogStatus, MigrationId, Runner, exit}; + +use super::SqliteMigrations; +use super::postcheck; +use super::tests::{ + DEV_0_1_14, DEV_0_1_14_BACKUP, LATEST, RELEASE, count, exec, load_dump, next_second, + plain_pool, reconverge, run, scalar, status, temp_db, version_row, +}; + +/// Settings rows a released baseline seeded with an equal default in code. +/// v1.0 seeds none of them and keeps the ones it finds, because a seeded +/// value cannot be told from one an operator set. +const DEV_IMAGE_SETTINGS: &[(&str, &str)] = &[ + ("control_plane_delay_seconds", "0.25"), + ("index_propagation_delay_ms", "10"), +]; +/// Before v0.1.7 the index delay key was `gsi_propagation_delay_ms`. +const V0_1_3_SETTINGS: &[(&str, &str)] = &[ + ("control_plane_delay_seconds", "0.25"), + ("gsi_propagation_delay_ms", "10"), +]; + +/// A released state: its name, its dump, whether it holds the seeded table, +/// and the settings rows it is accepted to keep beyond a fresh install's. +struct State { + name: &'static str, + dump: &'static str, + seeded: bool, + settings: &'static [(&'static str, &'static str)], +} + +macro_rules! dev_image { + ($name:literal) => { + State { + name: $name, + dump: include_str!(concat!("../../tests/fixtures/released/", $name, ".sql")), + seeded: true, + settings: DEV_IMAGE_SETTINGS, + } + }; +} + +const RELEASED_STATES: &[State] = &[ + State { + name: "v0.1.3-to-v0.1.6", + dump: include_str!("../../tests/fixtures/released/v0.1.3-to-v0.1.6.sql"), + seeded: false, + settings: V0_1_3_SETTINGS, + }, + dev_image!("dev-image-0.1.7"), + dev_image!("dev-image-0.1.8"), + dev_image!("dev-image-0.1.10"), + dev_image!("dev-image-0.1.11"), + dev_image!("dev-image-0.1.12"), + dev_image!("dev-image-0.1.13"), + dev_image!("dev-image-0.1.14"), + dev_image!("dev-image-0.1.14-backup"), +]; + +/// Rows only a bootstrap writes, absent from the catalog-only fixture. +const BOOTSTRAP_SETTINGS: &[&str] = &["default_account_id", "encryption_key"]; + +/// The database's objects with per-table ids replaced by `T_
` and +/// `I_T_
_`. +async fn mapped_definitions(db: &Path) -> BTreeMap> { + let pool = plain_pool(db).await; + let defs = postcheck::definitions(&pool).await.expect("definitions"); + let mut ids: Vec<(String, String)> = Vec::new(); + if count( + db, + "SELECT COUNT(*) FROM sqlite_master WHERE name = 'tables'", + ) + .await + > 0 + { + let tables: Vec<(String, String)> = + sqlx::query_as("SELECT table_id, table_name FROM tables") + .fetch_all(&pool) + .await + .expect("tables"); + let indexes: Vec<(String, String, String)> = sqlx::query_as( + "SELECT i.index_id, t.table_name, i.index_name FROM indexes i \ + JOIN tables t ON t.table_id = i.table_id", + ) + .fetch_all(&pool) + .await + .expect("indexes"); + ids.extend( + tables + .into_iter() + .map(|(id, name)| (id, format!("T_{name}"))), + ); + ids.extend( + indexes + .into_iter() + .map(|(id, t, i)| (id, format!("I_T_{t}_{i}"))), + ); + } + pool.close().await; + let map = |text: &str| { + ids.iter().fold(text.to_owned(), |acc, (id, name)| { + acc.replace(id.as_str(), name) + }) + }; + defs.into_values() + .map(|o| { + let mut definition: Vec = o.definition.iter().map(|p| map(p)).collect(); + definition.sort_unstable(); + (format!("{} {}", o.kind, map(&o.name)), definition) + }) + .collect() +} + +async fn settings(db: &Path) -> BTreeMap { + let pool = plain_pool(db).await; + let rows: Vec<(String, String)> = sqlx::query_as("SELECT key, value FROM settings") + .fetch_all(&pool) + .await + .expect("settings"); + pool.close().await; + rows.into_iter().collect() +} + +/// Row counts of every table but the ones a migration writes. +async fn row_counts(db: &Path) -> BTreeMap { + let pool = plain_pool(db).await; + let tables: Vec = sqlx::query_scalar( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' \ + AND name NOT IN ('settings', 'extenddb_migrations')", + ) + .fetch_all(&pool) + .await + .expect("tables"); + let mut counts = BTreeMap::new(); + for table in tables { + let n: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM \"{table}\"")) + .fetch_one(&pool) + .await + .expect("count"); + counts.insert(table, n); + } + pool.close().await; + counts +} + +/// Every backup item's content, in the order of its rowid (its `id` once +/// migration 1 rebuilt the table). +async fn backup_items(db: &Path) -> Vec<(i64, String, String, Option, String)> { + let pool = plain_pool(db).await; + let rows = sqlx::query_as( + "SELECT rowid, backup_arn, pk, sk, item_data FROM backup_items ORDER BY rowid", + ) + .fetch_all(&pool) + .await + .expect("backup items"); + pool.close().await; + rows +} + +/// A backup whose items have sort keys, in the columns every release wrote. +const BACKUP_WITH_SORT_KEYS: &str = "\ + INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, key_schema, \ + attribute_definitions) VALUES ('arn:with-sort-keys', 'b', 'x', 'cu_orders', '0', \ + '[{\"AttributeName\":\"pk\",\"KeyType\":\"HASH\"},{\"AttributeName\":\"sk\",\"KeyType\":\"RANGE\"}]', \ + '[{\"AttributeName\":\"pk\",\"AttributeType\":\"S\"},{\"AttributeName\":\"sk\",\"AttributeType\":\"S\"}]'); \ + INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES \ + ('arn:with-sort-keys', 'p0', 's01', '{\"pk\":{\"S\":\"p0\"},\"sk\":{\"S\":\"s01\"}}'), \ + ('arn:with-sort-keys', 'p0', 's02', '{\"pk\":{\"S\":\"p0\"},\"sk\":{\"S\":\"s02\"}}');"; + +/// A v1.0 catalog seeded as the fixtures were: the bootstrap rows, then +/// CreateTable `cu_orders` with its GSI, LSI and stream, and its TTL index. +async fn fresh_seeded() -> BTreeMap> { + let db = temp_db("fresh-seeded"); + run(&db).await.expect("fresh install"); + exec( + &db, + "INSERT INTO accounts (account_id, account_name) VALUES ('000000000000', 'default'); \ + INSERT INTO settings (key, value) VALUES ('default_account_id', '000000000000'); \ + INSERT INTO settings (key, value) VALUES ('encryption_key', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=');", + ) + .await; + let engine = crate::SqliteEngine::new(&db.to_string_lossy(), 2, "us-east-1", 400_000) + .await + .expect("engine"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(serde_json::json!({ + "TableName": "cu_orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "gsk", "AttributeType": "N"}, + {"AttributeName": "lsk", "AttributeType": "S"} + ], + "GlobalSecondaryIndexes": [{ + "IndexName": "by_gpk", + "KeySchema": [ + {"AttributeName": "gpk", "KeyType": "HASH"}, + {"AttributeName": "gsk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + }], + "LocalSecondaryIndexes": [{ + "IndexName": "by_lsk", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "lsk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"} + }], + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input, false) + .await + .expect("create table"); + MetadataEngine::create_ttl_index(&engine, "000000000000", "cu_orders", "expires") + .await + .expect("ttl index"); + engine.pool.close().await; + mapped_definitions(&db).await +} + +#[tokio::test] +async fn every_released_catalog_upgrades_to_the_fresh_schema() { + let fresh = fresh_seeded().await; + assert!( + fresh.contains_key("index idx_base_key_I_T_cu_orders_by_gpk"), + "{fresh:#?}" + ); + for state in RELEASED_STATES { + let name = state.name; + let db = temp_db(name); + load_dump(&db, state.dump).await; + exec(&db, BACKUP_WITH_SORT_KEYS).await; + let before = row_counts(&db).await; + let items_before = backup_items(&db).await; + let settings_before = settings(&db).await; + let status_before = status(&db).await; + assert!( + matches!(status_before, CatalogStatus::Behind { version: None, .. }), + "{name}: {status_before:?}" + ); + assert_eq!(status_before.exit_code(), exit::PENDING); + + let report = run(&db).await.unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!((report.from, report.to), (None, Some(LATEST)), "{name}"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST }, + "{name}" + ); + assert!(run(&db).await.expect("again").applied.is_empty(), "{name}"); + + // Rows: every table keeps its count, and every backup item its + // content and its order, with its old rowid as its id. + let after = row_counts(&db).await; + for (table, n) in &before { + assert_eq!(after.get(table), Some(n), "{name}: rows of {table}"); + } + assert_eq!( + backup_items(&db).await, + items_before, + "{name}: backup items" + ); + + // Objects, per-table ones mapped to names. + let upgraded = mapped_definitions(&db).await; + let expected: BTreeMap<&String, &Vec> = fresh + .iter() + .filter(|(object, _)| state.seeded || !object.contains("T_cu_orders")) + .collect(); + let actual: BTreeMap<&String, &Vec> = upgraded.iter().collect(); + assert_eq!( + actual, expected, + "{name}: objects differ from a fresh install" + ); + + // Settings: a fresh install's version row, the bootstrap rows as the + // release wrote them, and the state's accepted rows with its values. + let mut expected_settings: BTreeMap = + [("catalog_version".to_owned(), LATEST.to_string())].into(); + if state.seeded { + for key in BOOTSTRAP_SETTINGS { + let value = settings_before + .get(*key) + .unwrap_or_else(|| panic!("{name}: {key}")); + expected_settings.insert((*key).to_owned(), value.clone()); + } + } + for (key, value) in state.settings { + expected_settings.insert((*key).to_owned(), (*value).to_owned()); + } + assert_eq!(settings(&db).await, expected_settings, "{name}: settings"); + + // No post-check note: no released state has a pre-release shape. + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1").await; + let rebuilt = format!( + "rebuilt backup_items with an id column, copying {} row(s); the old table's \ + pages stay free inside the database file until a VACUUM", + items_before.len() + ); + assert_eq!(notes, Some(rebuilt), "{name}"); + + // A reconverge reruns migration 1 and leaves every item as it is. + exec( + &db, + "INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'x')", + ) + .await; + reconverge(&db) + .await + .unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!( + backup_items(&db).await, + items_before, + "{name}: after a reconverge" + ); + } +} + +/// A release before 1.0 that starts on a v1.0 catalog rewrites +/// `catalog_version` (SQLite dev builds) or the marker (`migrate --yes`). +/// v1.0 then reports it, refuses to migrate, and `--reconverge` repairs it, +/// leaving the ledger rows as the first application wrote them. +#[tokio::test] +async fn a_catalog_an_older_release_wrote_to_is_reported_and_reconverged() { + let db = temp_db("older-release"); + load_dump(&db, DEV_0_1_14).await; + run(&db).await.expect("upgrade"); + let ledger = |db: &Path| { + let db = db.to_path_buf(); + async move { + let pool = plain_pool(&db).await; + let rows: Vec<(i64, String, String, String, String)> = sqlx::query_as( + "SELECT id, checksum, notes, applied_at, applied_by FROM extenddb_migrations \ + ORDER BY id", + ) + .fetch_all(&pool) + .await + .expect("ledger"); + pool.close().await; + rows + } + }; + let rows = ledger(&db).await; + for trace in [ + "UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'", + "INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied')", + ] { + exec(&db, trace).await; + let found = status(&db).await; + assert_eq!(found.label(), "WrittenByOlderRelease", "{trace}"); + assert_eq!(found.exit_code(), exit::ATTENTION); + let err = run(&db).await.expect_err("migrate refuses"); + assert_eq!(err.exit_code(), exit::ATTENTION); + + next_second().await; + let report = reconverge(&db).await.expect("reconverge"); + // Every convergent migration ran again; migration 2, which is not + // convergent, is named as skipped. + let convergent = MigrationId::ALL + .into_iter() + .filter(|id| super::migration(*id).convergent()) + .count(); + assert_eq!(report.applied.len(), convergent, "{trace}: {report:?}"); + let skipped = format!("Skipped {}:", MigrationId::ResourceTagConditionKeys); + assert!( + report.notes.iter().any(|n| n.starts_with(&skipped)), + "{trace}: {report:?}" + ); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST }, + "{trace}" + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM settings WHERE key = 'resource_tag_migration'" + ) + .await, + 0 + ); + assert_eq!(ledger(&db).await, rows, "{trace}: the ledger rows are kept"); + } +} + +/// Running migration 1 again on a catalog at a later migration keeps what a +/// later migration added to the tables migration 1 creates: a column of +/// `tables` and one of `backup_items`, with their values. +#[tokio::test] +async fn a_reconverge_keeps_what_a_later_migration_added() { + let db = temp_db("later-columns"); + load_dump(&db, DEV_0_1_14_BACKUP).await; + run(&db).await.expect("upgrade"); + exec( + &db, + "ALTER TABLE tables ADD COLUMN later_note TEXT; \ + UPDATE tables SET later_note = 'written by a later migration'; \ + ALTER TABLE backup_items ADD COLUMN later_note TEXT; \ + UPDATE backup_items SET later_note = 'item ' || id;", + ) + .await; + let dump = |db: &Path, table: &'static str| { + let db = db.to_path_buf(); + async move { + let pool = plain_pool(&db).await; + let columns: Vec = + sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")) + .fetch_all(&pool) + .await + .expect("columns"); + let rows: Vec = sqlx::query_scalar(&format!( + "SELECT {} FROM {table} ORDER BY rowid", + columns + .iter() + .map(|c| format!("quote({c})")) + .collect::>() + .join(" || '|' || ") + )) + .fetch_all(&pool) + .await + .expect("rows"); + pool.close().await; + (columns, rows) + } + }; + let tables = dump(&db, "tables").await; + let items = dump(&db, "backup_items").await; + assert!(tables.0.contains(&"later_note".to_owned())); + assert_eq!(items.1.len(), 10, "the backup's items"); + exec( + &db, + "INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'x')", + ) + .await; + reconverge(&db).await.expect("reconverge"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert_eq!(dump(&db, "tables").await, tables); + assert_eq!(dump(&db, "backup_items").await, items); +} + +/// `migrate_through` stops at an intermediate id; the rest applies later. +#[tokio::test] +async fn a_released_catalog_can_stop_at_an_intermediate_migration() { + let db = temp_db("through"); + load_dump(&db, DEV_0_1_14).await; + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let report = Runner::new(&migrations, RELEASE) + .migrate_through(MigrationId::ResourceTagConditionKeys) + .await + .expect("through 2"); + migrations.close().await; + assert_eq!(report.to, Some(2)); + let CatalogStatus::Behind { version, pending } = status(&db).await else { + panic!("expected Behind"); + }; + assert_eq!(version, Some(2)); + assert_eq!( + pending.iter().map(|p| p.id).collect::>(), + (3..=LATEST).collect::>() + ); + run(&db).await.expect("the rest"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); +} + +/// The statuses a ledger alone decides, on a SQLite catalog: a changed +/// checksum, ids above this binary's, and `forget`. +#[tokio::test] +async fn changed_ahead_and_forgotten_ledgers() { + let db = temp_db("ledger-states"); + run(&db).await.expect("fresh install"); + let future = LATEST + 1; + + // An id a newer binary recorded as additive: served, and migrate leaves it. + exec( + &db, + &format!( + "INSERT INTO extenddb_migrations (id, name, kind, checksum, outcome, applied_by) \ + VALUES ({future}, 'future', 'additive', 'x', 'applied', '1.1.0')" + ), + ) + .await; + let found = status(&db).await; + assert_eq!(found.label(), "AheadCompatible", "{found:?}"); + assert_eq!(found.exit_code(), exit::OK); + assert!(run(&db).await.expect("migrate").applied.is_empty()); + + // A breaking one: refused. + exec( + &db, + &format!("UPDATE extenddb_migrations SET kind = 'breaking' WHERE id = {future}"), + ) + .await; + assert_eq!(status(&db).await.exit_code(), exit::AHEAD_INCOMPATIBLE); + assert_eq!( + run(&db).await.expect_err("refused").exit_code(), + exit::AHEAD_INCOMPATIBLE + ); + exec( + &db, + &format!("DELETE FROM extenddb_migrations WHERE id = {future}"), + ) + .await; + + // A body whose recorded checksum differs from this binary's. + let recorded = scalar(&db, "SELECT checksum FROM extenddb_migrations WHERE id = 1") + .await + .expect("row"); + exec( + &db, + "UPDATE extenddb_migrations SET checksum = 'edited' WHERE id = 1", + ) + .await; + let found = status(&db).await; + let CatalogStatus::Changed { + id, + recorded: edited, + .. + } = &found + else { + panic!("{found:?}"); + }; + assert_eq!((*id, edited.as_str()), (1, "edited")); + assert_eq!(found.exit_code(), exit::ATTENTION); + exec( + &db, + &format!("UPDATE extenddb_migrations SET checksum = '{recorded}' WHERE id = 1"), + ) + .await; + + // An unreleased id, migration 4, can be forgotten and runs again. + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let runner = Runner::new(&migrations, RELEASE); + let four = MigrationId::IndexTableBaseKey.number(); + runner.forget(four).await.expect("forget 4"); + assert!(matches!( + runner.status().await.expect("status"), + CatalogStatus::Behind { + version: Some(3), + .. + } + )); + let report = runner.migrate().await.expect("reapply"); + assert_eq!(report.applied.len(), 1); + assert_eq!(report.applied[0].id, four); + migrations.close().await; + assert_eq!(version_row(&db).await, Some(LATEST.to_string())); +} diff --git a/crates/storage-sqlite/src/migrations/serve_tests.rs b/crates/storage-sqlite/src/migrations/serve_tests.rs new file mode 100644 index 000000000..5b432edcb --- /dev/null +++ b/crates/storage-sqlite/src/migrations/serve_tests.rs @@ -0,0 +1,340 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The server factory reads the catalog status before it opens the +//! database: a production build serves only `Current` and `AheadCompatible` +//! and writes nothing when it refuses; a dev build migrates `Uninitialized` +//! and `Behind` first, and refuses the rest; an in-memory database is +//! created by the runner every time. The lock that keeps a migrator, a +//! second migrator and a server apart is tested here too. + +use std::path::Path; +use std::time::Duration; + +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId, exit}; +use extenddb_storage::server_components::{ + BackendError, ServerComponents, ServerComponentsOptions, +}; + +use super::migrate; +use super::tests::{ + DEV_0_1_14, LATEST, RELEASE, copies, exec, load_dump, plain_pool, run, status, temp_db, +}; + +const COLLIDING: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod","dynamodb:ResourceTag/env":"dev"}}}]}"#; + +fn options(dev: bool) -> ServerComponentsOptions { + let mut options = ServerComponentsOptions::default(); + options.bootstrap_if_uninitialized = dev; + options.release = "1.0.0-test"; + options +} + +async fn start(path: &str, dev: bool) -> Result { + let config = crate::config::SqliteConfig { + path: path.to_owned(), + pool_size: 2, + }; + crate::sqlite_server_components_factory(&config, "us-east-1", options(dev)).await +} + +/// The refusal a factory returned, with its exit code. +fn refusal(result: Result) -> MigrationError { + match result { + Err(BackendError::Catalog(e)) => e, + Err(other) => panic!("a refusal with an exit code, not {other}"), + Ok(_) => panic!("the server started"), + } +} + +/// A current catalog as `init` leaves it: every migration, and the +/// encryption key a server reads. +async fn current_catalog(db: &Path) -> String { + let location = db.to_string_lossy().into_owned(); + super::migrate(&location, "1.0.0-test", Duration::from_secs(5)) + .await + .expect("a current catalog"); + exec( + db, + "INSERT INTO settings (key, value) VALUES \ + ('encryption_key', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=')", + ) + .await; + location +} + +fn bytes(db: &Path) -> Vec { + std::fs::read(db).expect("read the database") +} + +#[tokio::test] +async fn a_production_server_refuses_a_missing_catalog_and_creates_no_database() { + let db = temp_db("serve-missing"); + let err = refusal(start(&db.to_string_lossy(), false).await); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + assert_eq!(err.exit_code(), exit::ERROR); + assert!(!db.exists(), "a refusing server created the database"); +} + +/// A dev server creates a catalog in a missing or empty file, and refuses a +/// file that holds another program's tables as `migrate --yes` does, +/// naming `extenddb init`, with the file left as it was. +#[tokio::test] +async fn a_dev_server_refuses_a_file_that_is_not_a_catalog() { + let db = temp_db("serve-dev-foreign"); + exec( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)", + ) + .await; + let before = bytes(&db); + let err = refusal(start(&db.to_string_lossy(), true).await); + assert_eq!(err, MigrationError::Status(CatalogStatus::Uninitialized)); + assert_eq!(bytes(&db), before, "the dev server wrote to the file"); + assert!(copies(&db).is_empty()); + + let empty = temp_db("serve-dev-empty"); + std::fs::write(&empty, b"").expect("an empty file"); + let components = start(&empty.to_string_lossy(), true) + .await + .expect("a dev server creates the catalog in an empty file"); + drop(components); + assert_eq!( + status(&empty).await, + CatalogStatus::Current { version: LATEST } + ); +} + +#[tokio::test] +async fn a_production_server_refuses_a_released_catalog_and_writes_nothing() { + let db = temp_db("serve-released"); + load_dump(&db, DEV_0_1_14).await; + let before = bytes(&db); + let err = refusal(start(&db.to_string_lossy(), false).await); + let MigrationError::Status(CatalogStatus::Behind { version, pending }) = &err else { + panic!("{err:?}"); + }; + assert_eq!(*version, None); + assert_eq!(pending.len(), MigrationId::ALL.len()); + assert_eq!(err.exit_code(), exit::PENDING); + assert_eq!( + bytes(&db), + before, + "a refusing server wrote to the database" + ); + assert!(copies(&db).is_empty()); +} + +#[tokio::test] +async fn a_dev_server_migrates_a_released_catalog_with_its_copy_and_serves() { + let db = temp_db("serve-dev-released"); + load_dump(&db, DEV_0_1_14).await; + let components = start(&db.to_string_lossy(), true) + .await + .expect("a dev server migrates and starts"); + drop(components); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert_eq!( + copies(&db).len(), + 1, + "the dev build copies before migrating" + ); +} + +#[tokio::test] +async fn a_dev_server_creates_a_missing_catalog_and_its_rows() { + let db = temp_db("serve-dev-missing"); + let components = start(&db.to_string_lossy(), true) + .await + .expect("a dev server creates the catalog"); + let key = components + .catalog_store + .get_setting("encryption_key") + .await + .expect("read"); + assert!(key.is_some(), "the dev rows are written"); + drop(components); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert!(copies(&db).is_empty(), "a new database has nothing to copy"); +} + +#[tokio::test] +async fn a_dev_server_refuses_a_catalog_that_needs_attention() { + let db = temp_db("serve-dev-attention"); + load_dump(&db, DEV_0_1_14).await; + let pool = plain_pool(&db).await; + let (account, user): (String, String) = + sqlx::query_as("SELECT account_id, user_name FROM iam_users LIMIT 1") + .fetch_one(&pool) + .await + .expect("an IAM user"); + sqlx::query( + "INSERT INTO iam_policies (account_id, principal_type, principal_name, policy_name, \ + policy_document) VALUES (?, 'user', ?, 'colliding', ?)", + ) + .bind(&account) + .bind(&user) + .bind(COLLIDING) + .execute(&pool) + .await + .expect("policy"); + pool.close().await; + let before = bytes(&db); + let err = refusal(start(&db.to_string_lossy(), true).await); + assert!( + matches!( + err, + MigrationError::Status(CatalogStatus::NeedsAttention { .. }) + ), + "{err:?}" + ); + assert_eq!(err.exit_code(), exit::ATTENTION); + assert_eq!(bytes(&db), before); + assert!(copies(&db).is_empty()); +} + +#[tokio::test] +async fn a_server_refuses_while_a_migration_holds_the_database() { + let db = temp_db("serve-locked"); + let location = current_catalog(&db).await; + let lock = super::lock_for_migration(&location, Duration::ZERO) + .await + .expect("the migration lock"); + let err = refusal(start(&location, false).await); + assert!(matches!(err, MigrationError::Locked(_)), "{err:?}"); + assert_eq!(err.exit_code(), exit::LOCKED); + drop(lock); + drop( + start(&location, false) + .await + .expect("starts once the lock is free"), + ); +} + +#[tokio::test] +async fn a_server_serves_a_catalog_ahead_by_additive_migrations_only() { + let db = temp_db("serve-ahead"); + let location = current_catalog(&db).await; + exec( + &db, + &format!( + "INSERT INTO extenddb_migrations (id, name, kind, checksum, outcome, notes, \ + applied_by) VALUES ({}, 'later', 'additive', 'x', 'applied', '', 'later')", + LATEST + 1 + ), + ) + .await; + drop( + start(&location, false) + .await + .expect("an additive catalog serves"), + ); + exec( + &db, + &format!( + "UPDATE extenddb_migrations SET kind = 'breaking' WHERE id = {}", + LATEST + 1 + ), + ) + .await; + let err = refusal(start(&location, false).await); + assert_eq!(err.exit_code(), exit::AHEAD_INCOMPATIBLE, "{err:?}"); +} + +#[tokio::test] +async fn an_in_memory_server_creates_its_catalog_through_the_runner() { + let components = start(":memory:", false) + .await + .expect("an in-memory server starts"); + let version = components + .catalog_store + .get_setting("catalog_version") + .await + .expect("read"); + assert_eq!(version, Some(LATEST.to_string())); +} + +#[tokio::test] +async fn two_migrators_apply_once() { + let db = temp_db("two"); + load_dump(&db, DEV_0_1_14).await; + let (a, b) = tokio::join!(run(&db), run(&db)); + let (a, b) = (a.expect("first"), b.expect("second")); + assert_eq!( + [a.applied.is_empty(), b.applied.is_empty()] + .iter() + .filter(|e| **e) + .count(), + 1, + "one applies and the other finds nothing to do: {a:?} {b:?}" + ); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert_eq!(copies(&db).len(), 1); +} + +#[cfg(unix)] +#[tokio::test] +async fn the_lock_is_retried_and_times_out_with_exit_6() { + let db = temp_db("lock"); + run(&db).await.expect("fresh"); + let server = crate::serve_lock::ServeLock::acquire(&db).expect("server lock"); + let started = std::time::Instant::now(); + let err = super::lock_for_migration(&db.to_string_lossy(), Duration::from_millis(600)) + .await + .expect_err("held"); + assert!(started.elapsed() >= Duration::from_millis(500), "it waited"); + assert_eq!(err.exit_code(), exit::LOCKED); + // Released while a migrator waits: the migrator proceeds. + let release = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(400)).await; + drop(server); + }); + super::lock_for_migration(&db.to_string_lossy(), Duration::from_secs(5)) + .await + .expect("taken after the server stops") + .expect("a file database"); + release.await.expect("release"); +} + +/// The server's factory holds the serve lock before it opens the engine; a +/// second `flock` from the same process is refused, so it migrates through +/// the entry point that takes the held lock. +#[cfg(unix)] +#[tokio::test] +async fn a_caller_holding_the_serve_lock_migrates_through_migrate_holding() { + let db = temp_db("holding"); + load_dump(&db, DEV_0_1_14).await; + let location = db.to_string_lossy().into_owned(); + let held = crate::serve_lock::ServeLock::acquire(&db).expect("serve lock"); + let err = migrate(&location, RELEASE, Duration::from_millis(300)) + .await + .expect_err("the lock is this process's own"); + assert_eq!(err.exit_code(), exit::LOCKED); + let report = super::migrate_holding(&location, RELEASE, &held) + .await + .expect("migrate holding the lock"); + assert_eq!(report.to, Some(LATEST)); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + + // Another database's lock is not this one's. + let other = temp_db("holding-other"); + let other_lock = crate::serve_lock::ServeLock::acquire(&other).expect("other lock"); + let refused = super::migrate_holding(&location, RELEASE, &other_lock).await; + assert!( + matches!(refused, Err(MigrationError::Refused(_))), + "{refused:?}" + ); + drop(held); +} diff --git a/crates/storage-sqlite/src/migrations/tests.rs b/crates/storage-sqlite/src/migrations/tests.rs new file mode 100644 index 000000000..a837e802e --- /dev/null +++ b/crates/storage-sqlite/src/migrations/tests.rs @@ -0,0 +1,540 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The SQLite runner: checksums, the expected definitions, the connection +//! protocol, the copy, the entry points and the in-memory case. The lock +//! is in `serve_tests`, beside the server's refusal while a migration +//! holds it. +//! Released catalogs are in `released_tests`, migrations 2 to 4 in +//! `golden_tests`, refusals and rollbacks in `refusal_tests`, and the killed +//! migrator in `crash_tests`. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + Body, CatalogStatus, CodeBody, Done, LedgerRow, MigrationBackend, MigrationError, MigrationId, + Outcome, RELEASED_THROUGH, Runner, Step, StoreBody, StoreKind, StoreRef, check_released, +}; +use sqlx::SqlitePool; +use sqlx::sqlite::SqlitePoolOptions; + +use super::{SqliteMigrations, SqliteStores, migrate, migration, postcheck}; + +pub(super) const RELEASE: &str = "1.0.0-test"; + +/// The last id this binary carries. +pub(super) const LATEST: u32 = MigrationId::LATEST.number(); + +/// A database path in a directory of its own, removed when dropped, so a +/// failed test leaves nothing behind. +pub(super) struct TempDb(PathBuf); + +impl std::ops::Deref for TempDb { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for TempDb { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl Drop for TempDb { + fn drop(&mut self) { + if let Some(dir) = self.0.parent() { + let _ = std::fs::remove_dir_all(dir); + } + } +} + +pub(super) fn temp_db(name: &str) -> TempDb { + let dir = std::env::temp_dir().join(format!( + "extenddb-migrations-{name}-{}", + uuid::Uuid::new_v4().simple() + )); + std::fs::create_dir_all(&dir).expect("temp dir"); + TempDb(dir.join("db.sqlite")) +} + +/// A plain pool over the file, as a test reads it. +pub(super) async fn plain_pool(db: &Path) -> SqlitePool { + SqlitePoolOptions::new() + .max_connections(1) + .connect(&format!("sqlite://{}?mode=rwc", db.display())) + .await + .expect("open") +} + +/// Execute a released dump into a new file with the mode a server gives it. +pub(super) async fn load_dump(db: &Path, dump: &str) { + let pool = plain_pool(db).await; + sqlx::raw_sql(dump).execute(&pool).await.expect("load dump"); + sqlx::raw_sql("PRAGMA foreign_keys = ON; PRAGMA journal_mode = WAL") + .execute(&pool) + .await + .expect("pragmas"); + pool.close().await; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(db, std::fs::Permissions::from_mode(0o600)).expect("chmod"); + } +} + +pub(super) async fn exec(db: &Path, sql: &str) { + let pool = plain_pool(db).await; + sqlx::raw_sql(sql).execute(&pool).await.expect(sql); + pool.close().await; +} + +pub(super) async fn scalar(db: &Path, sql: &str) -> Option { + let pool = plain_pool(db).await; + let value: Option = sqlx::query_scalar(sql) + .fetch_optional(&pool) + .await + .expect(sql); + pool.close().await; + value +} + +pub(super) async fn count(db: &Path, sql: &str) -> i64 { + let pool = plain_pool(db).await; + let n: i64 = sqlx::query_scalar(sql).fetch_one(&pool).await.expect(sql); + pool.close().await; + n +} + +pub(super) async fn version_row(db: &Path) -> Option { + scalar( + db, + "SELECT value FROM settings WHERE key = 'catalog_version'", + ) + .await +} + +pub(super) async fn status(db: &Path) -> CatalogStatus { + super::catalog_status(&db.to_string_lossy()) + .await + .expect("status") +} + +pub(super) async fn run( + db: &Path, +) -> Result { + migrate(&db.to_string_lossy(), RELEASE, Duration::from_secs(10)).await +} + +/// Reconverge the database at `db` through the runner. +pub(super) async fn reconverge( + db: &Path, +) -> Result { + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + let report = Runner::new(&migrations, RELEASE).reconverge().await; + migrations.close().await; + report +} + +/// Wait for the next second of the wall clock, so a second run takes a copy +/// with a name of its own. +pub(super) async fn next_second() { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock"); + tokio::time::sleep(Duration::from_nanos( + 1_000_000_000 - u64::from(now.subsec_nanos()) + 1_000_000, + )) + .await; +} + +/// Whether a test rewrites its expected file instead of comparing with it: +/// `EXTENDDB_BLESS_EXPECTED=1`, the rule the PostgreSQL tests use too. +pub(super) fn bless() -> bool { + std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") +} + +/// Every copy taken beside `db`. +pub(super) fn copies(db: &Path) -> Vec { + let name = db.file_name().expect("name").to_string_lossy().into_owned(); + let mut found: Vec = std::fs::read_dir(db.parent().expect("dir")) + .expect("dir") + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| { + p.file_name().is_some_and(|n| { + n.to_string_lossy() + .starts_with(&format!("{name}.pre-migration-")) + }) + }) + .collect(); + found.sort(); + found +} + +pub(super) const DEV_0_1_7: &str = + include_str!("../../tests/fixtures/released/dev-image-0.1.7.sql"); +pub(super) const DEV_0_1_14: &str = + include_str!("../../tests/fixtures/released/dev-image-0.1.14.sql"); +pub(super) const DEV_0_1_14_BACKUP: &str = + include_str!("../../tests/fixtures/released/dev-image-0.1.14-backup.sql"); + +#[test] +fn the_released_bodies_match_their_checksums() { + let lines = check_released("sqlite", migration, super::RELEASED, RELEASED_THROUGH) + .unwrap_or_else(|e| panic!("{e}")); + println!("{lines}"); +} + +/// `file_body!` embeds the file it names under the invoking crate's root and +/// records the file name for the checksum rules, so a file body names its +/// file once; its two-argument form is the same file's text. +#[test] +fn file_body_names_its_file_once() { + let body: StoreBody = + extenddb_storage::file_body!(StoreKind::Catalog, "migrations", "0001_baseline.sql"); + assert_eq!(body.store, StoreKind::Catalog); + let Body::File { name, text } = body.body else { + panic!("file_body! made a code body"); + }; + assert_eq!(name, "0001_baseline.sql"); + assert_eq!(text, include_str!("../../migrations/0001_baseline.sql")); + assert_eq!( + extenddb_storage::file_body!("migrations", "0001_baseline.sql"), + text + ); +} + +/// Migration 1's pinned source is its module and the file it executes. +#[test] +fn migration_1_pins_its_module_and_its_file() { + let source = super::m0001_baseline::SOURCE; + assert!(source.contains("pub(crate) struct Baseline;")); + assert!(source.ends_with(include_str!("../../migrations/0001_baseline.sql"))); + assert!(source.ends_with(super::m0001_baseline::FILE)); +} + +/// The committed expected definitions are what migration 1 creates in an +/// empty database. `EXTENDDB_BLESS_EXPECTED=1` rewrites the file. +#[tokio::test] +async fn the_expected_definitions_are_those_of_a_fresh_baseline() { + let db = temp_db("expected"); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + Runner::new(&migrations, RELEASE) + .migrate_through(MigrationId::Baseline) + .await + .expect("migration 1"); + let generated = postcheck::expected_file(migrations.pool()) + .await + .expect("dump"); + migrations.close().await; + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("migrations/0001_baseline.expected.json"); + if bless() { + std::fs::write(&path, &generated).expect("write expected"); + } + assert_eq!( + super::BASELINE_EXPECTED, + generated, + "regenerate with EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite \ + the_expected_definitions" + ); + assert!(generated.contains("\"name\": \"backup_items\"")); +} + +#[tokio::test] +async fn a_fresh_database_is_current_after_one_run_and_seeds_nothing() { + let db = temp_db("fresh"); + assert_eq!(status(&db).await, CatalogStatus::Uninitialized); + assert!(!db.exists(), "the status does not create the file"); + let report = run(&db).await.expect("migrate"); + assert_eq!((report.from, report.to), (None, Some(LATEST))); + assert!( + report.notes.is_empty(), + "an empty database is not copied: {report:?}" + ); + assert!(copies(&db).is_empty()); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM extenddb_migrations WHERE outcome = 'applied'" + ) + .await, + i64::from(LATEST) + ); + assert_eq!(version_row(&db).await, Some(LATEST.to_string())); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM settings").await, + 1, + "only the version row" + ); + assert!(run(&db).await.expect("again").applied.is_empty()); +} + +/// A file the runner creates is 0600 before SQLite writes to it, and SQLite +/// gives its WAL and shared-memory files the same mode. +#[cfg(unix)] +#[tokio::test] +async fn a_database_the_runner_creates_is_private() { + use std::os::unix::fs::PermissionsExt; + let db = temp_db("private"); + run(&db).await.expect("migrate"); + let mode = |path: &Path| std::fs::metadata(path).expect("file").permissions().mode() & 0o777; + assert_eq!(mode(&db), 0o600); + let pool = plain_pool(&db).await; + sqlx::query("INSERT INTO settings (key, value) VALUES ('probe', '1')") + .execute(&pool) + .await + .expect("write"); + for suffix in ["-wal", "-shm"] { + let sidecar = PathBuf::from(format!("{}{suffix}", db.display())); + assert_eq!(mode(&sidecar), 0o600, "{suffix}"); + } + pool.close().await; +} + +/// A body that rebuilds `tables`, the parent of `indexes`, `stream_shards`, +/// `vector_indexes` and `table_restores`, with the documented procedure. +struct RebuildTables; + +/// The rebuild: a copy with the same definition, the rows, the old table +/// dropped, the copy renamed, and the table's index created again. +async fn rebuild_tables_sql(pool: &SqlitePool) -> String { + let table: String = sqlx::query_scalar( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'tables'", + ) + .fetch_one(pool) + .await + .expect("tables"); + let index: String = sqlx::query_scalar( + "SELECT sql FROM sqlite_master WHERE name = 'idx_tables_pending_transition'", + ) + .fetch_one(pool) + .await + .expect("index"); + let copy = table.replacen("CREATE TABLE tables", "CREATE TABLE tables_new", 1); + assert_ne!(copy, table); + format!( + "{copy}; INSERT INTO tables_new SELECT * FROM tables; DROP TABLE tables; \ + ALTER TABLE tables_new RENAME TO tables; {index};" + ) +} + +#[async_trait] +impl CodeBody for RebuildTables { + fn fingerprint(&self) -> &'static str { + "test/rebuild-tables" + } + async fn detect(&self, _: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + async fn run(&self, stores: &SqliteStores) -> Result { + let sql = rebuild_tables_sql(&stores.pool).await; + sqlx::raw_sql(&sql) + .execute(&stores.pool) + .await + .map_err(|e| MigrationError::store("rebuild", e))?; + Ok(Done::default()) + } +} + +/// Inserts an index row for a table that does not exist. +pub(super) struct Orphan; + +#[async_trait] +impl CodeBody for Orphan { + fn fingerprint(&self) -> &'static str { + "test/orphan" + } + async fn detect(&self, _: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + async fn run(&self, stores: &SqliteStores) -> Result { + sqlx::query( + "INSERT INTO indexes (table_id, index_id, index_name, index_type, key_schema, \ + projection) VALUES ('missing', 'i', 'i', 'GSI', '[]', '{}')", + ) + .execute(&stores.pool) + .await + .map_err(|e| MigrationError::store("orphan", e))?; + Ok(Done::default()) + } +} + +pub(super) fn test_row(id: u32) -> Option { + Some(LedgerRow { + id, + name: "test".into(), + kind: extenddb_storage::migrations::Kind::Breaking, + checksum: "test".into(), + outcome: Outcome::Applied, + notes: String::new(), + applied_at: String::new(), + applied_by: RELEASE.into(), + }) +} + +pub(super) fn catalog_store(db: &Path) -> StoreRef { + StoreRef { + kind: StoreKind::Catalog, + name: db.to_string_lossy().into_owned(), + } +} + +async fn children(db: &Path) -> (i64, i64, i64) { + ( + count(db, "SELECT COUNT(*) FROM indexes").await, + count(db, "SELECT COUNT(*) FROM stream_shards").await, + count(db, "SELECT COUNT(*) FROM stream_records").await, + ) +} + +#[tokio::test] +async fn a_rebuild_of_tables_keeps_its_index_shard_and_stream_rows() { + let db = temp_db("rebuild"); + load_dump(&db, DEV_0_1_14).await; + let before = children(&db).await; + assert_eq!(before, (2, 4, 10)); + let migrations = SqliteMigrations::open(&db.to_string_lossy()) + .await + .expect("open"); + migrations + .apply( + &catalog_store(&db), + MigrationId::IndexTableBaseKey, + Step::Code(&RebuildTables), + test_row(LATEST + 5), + None, + ) + .await + .expect("rebuild under the protocol"); + let on: i64 = sqlx::query_scalar("PRAGMA foreign_keys") + .fetch_one(migrations.pool()) + .await + .expect("pragma"); + assert_eq!(on, 1, "foreign keys are on again after the body"); + migrations.close().await; + assert_eq!(children(&db).await, before); + + // The control: the same rebuild with foreign keys on deletes them. + let control = temp_db("rebuild-control"); + load_dump(&control, DEV_0_1_14).await; + let pool = plain_pool(&control).await; + let sql = rebuild_tables_sql(&pool).await; + sqlx::raw_sql(&format!("PRAGMA foreign_keys = ON; BEGIN; {sql} COMMIT;")) + .execute(&pool) + .await + .expect("rebuild with foreign keys on"); + pool.close().await; + assert_eq!(children(&control).await, (0, 0, 0)); +} + +/// The copy is owner-only whatever the database's mode: 0644 is the mode of +/// a catalog that 0.1.x `init` created and no server opened. +#[cfg(unix)] +#[tokio::test] +async fn the_copy_is_owner_only_and_has_a_name_no_run_reuses() { + use std::os::unix::fs::PermissionsExt; + let db = temp_db("copy"); + load_dump(&db, DEV_0_1_14).await; + std::fs::set_permissions(&db, std::fs::Permissions::from_mode(0o644)).expect("chmod"); + let report = run(&db).await.expect("migrate"); + let found = copies(&db); + assert_eq!(found.len(), 1, "{found:?}"); + let copy = &found[0]; + let name = copy + .file_name() + .expect("name") + .to_string_lossy() + .into_owned(); + let stamp = name + .strip_prefix(&format!("db.sqlite.pre-migration-legacy-to-{LATEST}-")) + .unwrap_or_else(|| panic!("{name}")); + assert_eq!(stamp.len(), "20261029T061706Z".len(), "{stamp}"); + assert!( + stamp.ends_with('Z') && stamp.as_bytes()[8] == b'T', + "{stamp}" + ); + assert_eq!(report.notes.len(), 1); + assert!( + report.notes[0].contains(©.display().to_string()), + "{report:?}" + ); + let mode = std::fs::metadata(copy).expect("copy").permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "the copy is owner-only"); + assert_eq!( + version_row(copy).await.as_deref(), + Some("0.0.3"), + "the copy is the database before the migration" + ); +} + +#[tokio::test] +async fn an_in_memory_database_migrates_on_the_engine_connection() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 400_000) + .await + .expect("engine"); + let migrations = + SqliteMigrations::on_pool(engine.pool.clone(), ":memory:").expect("one connection"); + let runner = Runner::new(&migrations, RELEASE); + assert_eq!( + runner.status().await.expect("status"), + CatalogStatus::Uninitialized + ); + let report = runner.migrate().await.expect("migrate"); + assert!(report.notes.is_empty(), "no copy of an in-memory database"); + assert_eq!( + runner.status().await.expect("status"), + CatalogStatus::Current { version: LATEST } + ); + migrations.close().await; + // The engine's connection is still open, and has foreign keys on. + let on: i64 = sqlx::query_scalar("PRAGMA foreign_keys") + .fetch_one(&engine.pool) + .await + .expect("engine pool"); + assert_eq!(on, 1); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM extenddb_migrations") + .fetch_one(&engine.pool) + .await + .expect("ledger on the engine connection"); + assert_eq!(rows, i64::from(LATEST)); + // The file entry points would open a second, empty database. + let refused = migrate(":memory:", RELEASE, Duration::from_secs(1)).await; + assert!( + matches!(refused, Err(MigrationError::Refused(_))), + "{refused:?}" + ); + let refused = super::catalog_status(":memory:").await; + assert!( + matches!(refused, Err(MigrationError::Refused(_))), + "{refused:?}" + ); +} + +/// A file engine's pool holds several connections, and a body on one of +/// them would run outside the runner's transaction on another. +#[tokio::test] +async fn on_pool_refuses_a_pool_of_several_connections() { + let db = temp_db("on-pool"); + let engine = crate::SqliteEngine::new(&db.to_string_lossy(), 4, "us-east-1", 400_000) + .await + .expect("engine"); + let refused = SqliteMigrations::on_pool(engine.pool.clone(), &db.to_string_lossy()); + assert!( + matches!(refused, Err(MigrationError::Refused(ref m)) if m.contains("allows 4")), + "{:?}", + refused.err() + ); + engine.pool.close().await; +} diff --git a/crates/storage-sqlite/src/operations.rs b/crates/storage-sqlite/src/operations.rs index d34136da1..13551cb0e 100644 --- a/crates/storage-sqlite/src/operations.rs +++ b/crates/storage-sqlite/src/operations.rs @@ -11,8 +11,6 @@ use extenddb_storage::error::StorageError; use extenddb_storage::operations::{ConnectionParts, OperationsEngine}; -use crate::schema::CATALOG_VERSION; - /// Backend operations for the SQLite engine. pub(crate) struct SqliteOperationsEngine; @@ -52,8 +50,26 @@ impl OperationsEngine for SqliteOperationsEngine { Ok(()) } + /// The highest migration this binary carries: the catalog is versioned + /// by its ledger, and its `catalog_version` row holds the same integer. fn catalog_version(&self) -> String { - CATALOG_VERSION.to_string() + extenddb_storage::migrations::MigrationId::LATEST + .number() + .to_string() + } + + fn setting_default(&self, key: &str) -> Option { + use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; + match key { + // Read by CreateTable, DeleteTable and the control-plane worker. + "control_plane_delay_seconds" => { + Some(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS.to_string()) + } + INDEX_PROPAGATION_DELAY_MS => { + Some(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS.to_string()) + } + _ => None, + } } fn is_sensitive_key(&self, _key: &str) -> bool { diff --git a/crates/storage-sqlite/src/schema.rs b/crates/storage-sqlite/src/schema.rs index d04ee5b56..598bbf23c 100644 --- a/crates/storage-sqlite/src/schema.rs +++ b/crates/storage-sqlite/src/schema.rs @@ -1,645 +1,18 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Authoritative SQLite catalog schema. +//! Catalog schema helpers. //! -//! Mirrors the semantics of the reference PostgreSQL catalog schema -//! (`crates/storage-postgres/migrations/001_schema.sql`, catalog version -//! 0.0.2). Table and column names are kept identical to PostgreSQL so the -//! catalog/management stores remain portable across backends. -//! -//! Type mapping PostgreSQL → SQLite: -//! - `JSONB` → `TEXT` (JSON serialized as a string) -//! - `BYTEA` → `BLOB` -//! - `BOOLEAN` → `INTEGER` (0/1) -//! - `BIGINT` → `INTEGER` -//! - `DOUBLE PRECISION` → `REAL` (`±9e999` literals store as `±Infinity`) -//! - `TIMESTAMPTZ` → `TEXT` in RFC 3339 UTC (`YYYY-MM-DDTHH:MM:SS.sssZ`) -//! - `SEQUENCE` → a `seq_counters` row updated inside the write transaction -//! -//! Per-DynamoDB-table data tables (`_ddb_`) are NOT created here; they -//! are created dynamically by `create_table`, exactly as in the PostgreSQL -//! backend. +//! The catalog schema itself is created and upgraded by the migrations in +//! `migrations/` (migration 1 is `migrations/0001_baseline.sql`). Table and +//! column names are kept identical to PostgreSQL so the catalog and +//! management stores stay portable across backends. Per-DynamoDB-table data +//! tables (`_ddb_`) are created by `create_table`, as in the +//! PostgreSQL backend. use extenddb_storage::management_store::{OpError, OpResult}; use sqlx::SqlitePool; -/// Compiled-in catalog version. Single source of truth for the SQLite backend; -/// mirrors the PostgreSQL backend's `CATALOG_VERSION`. -pub const CATALOG_VERSION: extenddb_core::version::CatalogVersion = - extenddb_core::version::CatalogVersion::new(0, 0, 4); - -/// Complete catalog schema, applied once on a fresh database. -/// -/// Every object uses `IF NOT EXISTS` so re-application is harmless. Foreign-key -/// enforcement requires `PRAGMA foreign_keys = ON` on each connection (set in -/// the engine's `after_connect` hook). -pub const SCHEMA_SQL: &str = r#" --- Accounts — multi-account support (REQ-AUTH-005). -CREATE TABLE IF NOT EXISTS accounts ( - account_id TEXT PRIMARY KEY, - account_name TEXT NOT NULL UNIQUE, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - --- Table metadata. -CREATE TABLE IF NOT EXISTS tables ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - table_name TEXT NOT NULL, - key_schema TEXT NOT NULL, - attribute_definitions TEXT NOT NULL, - billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', - provisioned_throughput TEXT, - stream_specification TEXT, - table_status TEXT NOT NULL DEFAULT 'CREATING', - creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - table_size_bytes INTEGER NOT NULL DEFAULT 0, - item_count INTEGER NOT NULL DEFAULT 0, - table_arn TEXT NOT NULL, - table_id TEXT NOT NULL, - ttl_attribute TEXT, - deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, - status_transition_at TEXT, - stream_label TEXT, - ttl_index_ready INTEGER NOT NULL DEFAULT 0, - table_class TEXT, - sse_specification TEXT, - on_demand_throughput TEXT, - PRIMARY KEY (account_id, table_name), - CONSTRAINT tables_table_id_unique UNIQUE (table_id) -); - -CREATE INDEX IF NOT EXISTS idx_tables_pending_transition - ON tables (status_transition_at) - WHERE status_transition_at IS NOT NULL; - --- Index metadata. index_id is always supplied by the engine (no DB-side UUID). -CREATE TABLE IF NOT EXISTS indexes ( - table_id TEXT NOT NULL, - index_id TEXT NOT NULL, - index_name TEXT NOT NULL, - index_type TEXT NOT NULL, - key_schema TEXT NOT NULL, - projection TEXT NOT NULL, - index_status TEXT NOT NULL DEFAULT 'ACTIVE', - provisioned_throughput TEXT, - propagation_delay_ms INTEGER, - PRIMARY KEY (table_id, index_name), - CONSTRAINT indexes_table_id_fkey - FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, - CONSTRAINT chk_propagation_delay_ms_non_negative - CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) -); - --- Vector index metadata. Kept out of `indexes` deliberately: a vector index is --- not described by a key schema, so reusing that table's `key_schema` column --- would mean storing something meaningless in a NOT NULL column. The engine --- supplies index_id, as it does for GSIs. --- --- `search_schema` is nullable because the HASH element is optional (measured --- against the live service): with one the search is partition-scoped and --- SearchConditionExpression is required, without one it spans the table. --- --- `backfilling` mirrors the measured lifecycle: false while CREATING before the --- scan starts, true while it runs, and the member is absent once ACTIVE. Stored --- as an integer so the ACTIVE state is representable as NULL rather than as a --- third boolean value. -CREATE TABLE IF NOT EXISTS vector_indexes ( - table_id TEXT NOT NULL, - index_id TEXT NOT NULL, - index_name TEXT NOT NULL, - dimensions INTEGER NOT NULL, - distance_function TEXT NOT NULL, - vector_attribute TEXT NOT NULL, - search_schema TEXT, - projection TEXT NOT NULL, - index_status TEXT NOT NULL DEFAULT 'CREATING', - backfilling INTEGER, - -- Items the backfill skipped because their stored bytes cannot enter the - -- index (unparseable row, malformed or wrong-dimension vector). NULL until - -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so - -- an operator can see that an ACTIVE index deliberately omits rows, rather - -- than the build looping forever on them or dying part-way. - skipped_item_count INTEGER, - PRIMARY KEY (table_id, index_name), - CONSTRAINT vector_indexes_table_id_fkey - FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, - CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), - CONSTRAINT chk_vector_backfilling_bool - CHECK (backfilling IS NULL OR backfilling IN (0, 1)), - -- An ACTIVE index must not carry the member at all, which is what the - -- service does. Enforced here as well as in core, so a bug in the backend - -- cannot persist a state the wire contract forbids. - CONSTRAINT chk_vector_active_has_no_backfilling - CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id - ON vector_indexes (index_id); - --- Resource tags. -CREATE TABLE IF NOT EXISTS tags ( - resource_arn TEXT NOT NULL, - tag_key TEXT NOT NULL, - tag_value TEXT NOT NULL, - PRIMARY KEY (resource_arn, tag_key) -); - --- Migration tracking. -CREATE TABLE IF NOT EXISTS schema_history ( - filename TEXT PRIMARY KEY, - applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - --- Settings (catalog version, data database name, runtime config). -CREATE TABLE IF NOT EXISTS settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); - --- Stream shards. -CREATE TABLE IF NOT EXISTS stream_shards ( - shard_id TEXT PRIMARY KEY, - table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, - parent_shard_id TEXT, - starting_sequence_number TEXT NOT NULL, - ending_sequence_number TEXT, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - -CREATE INDEX IF NOT EXISTS idx_stream_shards_table ON stream_shards (table_id); - --- Stream records. -CREATE TABLE IF NOT EXISTS stream_records ( - shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, - sequence_number TEXT NOT NULL, - table_id TEXT NOT NULL, - event_name TEXT NOT NULL, - record_data TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (shard_id, sequence_number) -); - -CREATE INDEX IF NOT EXISTS idx_stream_records_created ON stream_records (created_at); - --- Admin users. -CREATE TABLE IF NOT EXISTS admin_users ( - admin_name TEXT PRIMARY KEY, - password_hash TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - --- IAM users. -CREATE TABLE IF NOT EXISTS iam_users ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - user_name TEXT NOT NULL, - user_arn TEXT NOT NULL UNIQUE, - password_hash TEXT, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (account_id, user_name) -); - --- IAM user tags. -CREATE TABLE IF NOT EXISTS iam_user_tags ( - account_id TEXT NOT NULL, - user_name TEXT NOT NULL, - tag_key TEXT NOT NULL, - tag_value TEXT NOT NULL, - PRIMARY KEY (account_id, user_name, tag_key), - FOREIGN KEY (account_id, user_name) - REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE -); - --- Access keys. -CREATE TABLE IF NOT EXISTS access_keys ( - access_key_id TEXT PRIMARY KEY, - secret_key_encrypted BLOB NOT NULL, - account_id TEXT NOT NULL, - user_name TEXT NOT NULL, - is_active INTEGER NOT NULL DEFAULT 1, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - FOREIGN KEY (account_id, user_name) - REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE -); - --- IAM groups. -CREATE TABLE IF NOT EXISTS iam_groups ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - group_name TEXT NOT NULL, - group_arn TEXT NOT NULL UNIQUE, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (account_id, group_name) -); - --- IAM group membership. -CREATE TABLE IF NOT EXISTS iam_group_members ( - account_id TEXT NOT NULL, - group_name TEXT NOT NULL, - user_name TEXT NOT NULL, - PRIMARY KEY (account_id, group_name, user_name), - FOREIGN KEY (account_id, group_name) - REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, - FOREIGN KEY (account_id, user_name) - REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE -); - --- IAM roles. -CREATE TABLE IF NOT EXISTS iam_roles ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - role_name TEXT NOT NULL, - role_arn TEXT NOT NULL UNIQUE, - trust_policy TEXT NOT NULL, - permissions_boundary_arn TEXT, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (account_id, role_name) -); - --- IAM role tags. -CREATE TABLE IF NOT EXISTS iam_role_tags ( - account_id TEXT NOT NULL, - role_name TEXT NOT NULL, - tag_key TEXT NOT NULL, - tag_value TEXT NOT NULL, - PRIMARY KEY (account_id, role_name, tag_key), - FOREIGN KEY (account_id, role_name) - REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE -); - --- IAM sessions. -CREATE TABLE IF NOT EXISTS iam_sessions ( - session_token TEXT PRIMARY KEY, - access_key_id TEXT NOT NULL UNIQUE, - secret_key_encrypted BLOB NOT NULL, - account_id TEXT NOT NULL, - role_name TEXT NOT NULL, - session_name TEXT NOT NULL, - session_tags TEXT, - session_policy TEXT, - expires_at TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - FOREIGN KEY (account_id, role_name) - REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE -); - --- IAM policies. -CREATE TABLE IF NOT EXISTS iam_policies ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), - principal_name TEXT NOT NULL, - policy_name TEXT NOT NULL, - policy_document TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (account_id, principal_type, principal_name, policy_name) -); - --- IAM permissions boundaries. -CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( - account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, - principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), - principal_name TEXT NOT NULL, - policy_document TEXT NOT NULL, - PRIMARY KEY (account_id, principal_type, principal_name) -); - --- Idempotency tokens for TransactWriteItems. -CREATE TABLE IF NOT EXISTS idempotency_tokens ( - account_id TEXT NOT NULL, - token TEXT NOT NULL, - fingerprint TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - PRIMARY KEY (account_id, token) -); - -CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created ON idempotency_tokens (created_at); - --- Metrics (1-minute aggregation). ±9e999 store as ±Infinity in SQLite REAL, --- matching PostgreSQL's Infinity / -Infinity seed for min / max. -CREATE TABLE IF NOT EXISTS metrics ( - bucket TEXT NOT NULL, - metric TEXT NOT NULL, - table_name TEXT NOT NULL DEFAULT '', - index_name TEXT NOT NULL DEFAULT '', - operation TEXT NOT NULL DEFAULT '', - sum REAL NOT NULL DEFAULT 0, - count INTEGER NOT NULL DEFAULT 0, - min REAL NOT NULL DEFAULT 9e999, - max REAL NOT NULL DEFAULT -9e999, - PRIMARY KEY (bucket, metric, table_name, index_name, operation) -); - -CREATE INDEX IF NOT EXISTS idx_metrics_bucket ON metrics (bucket); - --- Login attempt tracking. -CREATE TABLE IF NOT EXISTS login_attempts ( - principal TEXT NOT NULL, - attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - success INTEGER NOT NULL, - source_ip TEXT -); - -CREATE INDEX IF NOT EXISTS idx_login_attempts_principal_time - ON login_attempts (principal, attempted_at DESC); - -CREATE INDEX IF NOT EXISTS idx_login_attempts_source_ip_time - ON login_attempts (source_ip, attempted_at DESC) - WHERE source_ip IS NOT NULL; - --- Backup metadata. -CREATE TABLE IF NOT EXISTS backups ( - backup_arn TEXT PRIMARY KEY, - backup_name TEXT NOT NULL, - table_id TEXT NOT NULL, - table_name TEXT NOT NULL, - account_id TEXT NOT NULL, - backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', - backup_type TEXT NOT NULL DEFAULT 'USER', - backup_size_bytes INTEGER NOT NULL DEFAULT 0, - item_count INTEGER NOT NULL DEFAULT 0, - key_schema TEXT NOT NULL, - attribute_definitions TEXT NOT NULL, - billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', - provisioned_throughput TEXT, - stream_specification TEXT, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - -CREATE INDEX IF NOT EXISTS idx_backups_table ON backups (account_id, table_name); - --- Backup items. `id` is the stable restore cursor: unlike an implicit rowid, --- an INTEGER PRIMARY KEY value is not renumbered by VACUUM. -CREATE TABLE IF NOT EXISTS backup_items ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, - pk TEXT NOT NULL, - sk TEXT, - item_data TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_backup_items_arn ON backup_items (backup_arn); - --- A backup's table definition beyond its keys: secondary indexes, billing --- mode and throughput, table class, encryption (catalog 0.0.4). JSON in the --- wire's shape behind a version marker; see --- `extenddb_storage::backup_definition`. A backup taken before 0.0.4 has no --- row and restores as before. -CREATE TABLE IF NOT EXISTS backup_definitions ( - backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, - definition TEXT NOT NULL -); - --- Which backup a restored table came from (catalog 0.0.4): reported by --- DescribeTable as RestoreSummary, and used to refuse DeleteBackup while the --- restore is running. Not a foreign key to backups, which may be deleted later. --- `restore_date_time` is the source backup's creation time, as the service --- reports it, supplied by the engine. `source_table_arn` is nullable only so --- that `migrate_table_restores` can add it to a database an earlier 0.0.4 --- build created; every row this build writes sets it. -CREATE TABLE IF NOT EXISTS table_restores ( - table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, - source_backup_arn TEXT NOT NULL, - source_table_arn TEXT, - restore_date_time TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_table_restores_backup ON table_restores (source_backup_arn); - --- Continuous backups / PITR status. -CREATE TABLE IF NOT EXISTS continuous_backups ( - account_id TEXT NOT NULL, - table_name TEXT NOT NULL, - pitr_enabled INTEGER NOT NULL DEFAULT 0, - earliest_restorable TEXT, - latest_restorable TEXT, - PRIMARY KEY (account_id, table_name) -); - --- Persistent queue for async GSI propagation. A row is inserted inside the --- base write transaction (zero crash window) and consumed by a background --- worker once `ready_at` has passed; survives process crash/restart. --- --- One row per async index: each row is self-describing — `index_context` --- carries the base key schema, attribute definitions, and the single target --- index definition captured at enqueue, so the worker applies with zero --- catalog reads. `worker_partition` is a stable hash of the base table key; --- all updates to a given base item share a partition and `ready_at` is kept --- monotonically non-decreasing within it, so the worker (which drains in `id` --- order) preserves per-key FIFO even with randomized propagation jitter. -CREATE TABLE IF NOT EXISTS gsi_pending ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - table_id TEXT NOT NULL, - worker_partition INTEGER NOT NULL, - old_item TEXT, - new_item TEXT, - index_context TEXT NOT NULL, - ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) -); - -CREATE INDEX IF NOT EXISTS idx_gsi_pending_claim - ON gsi_pending (worker_partition, ready_at, id); - --- Monotonic counters. Replaces PostgreSQL sequences. The stream counter is --- seeded to the current epoch in microseconds so sequence numbers are --- time-ordered and survive restarts (mirrors the PostgreSQL setval seed). -CREATE TABLE IF NOT EXISTS seq_counters ( - name TEXT PRIMARY KEY, - value INTEGER NOT NULL DEFAULT 0 -); - -INSERT OR IGNORE INTO seq_counters (name, value) - VALUES ('stream', CAST(strftime('%s','now') AS INTEGER) * 1000000); - --- Seed settings (mirror PostgreSQL defaults). --- Recorded catalog version. Upserts rather than INSERT OR IGNORE: this schema is --- re-applied by `extenddb migrate`, and with IGNORE the recorded version would --- never advance, so a migration could add objects and still leave the server --- refusing to start on a version mismatch. Must stay in step with --- `CATALOG_VERSION` above; they are checked against each other in a test. -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.4') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; -INSERT OR IGNORE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); -INSERT OR IGNORE INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10'); -INSERT OR IGNORE INTO settings (key, value) VALUES ('resource_tag_migration', 'applied'); -"#; - -/// Apply the full catalog schema to a fresh (or existing) database. -/// -/// Idempotent: every statement uses `IF NOT EXISTS` / `INSERT OR IGNORE`. -pub async fn apply(pool: &SqlitePool) -> OpResult<()> { - refuse_unwritable_catalog(pool).await?; - // Catalog 0.0.4 originally created backup_items without a stable cursor - // and table_restores without source_table_arn. Converge both before - // SCHEMA_SQL stamps 0.0.4, including databases that an earlier - // 0.0.4-in-progress binary already stamped. SQLite cannot add an INTEGER - // PRIMARY KEY with ALTER TABLE. - converge_table_shapes(pool).await?; - sqlx::raw_sql(SCHEMA_SQL) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("apply catalog schema: {e}")))?; - Ok(()) -} - -/// Refuse to write to a catalog whose recorded version is newer than this -/// binary's or cannot be parsed. -async fn refuse_unwritable_catalog(pool: &SqlitePool) -> OpResult<()> { - // The schema's last statement stamps the compiled version. Never stamp an - // older version onto a catalog a newer binary has already upgraded: that - // binary's startup gate would keep passing and this one's would start - // passing against tables it does not know. `extenddb migrate` refuses such - // a catalog before getting here; this guards every other caller. - if table_exists(pool, "settings").await? { - let stored: Option<(String,)> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_optional(pool) - .await - .map_err(|e| OpError::Internal(format!("read catalog_version: {e}")))?; - // Also refuses a stored version this binary cannot parse, rather than - // stamping over it (shared rule; see check_stored_catalog_version). - extenddb_storage::util::check_stored_catalog_version( - stored.as_ref().map(|(v,)| v.as_str()), - CATALOG_VERSION, - ) - .map_err(OpError::Internal)?; - } - Ok(()) -} - -/// Bring tables an earlier 0.0.4 build created to this build's shape. Each -/// step is a no-op when the table is absent or already converged. -async fn converge_table_shapes(pool: &SqlitePool) -> OpResult<()> { - migrate_backup_items(pool).await?; - migrate_table_restores(pool).await?; - Ok(()) -} - -/// Run only the table-shape steps of [`apply`], under the same version guard. -/// `extenddb migrate` reaches this through `run_data_migrations` when the -/// catalog is already stamped 0.0.4 but a table still has an earlier 0.0.4 -/// build's shape, which a version comparison alone cannot see. -pub async fn apply_shape_migrations(pool: &SqlitePool) -> OpResult<()> { - refuse_unwritable_catalog(pool).await?; - converge_table_shapes(pool).await -} - -/// The table-shape steps [`apply`] would run, one line each, without -/// writing anything. Empty when every table is in this build's shape (or -/// the catalog does not exist yet). -pub async fn pending_shape_migrations(pool: &SqlitePool) -> OpResult> { - let mut pending = Vec::new(); - if backup_items_needs_rebuild(pool).await? { - let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backup_items") - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("count backup_items: {e}")))?; - pending.push(format!("rebuild backup_items ({rows} rows)")); - } - if table_restores_lacks_source_table_arn(pool).await? { - pending.push("add table_restores.source_table_arn".to_owned()); - } - Ok(pending) -} - -async fn has_column(pool: &SqlitePool, table: &'static str, column: &str) -> OpResult { - sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pragma_table_info(?) WHERE name = ?)") - .bind(table) - .bind(column) - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("inspect {table} columns: {e}"))) -} - -async fn backup_items_needs_rebuild(pool: &SqlitePool) -> OpResult { - Ok( - table_exists(pool, "backup_items").await? - && !has_column(pool, "backup_items", "id").await?, - ) -} - -async fn table_restores_lacks_source_table_arn(pool: &SqlitePool) -> OpResult { - Ok(table_exists(pool, "table_restores").await? - && !has_column(pool, "table_restores", "source_table_arn").await?) -} - -/// Add `source_table_arn` to a `table_restores` an earlier 0.0.4 build -/// created. Rows it wrote keep NULL, which DescribeTable reports as an -/// absent `SourceTableArn`. -async fn migrate_table_restores(pool: &SqlitePool) -> OpResult<()> { - if !table_restores_lacks_source_table_arn(pool).await? { - return Ok(()); - } - tracing::info!("catalog upgrade: adding table_restores.source_table_arn"); - sqlx::query("ALTER TABLE table_restores ADD COLUMN source_table_arn TEXT") - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("add table_restores.source_table_arn: {e}")))?; - Ok(()) -} - -async fn migrate_backup_items(pool: &SqlitePool) -> OpResult<()> { - if !backup_items_needs_rebuild(pool).await? { - return Ok(()); - } - let started = std::time::Instant::now(); - - let mut tx = pool - .begin() - .await - .map_err(|e| OpError::Internal(format!("begin backup_items migration: {e}")))?; - // Counted inside the rebuild's transaction, so the logged count is the - // number of rows copied. - let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backup_items") - .fetch_one(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("count backup_items: {e}")))?; - tracing::info!( - "catalog upgrade: rebuilding backup_items with a stable restore cursor ({rows} rows); \ - the database is locked for writes until this finishes" - ); - sqlx::query("DROP TABLE IF EXISTS backup_items_new") - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("remove stale backup_items_new: {e}")))?; - sqlx::query( - "CREATE TABLE backup_items_new (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, \ - pk TEXT NOT NULL, sk TEXT, item_data TEXT NOT NULL)", - ) - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("create backup_items_new: {e}")))?; - sqlx::query( - "INSERT INTO backup_items_new (backup_arn, pk, sk, item_data) \ - SELECT backup_arn, pk, sk, item_data FROM backup_items ORDER BY rowid", - ) - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("copy backup_items: {e}")))?; - sqlx::query("DROP INDEX IF EXISTS idx_backup_items_arn") - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("drop old backup_items index: {e}")))?; - sqlx::query("DROP TABLE backup_items") - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("drop old backup_items: {e}")))?; - sqlx::query("ALTER TABLE backup_items_new RENAME TO backup_items") - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("rename backup_items_new: {e}")))?; - sqlx::query("CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn)") - .execute(&mut *tx) - .await - .map_err(|e| OpError::Internal(format!("recreate backup_items index: {e}")))?; - tx.commit() - .await - .map_err(|e| OpError::Internal(format!("commit backup_items migration: {e}")))?; - tracing::info!( - "catalog upgrade: rebuilt backup_items ({rows} rows) in {:.3}s", - started.elapsed().as_secs_f64() - ); - Ok(()) -} - /// Return whether a table with the given name exists in the catalog. pub async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { let exists: bool = sqlx::query_scalar( @@ -652,188 +25,6 @@ pub async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { Ok(exists) } -#[cfg(test)] -mod tests { - use super::{CATALOG_VERSION, SCHEMA_SQL}; - - /// The schema seeds the recorded catalog version as a SQL literal, and the - /// server compares that recorded value against `CATALOG_VERSION` at startup. - /// If the two drift, a freshly initialised deployment refuses to serve with a - /// version mismatch, which is a confusing failure a long way from its cause. - #[test] - fn the_seeded_catalog_version_matches_the_compiled_constant() { - let expected = format!( - "INSERT INTO settings (key, value) VALUES ('catalog_version', '{CATALOG_VERSION}')" - ); - assert!( - SCHEMA_SQL.contains(&expected), - "schema must seed catalog_version = {CATALOG_VERSION}; \ - update the literal in SCHEMA_SQL when bumping CATALOG_VERSION" - ); - } - - /// The seed must upsert. With `INSERT OR IGNORE` the recorded version never - /// advances, so `extenddb migrate` would add the new objects and still leave - /// the server refusing to start. - #[test] - fn the_catalog_version_seed_upserts_rather_than_ignoring() { - assert!( - !SCHEMA_SQL - .contains("INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version'"), - "catalog_version must not be seeded with INSERT OR IGNORE" - ); - } - - /// A fresh catalog seeds the resource-tag migration marker, so a - /// newly-initialised deployment serves without a separate migrate step. - /// An older catalog that lacks it is caught by the server's startup guard. - #[test] - fn the_schema_seeds_the_resource_tag_migration_marker() { - assert!( - SCHEMA_SQL.contains( - "INSERT OR IGNORE INTO settings (key, value) VALUES ('resource_tag_migration'" - ), - "schema must seed the resource_tag_migration marker on a fresh catalog" - ); - } - - #[tokio::test] - async fn applying_the_schema_records_the_marker_row() { - use sqlx::sqlite::SqlitePoolOptions; - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .unwrap(); - super::apply(&pool).await.unwrap(); - let value: Option<(String,)> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'resource_tag_migration'") - .fetch_optional(&pool) - .await - .unwrap(); - assert_eq!( - value.map(|(v,)| v).as_deref(), - Some(extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION_DONE) - ); - } -} - -#[cfg(test)] -mod apply_tests { - use super::{CATALOG_VERSION, apply}; - use sqlx::sqlite::SqlitePoolOptions; - - #[tokio::test] - async fn a_newer_catalog_is_not_stamped_down() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("pool"); - apply(&pool).await.expect("fresh apply"); - let newer = format!("{}.{}.{}", 0, 0, 999); - sqlx::query("UPDATE settings SET value = ? WHERE key = 'catalog_version'") - .bind(&newer) - .execute(&pool) - .await - .expect("stamp newer"); - let err = apply(&pool).await.expect_err("re-apply on a newer catalog"); - assert!( - format!("{err:?}").contains("newer than this binary"), - "{err:?}" - ); - let (stored,): (String,) = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_one(&pool) - .await - .expect("read"); - assert_eq!(stored, newer, "stored version must be untouched"); - assert!(stored != CATALOG_VERSION.to_string()); - } - - #[tokio::test] - async fn an_unparseable_catalog_version_is_not_overwritten() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("pool"); - apply(&pool).await.expect("fresh apply"); - sqlx::query("UPDATE settings SET value = 'garbage' WHERE key = 'catalog_version'") - .execute(&pool) - .await - .expect("stamp garbage"); - let err = apply(&pool) - .await - .expect_err("re-apply on an unreadable version"); - assert!( - format!("{err:?}").contains("refusing to overwrite"), - "{err:?}" - ); - let (stored,): (String,) = - sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") - .fetch_one(&pool) - .await - .expect("read"); - assert_eq!(stored, "garbage", "stored version must be untouched"); - } - - #[tokio::test] - async fn backup_items_migration_is_idempotent_and_preserves_row_order() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("pool"); - sqlx::raw_sql( - "CREATE TABLE backups (\ - backup_arn TEXT PRIMARY KEY, account_id TEXT NOT NULL, table_name TEXT NOT NULL); \ - CREATE TABLE backup_items (\ - backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, \ - pk TEXT NOT NULL, sk TEXT, item_data TEXT NOT NULL); \ - CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); \ - INSERT INTO backups (backup_arn, account_id, table_name) VALUES ('a', '0', 't'); \ - INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES \ - ('a', 'z', NULL, 'first'), \ - ('a', 'a', NULL, 'second'), \ - ('a', 'm', NULL, 'third');", - ) - .execute(&pool) - .await - .expect("old schema"); - - apply(&pool).await.expect("first migration"); - let rows: Vec<(i64, String, String)> = - sqlx::query_as("SELECT id, pk, item_data FROM backup_items ORDER BY id") - .fetch_all(&pool) - .await - .expect("migrated rows"); - assert_eq!( - rows, - vec![ - (1, "z".to_owned(), "first".to_owned()), - (2, "a".to_owned(), "second".to_owned()), - (3, "m".to_owned(), "third".to_owned()), - ] - ); - - apply(&pool).await.expect("second migration"); - let after: Vec<(i64, String, String)> = - sqlx::query_as("SELECT id, pk, item_data FROM backup_items ORDER BY id") - .fetch_all(&pool) - .await - .expect("rows after second apply"); - assert_eq!(after, rows); - let primary_key: i64 = sqlx::query_scalar( - "SELECT pk FROM pragma_table_info('backup_items') WHERE name = 'id'", - ) - .fetch_one(&pool) - .await - .expect("id column"); - assert_eq!(primary_key, 1); - } -} - /// Collect the `tracing` events emitted on the current thread, for tests /// that assert an operator-facing log line. Each entry is `" "`. /// @@ -911,94 +102,3 @@ pub(crate) mod log_capture { (Guard, lines) } } - -#[cfg(test)] -mod shape_tests { - use super::{apply, pending_shape_migrations}; - use sqlx::sqlite::SqlitePoolOptions; - - /// A database an earlier 0.0.4 build created has `table_restores` without - /// `source_table_arn`, already stamped 0.0.4. The dry run names the - /// column, apply adds it once and keeps the row, and a second apply is a - /// no-op. - #[tokio::test] - async fn an_earlier_0_0_4_table_restores_gains_source_table_arn() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("pool"); - apply(&pool).await.expect("fresh apply"); - assert!( - pending_shape_migrations(&pool) - .await - .expect("pending") - .is_empty() - ); - sqlx::raw_sql( - "PRAGMA foreign_keys = OFF; \ - DROP TABLE table_restores; \ - CREATE TABLE table_restores (\ - table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ - source_backup_arn TEXT NOT NULL, \ - restore_date_time TEXT NOT NULL \ - DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))); \ - INSERT INTO table_restores (table_id, source_backup_arn) VALUES ('t1', 'arn:b');", - ) - .execute(&pool) - .await - .expect("earlier 0.0.4 shape"); - assert_eq!( - pending_shape_migrations(&pool).await.expect("pending"), - vec!["add table_restores.source_table_arn".to_owned()] - ); - - apply(&pool).await.expect("converge"); - let row: (String, Option) = sqlx::query_as( - "SELECT source_backup_arn, source_table_arn FROM table_restores WHERE table_id = 't1'", - ) - .fetch_one(&pool) - .await - .expect("row kept"); - assert_eq!(row, ("arn:b".to_owned(), None)); - assert!( - pending_shape_migrations(&pool) - .await - .expect("pending") - .is_empty() - ); - - apply(&pool).await.expect("second apply is a no-op"); - let columns: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM pragma_table_info('table_restores') \ - WHERE name = 'source_table_arn'", - ) - .fetch_one(&pool) - .await - .expect("columns"); - assert_eq!(columns, 1); - } - - /// The shape steps `migrate` runs through `run_data_migrations` carry the - /// same version guard as `apply`. - #[tokio::test] - async fn shape_migrations_refuse_an_unreadable_catalog() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("pool"); - apply(&pool).await.expect("fresh apply"); - sqlx::query("UPDATE settings SET value = 'garbage' WHERE key = 'catalog_version'") - .execute(&pool) - .await - .expect("stamp garbage"); - let err = super::apply_shape_migrations(&pool) - .await - .expect_err("refused"); - assert!( - format!("{err:?}").contains("refusing to overwrite"), - "{err:?}" - ); - } -} diff --git a/crates/storage-sqlite/src/serve_lock.rs b/crates/storage-sqlite/src/serve_lock.rs index 0dcdf9506..07baee407 100644 --- a/crates/storage-sqlite/src/serve_lock.rs +++ b/crates/storage-sqlite/src/serve_lock.rs @@ -12,9 +12,11 @@ //! the process, and refuses to start if another process holds it. //! //! The commands that replace or rewrite the file take the same lock for their -//! duration: `init` and `migrate` (through the bootstrapper's migration lock) -//! and `destroy` (before it unlinks the file). Read-only commands (`settings`, -//! `manage`, `verify`, `status`, ...) do not, and run alongside a server. +//! duration: `init` and `migrate --yes` (through the migration runner's +//! `migrations::lock_for_migration`, which retries until its timeout) and +//! `destroy` (before it unlinks the file). Read-only commands (`settings`, +//! `manage`, `verify`, `status`, the `migrate` preview, ...) do not, and run +//! alongside a server. //! //! The lock file is named after the file SQLite actually opens, not after the //! configured string: the location is parsed exactly as sqlx parses it @@ -99,6 +101,15 @@ pub(crate) fn database_file(location: &str) -> Result, String> { Ok(Some(PathBuf::from(path))) } +/// Why [`ServeLock::try_acquire`] did not take the lock. +#[derive(Debug)] +pub(crate) enum LockFailure { + /// Another process holds it. + Held(String), + /// The lock file could not be opened or locked. + Failed(String), +} + /// Holds the lock until dropped. #[derive(Debug)] pub(crate) struct ServeLock { @@ -145,6 +156,14 @@ impl ServeLock { /// lock file cannot be opened (for example, the database's directory is /// not writable; the lock file lives next to the database). pub(crate) fn acquire(db_path: &Path) -> Result { + Self::try_acquire(db_path).map_err(|failure| match failure { + LockFailure::Held(m) | LockFailure::Failed(m) => m, + }) + } + + /// As [`ServeLock::acquire`], telling a lock another process holds from + /// any other failure. + pub(crate) fn try_acquire(db_path: &Path) -> Result { let path = Self::lock_path(db_path); let file = open_options_0600() .read(true) @@ -153,11 +172,11 @@ impl ServeLock { .truncate(false) .open(&path) .map_err(|e| { - format!( + LockFailure::Failed(format!( "cannot open lock file {}: {e} (the directory holding a SQLite database \ must be writable so the server can create its lock file)", path.display() - ) + )) })?; // The database and its sidecars are 0600 so other local users cannot // read secrets out of them. The lock file must match: flock needs @@ -165,17 +184,17 @@ impl ServeLock { // user hold the lock and keep the server from starting. `create` // honours the umask, and the file may predate this rule, so set the // mode explicitly on every open. - restrict_to_owner(&file, &path)?; + restrict_to_owner(&file, &path).map_err(LockFailure::Failed)?; try_lock_exclusive(&file).map_err(|e| { if e.kind() == std::io::ErrorKind::WouldBlock { - format!( + LockFailure::Held(format!( "another extenddb process is already using {} (lock held on {}); \ stop it first, or point this command at a different database", db_path.display(), path.display() - ) + )) } else { - format!("cannot lock {}: {e}", path.display()) + LockFailure::Failed(format!("cannot lock {}: {e}", path.display())) } })?; Ok(Self { _file: file, path }) @@ -195,6 +214,15 @@ impl ServeLock { } } + /// As [`ServeLock::acquire_for_location`], telling a lock another + /// process holds from any other failure. + pub(crate) fn try_acquire_for_location(location: &str) -> Result, LockFailure> { + match database_file(location).map_err(LockFailure::Failed)? { + Some(path) => Self::try_acquire(&path).map(Some), + None => Ok(None), + } + } + pub(crate) fn path(&self) -> &Path { &self.path } diff --git a/crates/storage-sqlite/src/snapshot_engine.rs b/crates/storage-sqlite/src/snapshot_engine.rs index f67679874..e8f12a53e 100644 --- a/crates/storage-sqlite/src/snapshot_engine.rs +++ b/crates/storage-sqlite/src/snapshot_engine.rs @@ -448,9 +448,10 @@ mod tests { }; async fn prepare(engine: &SqliteEngine) { - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; + // Migration 1 seeds no setting, so the rows are written. for key in ["control_plane_delay_seconds", "index_propagation_delay_ms"] { - sqlx::query("UPDATE settings SET value = '0' WHERE key = ?") + sqlx::query("INSERT OR REPLACE INTO settings (key, value) VALUES (?, '0')") .bind(key) .execute(&engine.pool) .await diff --git a/crates/storage-sqlite/src/sqlite_util.rs b/crates/storage-sqlite/src/sqlite_util.rs index c37ce480f..bcededfd7 100644 --- a/crates/storage-sqlite/src/sqlite_util.rs +++ b/crates/storage-sqlite/src/sqlite_util.rs @@ -1,12 +1,35 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Shared helpers for the SQLite backend: connection-URL building, RFC 3339 -//! timestamp conversion, and constraint-violation classification. +//! Shared helpers for the SQLite backend: connection-URL building, the +//! connection pragmas, RFC 3339 timestamp conversion, and constraint-violation +//! classification. use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; +/// The pragmas every connection of every SQLite pool runs with. `wal` sets +/// the journal mode, which is a property of the file; a connection that only +/// reads the catalog status leaves it as it is. +/// +/// # Errors +/// +/// A pragma failed. +pub(crate) async fn set_connection_pragmas( + conn: &mut sqlx::SqliteConnection, + wal: bool, +) -> Result<(), sqlx::Error> { + use sqlx::Executor; + if wal { + conn.execute("PRAGMA journal_mode = WAL").await?; + } + conn.execute("PRAGMA foreign_keys = ON").await?; + conn.execute("PRAGMA synchronous = NORMAL").await?; + conn.execute("PRAGMA busy_timeout = 5000").await?; + conn.execute("PRAGMA cache_size = -32000").await?; + Ok(()) +} + /// Normalize a configured path or connection string into a sqlx-openable /// SQLite URL. /// diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 075d25b28..4c61482c0 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -47,7 +47,6 @@ use sqlx::sqlite::SqlitePoolOptions; use tokio::sync::Mutex; use crate::INDEX_PROPAGATION_DELAY_QUERY; -use crate::schema::CATALOG_VERSION; use crate::sqlite_util::sqlite_url; /// The SQLite storage engine. Implements `StorageEngine` (all data/metadata/ @@ -129,15 +128,9 @@ impl SqliteEngine { } let pool = opts .after_connect(|conn, _| { - Box::pin(async move { - use sqlx::Executor; - conn.execute("PRAGMA journal_mode = WAL").await?; - conn.execute("PRAGMA foreign_keys = ON").await?; - conn.execute("PRAGMA synchronous = NORMAL").await?; - conn.execute("PRAGMA busy_timeout = 5000").await?; - conn.execute("PRAGMA cache_size = -32000").await?; - Ok(()) - }) + Box::pin( + async move { crate::sqlite_util::set_connection_pragmas(conn, true).await }, + ) }) .connect(&url) .await @@ -192,15 +185,17 @@ impl SqliteEngine { Arc::clone(&self.control_plane_notify) } - /// Bootstrap an ephemeral catalog on the engine's own (shared) connection, - /// at serve time. Used for in-memory databases, whose state does not - /// survive the `init` process, so schema, encryption key, default account, - /// and admin user must be created when the server starts. Idempotent. + /// Write the rows a server needs that `init` writes on a production + /// deployment (encryption key, default account, admin user) on the + /// engine's own connection, each only when absent. Used for an in-memory + /// database, whose state does not survive the `init` process, and by dev + /// builds, which need no `init`. The catalog schema is already in place: + /// the server's factory runs the migrations first. /// /// Returns `Some(password)` when a new admin user was created with a /// generated password (so the caller can surface it once), or `None` when /// the admin already existed or the password came from the environment. - pub(crate) async fn bootstrap_ephemeral( + pub(crate) async fn bootstrap_dev_rows( &self, admin_user: Option<&str>, admin_password: Option<&str>, @@ -209,12 +204,6 @@ impl SqliteEngine { let internal = |e: String| StorageError::Internal(e); - // Schema (creates settings, accounts, admin_users, … and seeds - // catalog_version + index_propagation_delay_ms). - crate::schema::apply(&self.pool) - .await - .map_err(|e| internal(format!("apply schema: {e:?}")))?; - // Encryption key. let key_exists: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM settings WHERE key = 'encryption_key')", @@ -406,58 +395,6 @@ impl SqliteEngine { Ok(()) } - /// Validate the on-disk catalog version matches the compiled expectation. - /// - /// # Errors - /// - [`StorageError::CatalogNotInitialized`] if the catalog is absent. - /// - [`StorageError::CatalogVersionMismatch`] if versions differ. - /// - [`StorageError::Internal`] if the stored version is malformed, carrying - /// `check_stored_catalog_version`'s refusal text. - pub async fn check_catalog_version(&self) -> Result<(), StorageError> { - let exists: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'settings')", - ) - .fetch_one(&self.pool) - .await - .map_err(|e| StorageError::Connection(e.to_string()))?; - - if !exists { - return Err(StorageError::CatalogNotInitialized); - } - - let found_str = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'catalog_version'", - ) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Connection(e.to_string()))? - .ok_or(StorageError::CatalogNotInitialized)? - .0; - - // An unreadable version gets the shared guard's refusal, the text - // `extenddb migrate` prints for the same catalog. - let found = found_str - .parse::() - .map_err(|e| { - StorageError::Internal( - extenddb_storage::util::check_stored_catalog_version( - Some(&found_str), - CATALOG_VERSION, - ) - .err() - .unwrap_or_else(|| e.to_string()), - ) - })?; - - if found != CATALOG_VERSION { - return Err(StorageError::CatalogVersionMismatch { - expected: CATALOG_VERSION.to_string(), - found: found_str, - }); - } - Ok(()) - } - /// Read the configured data database name (the file path) for the startup /// banner. Returns `"(not configured)"` when unset. pub async fn data_database_info(&self) -> String { @@ -472,43 +409,6 @@ impl SqliteEngine { } } -#[cfg(test)] -mod catalog_version_tests { - use super::SqliteEngine; - use extenddb_storage::error::StorageError; - - /// `serve` once printed the unreadable-version refusal as - /// "Internal error: "; it now carries the shared guard's - /// text, which `extenddb migrate` prints for the same catalog. - #[tokio::test] - async fn an_unreadable_version_carries_the_shared_refusal() { - let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) - .await - .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); - engine - .check_catalog_version() - .await - .expect("current catalog"); - for stored in ["garbage", "0.0", "v0.0.4"] { - sqlx::query("UPDATE settings SET value = ? WHERE key = 'catalog_version'") - .bind(stored) - .execute(&engine.pool) - .await - .expect("tamper"); - let expected = extenddb_storage::util::check_stored_catalog_version( - Some(stored), - crate::schema::CATALOG_VERSION, - ) - .expect_err("unreadable"); - match engine.check_catalog_version().await { - Err(StorageError::Internal(msg)) => assert_eq!(msg, expected, "{stored}"), - other => panic!("{stored}: {other:?}"), - } - } - } -} - #[cfg(test)] mod d1_write_lock_tests { use super::SqliteEngine; @@ -524,7 +424,7 @@ mod d1_write_lock_tests { let engine = SqliteEngine::new(":memory:", 2, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query( "INSERT INTO accounts (account_id, account_name) VALUES ('000000000000', 'default')", ) diff --git a/crates/storage-sqlite/src/stream_tests.rs b/crates/storage-sqlite/src/stream_tests.rs index 8ba92a55c..c474318e8 100644 --- a/crates/storage-sqlite/src/stream_tests.rs +++ b/crates/storage-sqlite/src/stream_tests.rs @@ -18,7 +18,7 @@ async fn engine() -> SqliteEngine { let engine = SqliteEngine::new(":memory:", 1, REGION, 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; for account in [ACCOUNT, OTHER_ACCOUNT] { sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, ?)") .bind(account) diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 8adff6173..d1e0e5769 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -1303,8 +1303,8 @@ mod gsi_throughput_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value") .execute(&engine.pool) .await .expect("delay"); @@ -1427,7 +1427,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") @@ -1513,7 +1513,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) @@ -1613,7 +1613,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) @@ -1730,7 +1730,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") @@ -1848,7 +1848,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind("000000000000") .execute(&engine.pool) @@ -1974,7 +1974,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind("000000000000") .execute(&engine.pool) @@ -2061,7 +2061,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind("000000000000") .execute(&engine.pool) @@ -2145,7 +2145,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) @@ -2246,7 +2246,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) @@ -2397,7 +2397,7 @@ mod reconciler_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) diff --git a/crates/storage-sqlite/src/vector_bench.rs b/crates/storage-sqlite/src/vector_bench.rs index e9a6f9167..f6f80dadc 100644 --- a/crates/storage-sqlite/src/vector_bench.rs +++ b/crates/storage-sqlite/src/vector_bench.rs @@ -84,7 +84,7 @@ mod tests { let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 4_096_000) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; let account = "000000000000"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind(account) diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index 05b30043c..a518457f1 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -49,7 +49,7 @@ pub(crate) async fn poll_control_plane_transitions( .flatten() .and_then(|v| v.parse::().ok()) .filter(|&v| v >= 0.0) - .unwrap_or(0.25); + .unwrap_or(crate::DEFAULT_CONTROL_PLANE_DELAY_SECONDS); let deadline = tokio::time::Instant::now() + Duration::from_secs_f64(delay + MARGIN_SECS); loop { match storage.process_control_plane_transitions().await { @@ -680,7 +680,7 @@ mod poison_row_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; // A row whose context parses but whose target index is gone is // applied-as-skip (Ok), so on the SECOND pass it drains cleanly. @@ -738,7 +738,7 @@ mod poison_row_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; // Poison row: malformed index_context. ready_at defaults to now (due). sqlx::query( @@ -780,7 +780,7 @@ mod poison_row_tests { let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; // Poison row (bad context) + a second poison row: both must drain. sqlx::query( @@ -841,7 +841,7 @@ mod vector_propagation_tests { let engine = SqliteEngine::new(db, 1, "us-east-1", 409_600) .await .expect("engine"); - crate::schema::apply(&engine.pool).await.expect("schema"); + crate::migrations::apply_for_tests(&engine.pool).await; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") .bind("000000000000") .execute(&engine.pool) diff --git a/crates/storage-sqlite/tests/fixtures/golden/0002_resource_tag_condition_keys.txt b/crates/storage-sqlite/tests/fixtures/golden/0002_resource_tag_condition_keys.txt new file mode 100644 index 000000000..eb7ee394d --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0002_resource_tag_condition_keys.txt @@ -0,0 +1,17 @@ +-- SELECT account_id, principal_type, principal_name, policy_name, policy_document FROM iam_policies ORDER BY principal_type, policy_name +946175009945 group devs tagged-group {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +946175009945 role app tagged-role {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +946175009945 user dev SelfServicePolicy {"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::946175009945:user/dev"}],"Version":"2012-10-17"} +946175009945 user dev tagged-user {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +-- SELECT account_id, principal_type, principal_name, policy_document FROM iam_permissions_boundaries ORDER BY principal_type +946175009945 role app {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +946175009945 user dev {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +-- SELECT session_token, role_name, session_policy FROM iam_sessions ORDER BY session_token +token-1 app {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +-- SELECT notes FROM extenddb_migrations WHERE id = 2 +rewrote account 946175009945 user/dev policy tagged-user (1 key(s)) +rewrote account 946175009945 user/dev policy (1 key(s)) +rewrote account 946175009945 group/devs policy tagged-group (1 key(s)) +rewrote account 946175009945 role/app policy tagged-role (1 key(s)) +rewrote account 946175009945 role/app policy (1 key(s)) +rewrote account 946175009945 assumed-role/app/session policy (1 key(s)) diff --git a/crates/storage-sqlite/tests/fixtures/golden/0003_sort_key_definitions.txt b/crates/storage-sqlite/tests/fixtures/golden/0003_sort_key_definitions.txt new file mode 100644 index 000000000..338ddea93 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0003_sort_key_definitions.txt @@ -0,0 +1,12 @@ +-- SELECT table_name, attribute_definitions FROM tables ORDER BY table_name +cu_orders [{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"}] +-- SELECT backup_arn, attribute_definitions FROM backups ORDER BY backup_arn +arn:b1 [{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"}] +arn:b2 [{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}] +arn:b3 [{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"}] +-- SELECT notes FROM extenddb_migrations WHERE id = 3 +restored cu_orders: sk (S) +cu_orders: partition key definition(s) [pk] are absent; reads and writes are unaffected, but index key type validation cannot check them +restored backup arn:b1 of cu_orders: sk (S) +restored backup arn:b3 of cu_orders: sk (S) +backup arn:b2 of cu_orders: sort key(s) [sk] have no attribute definition, and neither its items nor its source table give the type; restoring it fails until the definition is added diff --git a/crates/storage-sqlite/tests/fixtures/golden/0004_index_table_base_key.txt b/crates/storage-sqlite/tests/fixtures/golden/0004_index_table_base_key.txt new file mode 100644 index 000000000..2d73d67ed --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0004_index_table_base_key.txt @@ -0,0 +1,8 @@ +-- SELECT name, sql FROM sqlite_master WHERE name LIKE 'idx_base_key_%' ORDER BY name +idx_base_key_457d180b-237e-4c52-b4f4-45c14c6373fa CREATE INDEX "idx_base_key_457d180b-237e-4c52-b4f4-45c14c6373fa" ON "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" (base_pk, base_sk_s) +idx_base_key_fc40b212-8c5a-499b-9bb0-44bebe422a32 CREATE INDEX "idx_base_key_fc40b212-8c5a-499b-9bb0-44bebe422a32" ON "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" (base_pk, base_sk_s) +-- SELECT notes FROM extenddb_migrations WHERE id = 4 +cu_orders index by_gpk: created idx_base_key_fc40b212-8c5a-499b-9bb0-44bebe422a32 on (base_pk, base_sk_s) +cu_orders index by_lsk: created idx_base_key_457d180b-237e-4c52-b4f4-45c14c6373fa on (base_pk, base_sk_s) +cu_orders index gone: skipped, its index table _ddb_6f9619ff-8b86-4011-b42d-00c04fc964ff does not exist +going index by_gpk: skipped, the table is being deleted diff --git a/crates/storage-sqlite/tests/fixtures/released/README.md b/crates/storage-sqlite/tests/fixtures/released/README.md new file mode 100644 index 000000000..8dd7f934f --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/README.md @@ -0,0 +1,56 @@ +# Released SQLite catalogs + +Each file is a text dump of a database that a released ExtendDB binary +created, taken with Python's `sqlite3` `iterdump` (the statements the +`sqlite3` shell's `.dump` prints). The tests execute a dump into a new file +and upgrade it. + +| File | Made by | +|---|---| +| `dev-image-0.1.7.sql`, `dev-image-0.1.8.sql`, `dev-image-0.1.10.sql` to `dev-image-0.1.14.sql` | `extenddb/extenddb-dev:` `serve --foreground` on an empty volume, then CreateTable `cu_orders` (pk S, sk S, GSI `by_gpk` on gpk S and gsk N, LSI `by_lsk`, stream NEW_AND_OLD_IMAGES), UpdateTimeToLive `expires` and 10 PutItem. The IAM user `dev` is the one the dev image creates | +| `dev-image-0.1.14-backup.sql` | The same on `extenddb/extenddb-dev:0.1.14`, then CreateBackup `cu_orders_fixture` of `cu_orders`, waited for until AVAILABLE: one `backups` row and 10 `backup_items` rows whose items have sort keys | +| `v0.1.3-to-v0.1.6.sql` | The v0.1.3 `SCHEMA_SQL`, identical from v0.1.3 to v0.1.6, executed on an empty file. Catalog only: no bootstrap rows and no tables | + +There is no 0.1.9 state: no image was published for it, and its SQLite, +storage, server and app crates are identical to v0.1.10's. The npm builds of +0.1.10 to 0.1.14 use the same features as the images and dump identically, so +they are not repeated here. + +The `encryption_key` row holds a fixed test key (32 zero bytes), not the key +the release generated. The access key is the public AWS example key, its +stored secret is encrypted under the key that was replaced, and the admin +password hash is of a throwaway password. No dump holds any other secret. + +Each dump starts with `PRAGMA foreign_keys=OFF`, as the shell's does, because +tables are dumped in name order and child rows come before their parents. + +## Regenerating a dev-image dump + +`regenerate.sh` runs the published image on a local port with an empty work +directory as its volume, seeds it with `seed.py` (boto3), stops it, and +writes the dump with `dump.py`, which replaces the encryption key on a copy +of the database. It needs Docker, `curl` and Python 3 with boto3. From this +directory: + +``` +./regenerate.sh 0.1.14 21500 /tmp/fixture-0.1.14 +./regenerate.sh 0.1.14 21501 /tmp/fixture-0.1.14-backup --backup +``` + +The first writes `dev-image-0.1.14.sql`, the second +`dev-image-0.1.14-backup.sql`. The work directory must be empty or absent and +is left in place for you to delete; the container (named +`extenddb-fixture-`, or `$CONTAINER`) is removed. Ids, account +numbers and timestamps are new on every run, so the golden files under +`../golden/` that a regenerated dump feeds are regenerated with +`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite migrations`. + +`v0.1.3-to-v0.1.6.sql` is not a server's output: execute the `SCHEMA_SQL` of +tag `v0.1.3` (`crates/storage-sqlite/src/schema.rs`) on an empty file and +dump it with `python3 dump.py v0.1.3-to-v0.1.6.sql`. + +The PostgreSQL fixtures (`crates/storage-postgres/tests/fixtures/released/`) +come from a separate script with another dataset and another key rule. Each +backend's released-catalogs test compares an upgraded state with a fresh +install of the same backend seeded the same way, so the two datasets do not +need to match. diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.10.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.10.sql new file mode 100644 index 000000000..92e4a8236 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.10.sql @@ -0,0 +1,414 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'B7F8EAEF5EA93340ABF380D667F59DADFFF39B35D8AC2EB7067662C652398C17A9D526612EDFBF57D404C48F88D53E8572C8C9264EE1B828E0DEF1581B58C9245B4138E9','562708335319','dev',1,'2026-10-29T00:41:01.507Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('562708335319','default','2026-10-29T00:41:01.152Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$LnOMt.kHcPzby5DPGUTx7OJWLFCcs9axfxeBpzFbEd2acSGUBUZXu','2026-10-29T00:41:01.399Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('562708335319','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::562708335319:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:01.401Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('562708335319','dev','arn:aws:iam::562708335319:user/dev',NULL,'2026-10-29T00:41:01.401Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('b8f7543a-0cd8-4279-9bd6-687fe768f68e','e1d4790f-aeb6-479b-a7e7-a90c2f08f12b','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('b8f7543a-0cd8-4279-9bd6-687fe768f68e','8e60e230-37e8-4b47-980b-a11185492b80','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',6221.0,1,6221.0,6221.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',2111.0,1,2111.0,2111.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',2616.0,3,564.0,1397.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',13.0,13,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',313.0,2,60.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',10395.0,5,497.0,6178.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',2.0,2,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',86084.0,10,666.0,36339.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',85385.0,13,211.0,36304.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',260.0,1,260.0,260.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',9.0,2,4.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',756.0,2,335.0,421.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234461000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','562708335319'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000001','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"8af07ba9-a781-4a12-b9c8-f6f735647739","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234461000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.000Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000002','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"c3cab428-3e1a-4bea-9f59-2662b2a4f33a","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234461000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.009Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000003','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"af2dc4a2-fa15-4eb3-8856-a3cc887c145f","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234461000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.011Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000004','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"ce9dcb20-0357-424a-bd20-6fa795d471d3","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234461000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.013Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000005','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"2f453bfc-d3fb-471b-aa5a-a8ac1eaf3928","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234461000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.015Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000006','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"18d5d709-c81a-4da0-ab87-5e3e5c1e3970","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234461000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.052Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000007','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"69229bc5-76e6-4f9c-b9b6-67f305ad69f6","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234461000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.054Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000008','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"13102bd0-739f-4891-b3c9-b09fcef438dd","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234461000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.056Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000009','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"05934337-fe59-4efe-9214-b2c72ea77363","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234461000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.057Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000010','b8f7543a-0cd8-4279-9bd6-687fe768f68e','Insert','{"eventID":"9a89730b-9afc-4eda-af20-f35490228250","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234463,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234461000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:03.059Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','b8f7543a-0cd8-4279-9bd6-687fe768f68e',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.975Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','b8f7543a-0cd8-4279-9bd6-687fe768f68e',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.975Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','b8f7543a-0cd8-4279-9bd6-687fe768f68e',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.975Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','b8f7543a-0cd8-4279-9bd6-687fe768f68e',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.975Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('562708335319','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:01.975251678Z',0,0,'arn:aws:dynamodb:us-east-1:562708335319:table/cu_orders','b8f7543a-0cd8-4279-9bd6-687fe768f68e','expires',0,NULL,'2026-10-29T00:41:01',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" ON "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" ON "_ddb_e1d4790f-aeb6-479b-a7e7-a90c2f08f12b" (base_pk, base_sk_s); +CREATE INDEX "idx_order_8e60e230-37e8-4b47-980b-a11185492b80" ON "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_8e60e230-37e8-4b47-980b-a11185492b80" ON "_ddb_8e60e230-37e8-4b47-980b-a11185492b80" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_b8f7543a-0cd8-4279-9bd6-687fe768f68e" ON "_ddb_b8f7543a-0cd8-4279-9bd6-687fe768f68e" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.11.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.11.sql new file mode 100644 index 000000000..26eb4fc86 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.11.sql @@ -0,0 +1,414 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'2BA0A3F45162242D7AAACFFB3DB318013D512442BBBBA546C6D701C95E694A36833A29279B8F467F00174A24DC87AD94657AE95F10DF8206EF9E192A4A3B2F3EBD237110','625913564445','dev',1,'2026-10-29T00:41:00.986Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('625913564445','default','2026-10-29T00:41:00.729Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$r.BSHpF06s2YLif58qgjE.n5ohhVcSsmi9IhR1sbR3wVnxluChMQq','2026-10-29T00:41:00.962Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('625913564445','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::625913564445:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:00.975Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('625913564445','dev','arn:aws:iam::625913564445:user/dev',NULL,'2026-10-29T00:41:00.975Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('4cd132d3-06c8-4f3a-acf4-89134f1d084a','a22a993f-2532-43ea-b925-ebbd7026b1e7','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('4cd132d3-06c8-4f3a-acf4-89134f1d084a','008befca-5092-43d5-999d-fe20e2c2d5d6','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1593.0,3,339.0,697.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',14.0,14,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',10041.0,5,279.0,6846.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',16807.0,10,709.0,9489.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',16736.0,14,173.0,8465.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',1194.0,3,291.0,517.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',6874.0,1,6874.0,6874.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',464.0,3,60.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',216.0,1,216.0,216.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',2112.0,1,2112.0,2112.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',12.0,3,3.0,5.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234460000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','625913564445'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000001','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"a3dedb88-e05a-447b-84d6-2834f4633c11","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234460000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.515Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000002','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"59b5cae1-7867-4a59-bfff-6be3d083a3e0","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234460000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.525Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000003','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"126a94cc-1fe1-4cf2-be12-996f693b9495","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234460000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.527Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000004','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"604758bd-c420-4e29-8765-db715bc03cfb","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234460000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.529Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000005','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"d61b6d55-e764-406e-9bc1-6f753d8bdada","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234460000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.530Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000006','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"5b1f79c4-bf6a-4f3b-b443-276ed3d8d705","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234460000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.533Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000007','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"4c8f8541-bb8f-4f76-a401-8a7686ffde99","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234460000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.535Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000008','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"2eeb7d9d-6752-4290-a02c-cebceed99392","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234460000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.537Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000009','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"440c5a1d-dc0f-4b0f-a5e3-2dd9238fb1b4","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234460000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.539Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000010','4cd132d3-06c8-4f3a-acf4-89134f1d084a','Insert','{"eventID":"eac9140f-dea5-4a43-bc73-fa4d3be11b4d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234460000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.541Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','4cd132d3-06c8-4f3a-acf4-89134f1d084a',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.493Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','4cd132d3-06c8-4f3a-acf4-89134f1d084a',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.493Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','4cd132d3-06c8-4f3a-acf4-89134f1d084a',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.493Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','4cd132d3-06c8-4f3a-acf4-89134f1d084a',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.493Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('625913564445','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:01.49275667Z',0,0,'arn:aws:dynamodb:us-east-1:625913564445:table/cu_orders','4cd132d3-06c8-4f3a-acf4-89134f1d084a','expires',0,NULL,'2026-10-29T00:41:01',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_a22a993f-2532-43ea-b925-ebbd7026b1e7" ON "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_a22a993f-2532-43ea-b925-ebbd7026b1e7" ON "_ddb_a22a993f-2532-43ea-b925-ebbd7026b1e7" (base_pk, base_sk_s); +CREATE INDEX "idx_order_008befca-5092-43d5-999d-fe20e2c2d5d6" ON "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_008befca-5092-43d5-999d-fe20e2c2d5d6" ON "_ddb_008befca-5092-43d5-999d-fe20e2c2d5d6" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_4cd132d3-06c8-4f3a-acf4-89134f1d084a" ON "_ddb_4cd132d3-06c8-4f3a-acf4-89134f1d084a" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.12.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.12.sql new file mode 100644 index 000000000..ba6e7e2b3 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.12.sql @@ -0,0 +1,414 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'1C620AED22CC47AC273750A5F9FAC9EFF9CA41FC4849C99697B4108D91A092013B8EC0B677289D462C0DF526A88334A4196340805493980350A020A19C97225835932100','253835544748','dev',1,'2026-10-29T00:41:00.286Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('253835544748','default','2026-10-29T00:41:00.036Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$1OPE1c511MBkPuSVZ0lnHuOK8bnGIeF49wonWMvrSem/rExJ0fgBi','2026-10-29T00:41:00.274Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('253835544748','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::253835544748:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:00.277Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('253835544748','dev','arn:aws:iam::253835544748:user/dev',NULL,'2026-10-29T00:41:00.277Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('7d96e787-5930-4df7-b533-997e11c354b1','5754a525-d171-4bef-8ba7-de77cd4f561c','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('7d96e787-5930-4df7-b533-997e11c354b1','8975c67d-7bc2-4424-9b5e-f7c51d2c835c','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',11606.0,1,11606.0,11606.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',3744.0,1,3744.0,3744.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',1181.0,3,362.0,439.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',14.0,14,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',20384.0,10,718.0,9450.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',13.0,3,4.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',20251.0,14,195.0,9407.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',231.0,1,231.0,231.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',17328.0,5,300.0,11582.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',2527.0,3,367.0,1195.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',515.0,3,60.0,253.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234460000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','253835544748'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000001','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"c5c05397-16e9-4c64-9f88-0fb324f411f1","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234460000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.793Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000002','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"f11dc642-9b21-4a79-b410-22f3ec0c0311","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234460000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.796Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000003','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"cad90076-c1e1-4223-a51e-c93b57755883","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234460000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.798Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000004','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"a31fb7bf-c954-470d-90ea-da5f4ab668ce","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234460000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.801Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000005','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"40057916-90b0-43b7-a35f-1772b45800fc","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234460000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.805Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000006','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"c01a5bdb-5ab6-47a5-8753-6ae97b698399","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234460000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.808Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000007','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"4c51d631-1ef2-4c9e-be76-7a04e9733feb","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234460000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.819Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000008','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"744e8e9b-788d-4984-a247-9ff4c99a26cc","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234460000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.821Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000009','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"f6816c13-08a0-47e9-991d-24f2c0dc3b17","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234460000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.823Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000010','7d96e787-5930-4df7-b533-997e11c354b1','Insert','{"eventID":"e7f55a5e-2d2f-4cc6-bb79-28cb862ddee1","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234461,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234460000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:01.826Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','7d96e787-5930-4df7-b533-997e11c354b1',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:00.747Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','7d96e787-5930-4df7-b533-997e11c354b1',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:00.747Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','7d96e787-5930-4df7-b533-997e11c354b1',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:00.747Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','7d96e787-5930-4df7-b533-997e11c354b1',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:00.747Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('253835544748','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:00.74678012Z',0,0,'arn:aws:dynamodb:us-east-1:253835544748:table/cu_orders','7d96e787-5930-4df7-b533-997e11c354b1','expires',0,NULL,'2026-10-29T00:41:00',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_5754a525-d171-4bef-8ba7-de77cd4f561c" ON "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_5754a525-d171-4bef-8ba7-de77cd4f561c" ON "_ddb_5754a525-d171-4bef-8ba7-de77cd4f561c" (base_pk, base_sk_s); +CREATE INDEX "idx_order_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" ON "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" ON "_ddb_8975c67d-7bc2-4424-9b5e-f7c51d2c835c" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_7d96e787-5930-4df7-b533-997e11c354b1" ON "_ddb_7d96e787-5930-4df7-b533-997e11c354b1" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.13.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.13.sql new file mode 100644 index 000000000..1333a202c --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.13.sql @@ -0,0 +1,414 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'4369F3DC9B788ED20A44661EAB3CB2FB8910548C1E602AE8CCFC85F1828C4F0C25E30E6D2521B337802113F84CEFAB2FE5FCC9B8F453A5B7A7E76469283296273A73CD15','521807205336','dev',1,'2026-10-29T00:41:01.384Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('521807205336','default','2026-10-29T00:41:01.118Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$rTPfQHcAUnRr9Ib.b6aTF.AVBwS6Gs/xi77LoxxcljuNLszHZO2my','2026-10-29T00:41:01.359Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('521807205336','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::521807205336:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:01.368Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('521807205336','dev','arn:aws:iam::521807205336:user/dev',NULL,'2026-10-29T00:41:01.368Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('9926feb0-8043-4f41-91a6-80eeb4a77f31','efa242c1-bda6-4a98-97ca-de606b0e0777','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('9926feb0-8043-4f41-91a6-80eeb4a77f31','c03fba74-6548-45f5-aefc-87e790019d16','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',720.0,2,293.0,427.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',2.0,2,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',60787.0,13,166.0,30595.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',313.0,2,60.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',9.0,2,4.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',208.0,1,208.0,208.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',12381.0,5,278.0,5940.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',5247.0,1,5247.0,5247.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',61387.0,10,774.0,30635.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',5976.0,1,5976.0,5976.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',13.0,13,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1797.0,3,340.0,788.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234461000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','521807205336'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000001','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"02ea5a7a-bf0e-4ef8-9933-d23c7c363811","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234461000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.827Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000002','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"4e218371-20cd-4eff-8e17-ade600c24393","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234461000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.837Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000003','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"2169f015-e173-4f98-95a9-1b5dd468dbfd","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234461000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.839Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000004','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"4e6bc529-a0c5-4028-a67a-0d5124b129a0","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234461000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.841Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000005','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"80eee8ce-7912-4840-9942-b0f0aebcce5d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234461000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.858Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000006','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"647357fb-614e-41ff-ab77-048b22d16665","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234461000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.860Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000007','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"bab67d00-7ec4-46d0-9dfc-63d6d9522e6c","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234461000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.862Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234461000008','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"ebe16209-8aff-43f0-a023-5be15e2b2ed2","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234461000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.865Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000009','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"a1000a20-8a10-47db-a0ac-d86c6f243513","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234461000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.867Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234461000010','9926feb0-8043-4f41-91a6-80eeb4a77f31','Insert','{"eventID":"cc0bac46-e695-4765-9b58-af81e8dc6225","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234461000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.899Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','9926feb0-8043-4f41-91a6-80eeb4a77f31',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.800Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','9926feb0-8043-4f41-91a6-80eeb4a77f31',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.800Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','9926feb0-8043-4f41-91a6-80eeb4a77f31',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.800Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','9926feb0-8043-4f41-91a6-80eeb4a77f31',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.800Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('521807205336','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:01.800302707Z',0,0,'arn:aws:dynamodb:us-east-1:521807205336:table/cu_orders','9926feb0-8043-4f41-91a6-80eeb4a77f31','expires',0,NULL,'2026-10-29T00:41:01',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_efa242c1-bda6-4a98-97ca-de606b0e0777" ON "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_efa242c1-bda6-4a98-97ca-de606b0e0777" ON "_ddb_efa242c1-bda6-4a98-97ca-de606b0e0777" (base_pk, base_sk_s); +CREATE INDEX "idx_order_c03fba74-6548-45f5-aefc-87e790019d16" ON "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_c03fba74-6548-45f5-aefc-87e790019d16" ON "_ddb_c03fba74-6548-45f5-aefc-87e790019d16" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_9926feb0-8043-4f41-91a6-80eeb4a77f31" ON "_ddb_9926feb0-8043-4f41-91a6-80eeb4a77f31" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14-backup.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14-backup.sql new file mode 100644 index 000000000..51938d74d --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14-backup.sql @@ -0,0 +1,429 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +INSERT INTO "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +CREATE TABLE "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'DA6067B68A01FC0025FFABB7A1045DB8E038647980EEC1978C78F12D69F0EEC1C78A7D1D6671C218AA5DFC91512AD9AC3CDC2F21F4517AF5E3CA18F0D7ED2E44E82531BE','631577924105','dev',1,'2026-10-29T06:22:10.956Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('631577924105','default','2026-10-29T06:22:10.716Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$VIOJYME8Z53BwWEpZERaKeb7NmZdPq63VTmaJ1zHBFTt.bmFnq3Ni','2026-10-29T06:22:10.949Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "backup_items" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','',NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "backups" VALUES('arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders/backup/1793254932797-4116998c','cu_orders_fixture','26b47adf-4df9-438a-9517-4281822d2e70','cu_orders','631577924105','AVAILABLE','USER',0,10,'[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,NULL,'2026-10-29T06:22:12.797Z'); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('631577924105','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::631577924105:user/dev"}],"Version":"2012-10-17"}','2026-10-29T06:22:10.951Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('631577924105','dev','arn:aws:iam::631577924105:user/dev',NULL,'2026-10-29T06:22:10.951Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('26b47adf-4df9-438a-9517-4281822d2e70','98c725a3-a197-4e5f-97d3-d613001d9c8e','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('26b47adf-4df9-438a-9517-4281822d2e70','cf2a8986-57be-4c48-95b9-08d343dbc9f1','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','StorageQueryLatency','','Request','Data',8638.0,15,136.0,927.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','Query',733.0,2,312.0,421.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',925.0,1,925.0,925.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','Query',2.0,2,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ReturnedBytes','cu_orders','','Query',404.0,2,151.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','StorageQueryCount','','Request','Data',15.0,15,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',7513.0,10,601.0,1147.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','StorageQueryLatency','','Request','Catalog',3554.0,5,225.0,1853.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ReturnedItemCount','cu_orders','','Query',8.0,2,3.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','Scan',279.0,1,279.0,279.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1002.0,3,315.0,364.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','','','DescribeBackup',156.0,1,156.0,156.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','CreateBackup',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','DescribeBackup',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',2243.0,1,2243.0,2243.0); +INSERT INTO "metrics" VALUES('2026-10-29T06:22:00Z','SuccessfulRequestLatency','cu_orders','','CreateBackup',679.0,1,679.0,679.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793254930000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('resource_tag_migration','applied'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','631577924105'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000001','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"b43ccf84-5978-4c59-9016-1b5eaf9a9a98","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793254930000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.522Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000003','000001793254930000002','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"32e804c5-c786-4973-aace-f7fb32b4904c","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793254930000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.524Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000003','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"70bb70bd-e20e-434b-add6-3efae1754719","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793254930000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.526Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000004','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"a5c52a55-5947-4d07-9e8a-96f17ab05d6d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793254930000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.527Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000003','000001793254930000005','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"de36bf95-6800-48d1-8b89-badc9382d4bc","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793254930000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.529Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000006','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"3e49fc89-af46-4fc9-b7d9-785de1782696","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793254930000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.531Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000007','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"c910e6d4-bc4c-4b28-abb3-6d86210d14ac","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793254930000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.533Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000003','000001793254930000008','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"1e56467d-9db9-48e3-99ef-e2cb7ceea48d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793254930000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.535Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000009','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"df566a1c-72c5-44e6-bf70-5cbbcb70ebd4","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793254930000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.536Z'); +INSERT INTO "stream_records" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','000001793254930000010','26b47adf-4df9-438a-9517-4281822d2e70','Insert','{"eventID":"ffd6366c-df39-4bae-8bed-49746d5d452d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793254932,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793254930000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T06:22:12.538Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000000','26b47adf-4df9-438a-9517-4281822d2e70',NULL,'000000000000000000000',NULL,'2026-10-29T06:22:11.506Z'); +INSERT INTO "stream_shards" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000001','26b47adf-4df9-438a-9517-4281822d2e70',NULL,'000000000000000000000',NULL,'2026-10-29T06:22:11.506Z'); +INSERT INTO "stream_shards" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000002','26b47adf-4df9-438a-9517-4281822d2e70',NULL,'000000000000000000000',NULL,'2026-10-29T06:22:11.506Z'); +INSERT INTO "stream_shards" VALUES('shardId-26b47adf-4df9-438a-9517-4281822d2e70-0000000000000003','26b47adf-4df9-438a-9517-4281822d2e70',NULL,'000000000000000000000',NULL,'2026-10-29T06:22:11.506Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('631577924105','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T06:22:11.506269382Z',0,0,'arn:aws:dynamodb:us-east-1:631577924105:table/cu_orders','26b47adf-4df9-438a-9517-4281822d2e70','expires',0,NULL,'2026-10-29T06:22:11.506',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_98c725a3-a197-4e5f-97d3-d613001d9c8e" ON "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_98c725a3-a197-4e5f-97d3-d613001d9c8e" ON "_ddb_98c725a3-a197-4e5f-97d3-d613001d9c8e" (base_pk, base_sk_s); +CREATE INDEX "idx_order_cf2a8986-57be-4c48-95b9-08d343dbc9f1" ON "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_cf2a8986-57be-4c48-95b9-08d343dbc9f1" ON "_ddb_cf2a8986-57be-4c48-95b9-08d343dbc9f1" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_26b47adf-4df9-438a-9517-4281822d2e70" ON "_ddb_26b47adf-4df9-438a-9517-4281822d2e70" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql new file mode 100644 index 000000000..da402399b --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql @@ -0,0 +1,415 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_66965b15-a794-47d8-b104-08643cdbec37" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_66965b15-a794-47d8-b104-08643cdbec37" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'B3F9494FAF16573F87100E69ED9BEC240F408F1855997DD99A7E4DD00509289ABD89C3E98B900D143500DB4C999AF799C506B99EB5BD27B7543D09D9FE99921F8BA790C4','946175009945','dev',1,'2026-10-29T00:40:50.224Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('946175009945','default','2026-10-29T00:40:49.975Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$LruWm2kSSfkjAEBHBZnwV.EDzDNHNxvWPkUVzk7Ix6QMCLVilJqH.','2026-10-29T00:40:50.211Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('946175009945','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::946175009945:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:40:50.213Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('946175009945','dev','arn:aws:iam::946175009945:user/dev',NULL,'2026-10-29T00:40:50.213Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('66965b15-a794-47d8-b104-08643cdbec37','bf5aef51-08c6-481a-bdf6-7c56c59a6946','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('66965b15-a794-47d8-b104-08643cdbec37','0cdcd40b-7047-4341-a0ee-e24ded5711ed','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','Scan',269.0,1,269.0,269.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','StorageQueryLatency','','Request','Data',30030.0,14,222.0,7455.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',30724.0,10,1026.0,7510.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1520.0,3,456.0,552.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','Query',1276.0,3,333.0,548.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','Query',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',2138.0,1,2138.0,2138.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','StorageQueryCount','','Request','Data',14.0,14,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','StorageQueryLatency','','Request','Catalog',4227.0,5,366.0,1746.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',1257.0,1,1257.0,1257.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ReturnedItemCount','cu_orders','','Query',13.0,3,4.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:40:00Z','ReturnedBytes','cu_orders','','Query',515.0,3,60.0,253.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234449000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('resource_tag_migration','applied'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','946175009945'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000001','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"6c0b4834-eeb3-4266-b1df-2e84944f10b6","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234449000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.710Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000003','000001793234449000002','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"0a85ac12-b2a2-4454-b34a-975ce98a5bc9","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234449000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.717Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000003','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"0cdb96a1-10e7-4e9b-9092-352381d2089d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234449000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.721Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000004','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"eb5c7398-931f-47a1-a1a6-810c762b2548","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234449000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.724Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000003','000001793234449000005','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"fe897b4d-43ad-4e62-af15-40b781f270aa","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234449000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.732Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000006','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"7505663d-a88f-484f-b9d7-7bab2ab80f25","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234449000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.741Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000007','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"3ab363ac-3a5e-4174-9fb8-caaa45e05cdc","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234449000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.745Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000003','000001793234449000008','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"828bbeb3-04ee-4179-a4d8-bf178c86cdf7","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234449000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.748Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000009','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"ac005be0-8452-44a7-90b0-78a5da00eb4f","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234449000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.752Z'); +INSERT INTO "stream_records" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','000001793234449000010','66965b15-a794-47d8-b104-08643cdbec37','Insert','{"eventID":"a36a53e8-8a33-42c8-9c8a-644e896c9276","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234451,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234449000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:40:51.755Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000000','66965b15-a794-47d8-b104-08643cdbec37',NULL,'000000000000000000000',NULL,'2026-10-29T00:40:50.687Z'); +INSERT INTO "stream_shards" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000001','66965b15-a794-47d8-b104-08643cdbec37',NULL,'000000000000000000000',NULL,'2026-10-29T00:40:50.687Z'); +INSERT INTO "stream_shards" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000002','66965b15-a794-47d8-b104-08643cdbec37',NULL,'000000000000000000000',NULL,'2026-10-29T00:40:50.687Z'); +INSERT INTO "stream_shards" VALUES('shardId-66965b15-a794-47d8-b104-08643cdbec37-0000000000000003','66965b15-a794-47d8-b104-08643cdbec37',NULL,'000000000000000000000',NULL,'2026-10-29T00:40:50.687Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('946175009945','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:40:50.687411823Z',0,0,'arn:aws:dynamodb:us-east-1:946175009945:table/cu_orders','66965b15-a794-47d8-b104-08643cdbec37','expires',0,NULL,'2026-10-29T00:40:50.687',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_bf5aef51-08c6-481a-bdf6-7c56c59a6946" ON "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_bf5aef51-08c6-481a-bdf6-7c56c59a6946" ON "_ddb_bf5aef51-08c6-481a-bdf6-7c56c59a6946" (base_pk, base_sk_s); +CREATE INDEX "idx_order_0cdcd40b-7047-4341-a0ee-e24ded5711ed" ON "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_0cdcd40b-7047-4341-a0ee-e24ded5711ed" ON "_ddb_0cdcd40b-7047-4341-a0ee-e24ded5711ed" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_66965b15-a794-47d8-b104-08643cdbec37" ON "_ddb_66965b15-a794-47d8-b104-08643cdbec37" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.7.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.7.sql new file mode 100644 index 000000000..e5a00659b --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.7.sql @@ -0,0 +1,412 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'1866B25F86C0A4C57167553D604713BA06406D748459356A04ADF5AAE5CC4BBD0E86BB8E43A5F89DED75444B579E15DFB24BE8D9A71EE781F9A7342DB988D14C62AB9532','380366246314','dev',1,'2026-10-29T00:41:00.583Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('380366246314','default','2026-10-29T00:41:00.337Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$4siKVOubqEVeH1r0X6S5K.GemKWvtMzD5wsKxmnaDl/I4SvpYWAyy','2026-10-29T00:41:00.570Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('380366246314','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::380366246314:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:00.573Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('380366246314','dev','arn:aws:iam::380366246314:user/dev',NULL,'2026-10-29T00:41:00.573Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('25631d35-4f30-4da5-ae6a-4879d8eb79b6','fc40b212-8c5a-499b-9bb0-44bebe422a32','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('25631d35-4f30-4da5-ae6a-4879d8eb79b6','457d180b-237e-4c52-b4f4-45c14c6373fa','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',28626.0,14,197.0,21037.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',10971.0,5,255.0,8735.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1518.0,3,336.0,696.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',956.0,1,956.0,956.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',1110.0,3,278.0,498.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',12.0,3,3.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',237.0,1,237.0,237.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',28378.0,10,597.0,21064.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',464.0,3,60.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',14.0,14,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',9554.0,1,9554.0,9554.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234460000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','380366246314'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000001','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"8d4774c1-57b1-402c-929c-82f020d2243e","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234460000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.161Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000002','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"373ad8a0-19d9-4d13-848d-7e0b40456b20","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234460000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.163Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000003','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"78f0b401-612b-4dcd-a661-05b748e90c5a","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234460000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.165Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000004','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"b92c7ed5-c79e-4db3-9ca7-d200f6d3135c","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234460000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.167Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000005','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"014045e0-c4ea-402e-83c1-f80c382a228c","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234460000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.169Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000006','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"b465dc38-1097-451c-8c5c-7a292323079b","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234460000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.171Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000007','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"68510cca-bd99-49f6-b778-4d3dd03382be","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234460000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.193Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000008','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"81c6fae5-e4d5-427f-8d27-647ab9386f44","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234460000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.195Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000009','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"8ba3687a-c903-45a5-ace2-72c0c85c1b8f","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234460000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.197Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000010','25631d35-4f30-4da5-ae6a-4879d8eb79b6','Insert','{"eventID":"299c0428-4a36-4f3d-a168-0e6e11161d06","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234460000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.199Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','25631d35-4f30-4da5-ae6a-4879d8eb79b6',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.126Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','25631d35-4f30-4da5-ae6a-4879d8eb79b6',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.126Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','25631d35-4f30-4da5-ae6a-4879d8eb79b6',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.126Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','25631d35-4f30-4da5-ae6a-4879d8eb79b6',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.126Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('380366246314','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:01.125133157Z',0,0,'arn:aws:dynamodb:us-east-1:380366246314:table/cu_orders','25631d35-4f30-4da5-ae6a-4879d8eb79b6','expires',0,NULL,'2026-10-29T00:41:01',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_fc40b212-8c5a-499b-9bb0-44bebe422a32" ON "_ddb_fc40b212-8c5a-499b-9bb0-44bebe422a32" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_order_457d180b-237e-4c52-b4f4-45c14c6373fa" ON "_ddb_457d180b-237e-4c52-b4f4-45c14c6373fa" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_ttl_25631d35-4f30-4da5-ae6a-4879d8eb79b6" ON "_ddb_25631d35-4f30-4da5-ae6a-4879d8eb79b6" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.8.sql b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.8.sql new file mode 100644 index 000000000..c0137642d --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.8.sql @@ -0,0 +1,414 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p0','l09',NULL,NULL,'p0','s00',NULL,NULL,'{"lsk":{"S":"l09"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p1','l08',NULL,NULL,'p1','s01',NULL,NULL,'{"lsk":{"S":"l08"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p2','l07',NULL,NULL,'p2','s02',NULL,NULL,'{"lsk":{"S":"l07"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p0','l06',NULL,NULL,'p0','s03',NULL,NULL,'{"lsk":{"S":"l06"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p1','l05',NULL,NULL,'p1','s04',NULL,NULL,'{"lsk":{"S":"l05"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p2','l04',NULL,NULL,'p2','s05',NULL,NULL,'{"lsk":{"S":"l04"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p0','l03',NULL,NULL,'p0','s06',NULL,NULL,'{"lsk":{"S":"l03"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p1','l02',NULL,NULL,'p1','s07',NULL,NULL,'{"lsk":{"S":"l02"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p2','l01',NULL,NULL,'p2','s08',NULL,NULL,'{"lsk":{"S":"l01"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" VALUES('p0','l00',NULL,NULL,'p0','s09',NULL,NULL,'{"lsk":{"S":"l00"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + base_pk TEXT NOT NULL, + base_sk_s TEXT, + base_sk_n TEXT, + base_sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, base_pk, base_sk_s) +); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g0',NULL,'1',NULL,'p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g1',NULL,'21000011',NULL,'p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g0',NULL,'21000012',NULL,'p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g1',NULL,'21000013',NULL,'p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g0',NULL,'21000014',NULL,'p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g1',NULL,'21000015',NULL,'p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g0',NULL,'21000016',NULL,'p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g1',NULL,'21000017',NULL,'p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g0',NULL,'21000018',NULL,'p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" VALUES('g1',NULL,'21000019',NULL,'p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" ( + pk TEXT NOT NULL, + sk_s TEXT, + sk_n TEXT, + sk_b BLOB, + item_data TEXT NOT NULL, + PRIMARY KEY (pk, sk_s) +); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p0','s00',NULL,NULL,'{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p1','s01',NULL,NULL,'{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p2','s02',NULL,NULL,'{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p0','s03',NULL,NULL,'{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p1','s04',NULL,NULL,'{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p2','s05',NULL,NULL,'{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p0','s06',NULL,NULL,'{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p1','s07',NULL,NULL,'{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p2','s08',NULL,NULL,'{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}}'); +INSERT INTO "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" VALUES('p0','s09',NULL,NULL,'{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}}'); +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +INSERT INTO "access_keys" VALUES('AKIAIOSFODNN7EXAMPLE',X'E0595559CC9DCEC60E553ABCD6C3F6C6FE3749EC6238A1E3FC823C654F99F4E8D6F034A49958EA2FA4333CBD52B1E272153D92C20138D8EFC756D2942A81BEB753C66178','504105003262','dev',1,'2026-10-29T00:41:01.012Z'); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "accounts" VALUES('504105003262','default','2026-10-29T00:41:00.749Z'); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "admin_users" VALUES('admin','$2b$12$yTvblHKpHs.xWDXk0hSW3egUPxGVvxXQ5LH5U9.5edt7THVfQ3c8O','2026-10-29T00:41:00.991Z'); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +INSERT INTO "iam_policies" VALUES('504105003262','user','dev','SelfServicePolicy','{"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::504105003262:user/dev"}],"Version":"2012-10-17"}','2026-10-29T00:41:00.996Z'); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +INSERT INTO "iam_users" VALUES('504105003262','dev','arn:aws:iam::504105003262:user/dev',NULL,'2026-10-29T00:41:00.994Z'); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +INSERT INTO "indexes" VALUES('aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','1b7fae79-2959-4b69-adcd-bc4b790c140d','by_gpk','GSI','[{"AttributeName":"gpk","KeyType":"HASH"},{"AttributeName":"gsk","KeyType":"RANGE"}]','{"ProjectionType":"ALL"}','ACTIVE',NULL,NULL); +INSERT INTO "indexes" VALUES('aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','081d4835-c5c3-4dd5-81d7-94cae593aa14','by_lsk','LSI','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"lsk","KeyType":"RANGE"}]','{"ProjectionType":"KEYS_ONLY"}','ACTIVE',NULL,NULL); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','DescribeTable',1816.0,3,431.0,735.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Data',47816.0,14,186.0,16903.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Query',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Data',14.0,14,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','CreateTable',2178.0,1,2178.0,2178.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','','Scan',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_lsk','Query',0.5,1,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryCount','','Request','Catalog',5.0,5,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','DescribeTable',3.0,3,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Query',1179.0,3,348.0,468.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','CreateTable',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedWriteCapacityUnits','cu_orders','','PutItem',10.0,10,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Query',13.0,3,4.0,5.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','UpdateTimeToLive',1189.0,1,1189.0,1189.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedItemCount','cu_orders','','Scan',10.0,1,10.0,10.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','UpdateTimeToLive',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Query',515.0,3,60.0,253.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','StorageQueryLatency','','Request','Catalog',4548.0,5,364.0,1774.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','PutItem',48067.0,10,647.0,16941.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','SuccessfulRequestLatency','cu_orders','','Scan',234.0,1,234.0,234.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','RequestCount','','','Scan',1.0,1,1.0,1.0); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ConsumedReadCapacityUnits','cu_orders','by_gpk','Query',1.0,2,0.5,0.5); +INSERT INTO "metrics" VALUES('2026-10-29T00:41:00Z','ReturnedBytes','cu_orders','','Scan',508.0,1,508.0,508.0); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234460000010); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.3'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('index_propagation_delay_ms','10'); +INSERT INTO "settings" VALUES('encryption_key','AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); +INSERT INTO "settings" VALUES('default_account_id','504105003262'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000001','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"13a30289-be11-4e14-b575-209fb196cacd","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s00"}},"NewImage":{"expires":{"N":"4102444800"},"gpk":{"S":"g0"},"gsk":{"N":"0"},"lsk":{"S":"l09"},"payload":{"S":"item 0"},"pk":{"S":"p0"},"sk":{"S":"s00"}},"SequenceNumber":"000001793234460000001","SizeBytes":49,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.516Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000002','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"82a5f72c-3759-4cd0-9d24-3e343487b4d6","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s01"}},"NewImage":{"expires":{"N":"4102444801"},"gpk":{"S":"g1"},"gsk":{"N":"1"},"lsk":{"S":"l08"},"payload":{"S":"item 1"},"pk":{"S":"p1"},"sk":{"S":"s01"}},"SequenceNumber":"000001793234460000002","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.525Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000003','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"6e15d310-c09d-4a42-9103-7d98fdb65b3d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s02"}},"NewImage":{"expires":{"N":"4102444802"},"gpk":{"S":"g0"},"gsk":{"N":"2"},"lsk":{"S":"l07"},"payload":{"S":"item 2"},"pk":{"S":"p2"},"sk":{"S":"s02"}},"SequenceNumber":"000001793234460000003","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.527Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000004','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"5bccd10d-a61d-480d-9cda-fcf02e1f7b2a","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s03"}},"NewImage":{"expires":{"N":"4102444803"},"gpk":{"S":"g1"},"gsk":{"N":"3"},"lsk":{"S":"l06"},"payload":{"S":"item 3"},"pk":{"S":"p0"},"sk":{"S":"s03"}},"SequenceNumber":"000001793234460000004","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.545Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000005','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"ab50bc7f-10c8-42a8-b32b-48fdc12ae42a","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s04"}},"NewImage":{"expires":{"N":"4102444804"},"gpk":{"S":"g0"},"gsk":{"N":"4"},"lsk":{"S":"l05"},"payload":{"S":"item 4"},"pk":{"S":"p1"},"sk":{"S":"s04"}},"SequenceNumber":"000001793234460000005","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.554Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000006','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"6f65a1e5-b9ee-4af2-8ac2-c44cb4f23981","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s05"}},"NewImage":{"expires":{"N":"4102444805"},"gpk":{"S":"g1"},"gsk":{"N":"5"},"lsk":{"S":"l04"},"payload":{"S":"item 5"},"pk":{"S":"p2"},"sk":{"S":"s05"}},"SequenceNumber":"000001793234460000006","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.556Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000007','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"67289088-af5c-4220-9615-ac08c608af4b","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s06"}},"NewImage":{"expires":{"N":"4102444806"},"gpk":{"S":"g0"},"gsk":{"N":"6"},"lsk":{"S":"l03"},"payload":{"S":"item 6"},"pk":{"S":"p0"},"sk":{"S":"s06"}},"SequenceNumber":"000001793234460000007","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.558Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000003','000001793234460000008','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"63686c46-8907-4e18-8e9b-fb77ad458117","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p1"},"sk":{"S":"s07"}},"NewImage":{"expires":{"N":"4102444807"},"gpk":{"S":"g1"},"gsk":{"N":"7"},"lsk":{"S":"l02"},"payload":{"S":"item 7"},"pk":{"S":"p1"},"sk":{"S":"s07"}},"SequenceNumber":"000001793234460000008","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.569Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000009','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"f3991d77-142b-4b80-b4cc-060eeed6ca29","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p2"},"sk":{"S":"s08"}},"NewImage":{"expires":{"N":"4102444808"},"gpk":{"S":"g0"},"gsk":{"N":"8"},"lsk":{"S":"l01"},"payload":{"S":"item 8"},"pk":{"S":"p2"},"sk":{"S":"s08"}},"SequenceNumber":"000001793234460000009","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.571Z'); +INSERT INTO "stream_records" VALUES('shardId-cu_orders-0000000000000001','000001793234460000010','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','Insert','{"eventID":"f77c3b0e-da1e-481b-97d3-9ab573d10b0d","eventName":"INSERT","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"us-east-1","dynamodb":{"ApproximateCreationDateTime":1793234462,"Keys":{"pk":{"S":"p0"},"sk":{"S":"s09"}},"NewImage":{"expires":{"N":"4102444809"},"gpk":{"S":"g1"},"gsk":{"N":"9"},"lsk":{"S":"l00"},"payload":{"S":"item 9"},"pk":{"S":"p0"},"sk":{"S":"s09"}},"SequenceNumber":"000001793234460000010","SizeBytes":51,"StreamViewType":"NEW_AND_OLD_IMAGES"}}','2026-10-29T00:41:02.573Z'); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000000','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.498Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000001','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.498Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000002','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.498Z'); +INSERT INTO "stream_shards" VALUES('shardId-cu_orders-0000000000000003','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec',NULL,'000000000000000000000',NULL,'2026-10-29T00:41:01.498Z'); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +INSERT INTO "tables" VALUES('504105003262','cu_orders','[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}]','[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"S"},{"AttributeName":"gpk","AttributeType":"S"},{"AttributeName":"gsk","AttributeType":"N"},{"AttributeName":"lsk","AttributeType":"S"}]','PAY_PER_REQUEST',NULL,'{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"}','ACTIVE','2026-10-29T00:41:01.497730743Z',0,0,'arn:aws:dynamodb:us-east-1:504105003262:table/cu_orders','aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec','expires',0,NULL,'2026-10-29T00:41:01',1,NULL,NULL,NULL); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + -- Items the backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE UNIQUE INDEX idx_vector_indexes_index_id + ON vector_indexes (index_id); +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +CREATE INDEX "idx_order_1b7fae79-2959-4b69-adcd-bc4b790c140d" ON "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" (pk, sk_n, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_1b7fae79-2959-4b69-adcd-bc4b790c140d" ON "_ddb_1b7fae79-2959-4b69-adcd-bc4b790c140d" (base_pk, base_sk_s); +CREATE INDEX "idx_order_081d4835-c5c3-4dd5-81d7-94cae593aa14" ON "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" (pk, sk_s, base_pk, base_sk_s); +CREATE INDEX "idx_base_key_081d4835-c5c3-4dd5-81d7-94cae593aa14" ON "_ddb_081d4835-c5c3-4dd5-81d7-94cae593aa14" (base_pk, base_sk_s); +CREATE INDEX "idx_ttl_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" ON "_ddb_aa560d78-7f1f-4fcf-ba52-1b3e8e8031ec" (CAST(json_extract(item_data, '$."expires".N') AS INTEGER)) WHERE json_extract(item_data, '$."expires".N') IS NOT NULL; +DELETE FROM "sqlite_sequence"; +INSERT INTO "sqlite_sequence" VALUES('gsi_pending',10); +COMMIT; diff --git a/crates/storage-sqlite/tests/fixtures/released/dump.py b/crates/storage-sqlite/tests/fixtures/released/dump.py new file mode 100644 index 000000000..cecedb727 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/dump.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Dump a released SQLite database as a fixture. + +Usage: dump.py + +Works on a copy: checkpoints the WAL, replaces the `encryption_key` row with +the fixed test key (32 zero bytes), and writes Python's sqlite3 `iterdump` +(the statements the sqlite3 shell's `.dump` prints) after +`PRAGMA foreign_keys=OFF`, because tables are dumped in name order and child +rows can come before their parents. The database itself is not changed. +""" +import pathlib +import shutil +import sqlite3 +import sys +import tempfile + +TEST_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + + +def main(): + source = pathlib.Path(sys.argv[1]) + output = pathlib.Path(sys.argv[2]) + with tempfile.TemporaryDirectory() as tmp: + copy = pathlib.Path(tmp) / "db.sqlite" + for suffix in ("", "-wal", "-shm"): + part = source.with_name(source.name + suffix) + if part.exists(): + shutil.copy(part, copy.with_name(copy.name + suffix)) + conn = sqlite3.connect(copy) + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + conn.execute("UPDATE settings SET value = ? WHERE key = 'encryption_key'", (TEST_KEY,)) + conn.commit() + lines = ["PRAGMA foreign_keys=OFF;"] + lines.extend(conn.iterdump()) + conn.close() + output.write_text("\n".join(lines) + "\n") + print("%s: %d statements" % (output, len(lines))) + + +if __name__ == "__main__": + main() diff --git a/crates/storage-sqlite/tests/fixtures/released/regenerate.sh b/crates/storage-sqlite/tests/fixtures/released/regenerate.sh new file mode 100755 index 000000000..165c11b96 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/regenerate.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Regenerate one dev-image fixture from a published release. +# +# regenerate.sh [--backup] +# +# Runs extenddb/extenddb-dev: on 127.0.0.1: with the work +# directory as its data volume, seeds it with seed.py, stops it, and dumps +# the database with dump.py to dev-image-.sql beside this script +# (dev-image--backup.sql with --backup). The container is named +# $CONTAINER, by default extenddb-fixture-, and is removed at the +# end. The work directory is left for the caller to delete. +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +version=$1 +port=$2 +work=$3 +backup=${4:-} +name=${CONTAINER:-extenddb-fixture-$version} +suffix="" +if [ "$backup" = "--backup" ]; then suffix="-backup"; fi + +mkdir -p "$work" +if [ -n "$(ls -A "$work")" ]; then + echo "$work is not empty" >&2 + exit 1 +fi +docker run -d --name "$name" --user "$(id -u):$(id -g)" \ + -p "127.0.0.1:$port:18080" -v "$work:/var/lib/extenddb" \ + "extenddb/extenddb-dev:$version" > /dev/null +cleanup() { docker rm -f "$name" > /dev/null 2>&1 || true; } +trap cleanup EXIT + +healthy="" +for _ in $(seq 1 120); do + if curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/health" | grep -q 200; then + healthy=yes + break + fi + sleep 0.5 +done +if [ -z "$healthy" ]; then + docker logs "$name" >&2 + echo "the server did not become healthy" >&2 + exit 1 +fi +python3 "$here/seed.py" "http://127.0.0.1:$port" $backup +docker stop -t 20 "$name" > /dev/null +python3 "$here/dump.py" "$work/extenddb.sqlite" "$here/dev-image-$version$suffix.sql" diff --git a/crates/storage-sqlite/tests/fixtures/released/seed.py b/crates/storage-sqlite/tests/fixtures/released/seed.py new file mode 100644 index 000000000..fa30da4b0 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/seed.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Write the content every released SQLite fixture holds, through the API. + +Usage: seed.py [--backup] + +Creates `cu_orders` (pk S, sk S, GSI `by_gpk` on gpk S and gsk N, LSI +`by_lsk`, stream NEW_AND_OLD_IMAGES), enables TTL on `expires` and writes 10 +items. With `--backup`, also takes an on-demand backup of the table and waits +until it is AVAILABLE, so the database holds backup rows whose items have +sort keys. Exits non-zero on the first step that fails, except TTL, which is +reported and skipped. +""" +import argparse +import sys +import time + +import boto3 +import botocore +from botocore.config import Config + + +def wait(what, check, attempts=120, pause=0.5): + for _ in range(attempts): + if check(): + return + time.sleep(pause) + print("%s: timed out" % what, file=sys.stderr) + sys.exit(1) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("endpoint") + ap.add_argument("--backup", action="store_true") + a = ap.parse_args() + ddb = boto3.client( + "dynamodb", + endpoint_url=a.endpoint, + region_name="us-east-1", + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + config=Config(retries={"max_attempts": 3}), + ) + table = "cu_orders" + ddb.create_table( + TableName=table, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "gsk", "AttributeType": "N"}, + {"AttributeName": "lsk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "by_gpk", + "KeySchema": [ + {"AttributeName": "gpk", "KeyType": "HASH"}, + {"AttributeName": "gsk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + LocalSecondaryIndexes=[ + { + "IndexName": "by_lsk", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "lsk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + BillingMode="PAY_PER_REQUEST", + ) + + def active(): + d = ddb.describe_table(TableName=table)["Table"] + gsis = d.get("GlobalSecondaryIndexes", []) + return d["TableStatus"] == "ACTIVE" and all(g["IndexStatus"] == "ACTIVE" for g in gsis) + + wait("table ACTIVE", active) + print("create_table %s: ok" % table) + try: + ddb.update_time_to_live( + TableName=table, TimeToLiveSpecification={"Enabled": True, "AttributeName": "expires"} + ) + print("update_time_to_live: ok") + except botocore.exceptions.ClientError as e: + print("update_time_to_live: skipped (%s)" % e.response["Error"]["Code"]) + for i in range(10): + ddb.put_item( + TableName=table, + Item={ + "pk": {"S": "p%d" % (i % 3)}, + "sk": {"S": "s%02d" % i}, + "gpk": {"S": "g%d" % (i % 2)}, + "gsk": {"N": str(i)}, + "lsk": {"S": "l%02d" % (9 - i)}, + "expires": {"N": str(4102444800 + i)}, + "payload": {"S": "item %d" % i}, + }, + ) + print("put_item x10: ok") + + def indexed(): + r = ddb.query( + TableName=table, + IndexName="by_gpk", + KeyConditionExpression="gpk = :g", + ExpressionAttributeValues={":g": {"S": "g0"}}, + ) + return r["Count"] == 5 + + wait("query by_gpk g0 returns 5 items", indexed, attempts=40, pause=0.25) + if ddb.scan(TableName=table)["Count"] != 10: + print("scan did not return 10 items", file=sys.stderr) + sys.exit(2) + print("queries: ok") + if a.backup: + arn = ddb.create_backup(TableName=table, BackupName="cu_orders_fixture")[ + "BackupDetails" + ]["BackupArn"] + + def available(): + d = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + return d["BackupStatus"] == "AVAILABLE" + + wait("backup AVAILABLE", available) + print("create_backup: ok") + + +if __name__ == "__main__": + main() diff --git a/crates/storage-sqlite/tests/fixtures/released/v0.1.3-to-v0.1.6.sql b/crates/storage-sqlite/tests/fixtures/released/v0.1.3-to-v0.1.6.sql new file mode 100644 index 000000000..23c74f8c2 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/released/v0.1.3-to-v0.1.6.sql @@ -0,0 +1,267 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); +CREATE TABLE backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); +CREATE TABLE gsi_pending ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + table_id TEXT NOT NULL, + worker_partition INTEGER NOT NULL, + old_item TEXT, + new_item TEXT, + index_context TEXT NOT NULL, + ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) + REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, group_name) +); +CREATE TABLE iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); +CREATE TABLE iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); +CREATE TABLE iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, role_name) +); +CREATE TABLE iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + FOREIGN KEY (account_id, role_name) + REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); +CREATE TABLE iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) + REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); +CREATE TABLE iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, user_name) +); +CREATE TABLE idempotency_tokens ( + account_id TEXT NOT NULL, + token TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (account_id, token) +); +CREATE TABLE indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_propagation_delay_ms_non_negative + CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) +); +CREATE TABLE login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + success INTEGER NOT NULL, + source_ip TEXT +); +CREATE TABLE metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 9e999, + max REAL NOT NULL DEFAULT -9e999, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); +CREATE TABLE schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO "seq_counters" VALUES('stream',1793234645000000); +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT INTO "settings" VALUES('catalog_version','0.0.2'); +INSERT INTO "settings" VALUES('control_plane_delay_seconds','0.25'); +INSERT INTO "settings" VALUES('gsi_propagation_delay_ms','10'); +CREATE TABLE stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (shard_id, sequence_number) +); +CREATE TABLE stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +CREATE TABLE tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + table_class TEXT, + sse_specification TEXT, + on_demand_throughput TEXT, + PRIMARY KEY (account_id, table_name), + CONSTRAINT tables_table_id_unique UNIQUE (table_id) +); +CREATE TABLE tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); +CREATE INDEX idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; +CREATE INDEX idx_stream_shards_table ON stream_shards (table_id); +CREATE INDEX idx_stream_records_created ON stream_records (created_at); +CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); +CREATE INDEX idx_metrics_bucket ON metrics (bucket); +CREATE INDEX idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at DESC); +CREATE INDEX idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at DESC) + WHERE source_ip IS NOT NULL; +CREATE INDEX idx_backups_table ON backups (account_id, table_name); +CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn); +CREATE INDEX idx_gsi_pending_claim + ON gsi_pending (worker_partition, ready_at, id); +DELETE FROM "sqlite_sequence"; +COMMIT; diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index b604d9265..e8d61e952 100755 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -8,6 +8,9 @@ rust-version.workspace = true license.workspace = true [features] +# `test-util` exposes `migrations::check_released`, the checksum test each +# backend crate runs on its migration table. Production code never enables it. +test-util = [] # Compiles the `snapshot_conformance` check that each backend's tests run # against a real database. Production builds do not compile it. test-hooks = [] @@ -24,6 +27,7 @@ futures = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } thiserror = { workspace = true } time = { workspace = true } tokio = { workspace = true } diff --git a/crates/storage/src/backend.rs b/crates/storage/src/backend.rs index 54634257e..0c7007788 100644 --- a/crates/storage/src/backend.rs +++ b/crates/storage/src/backend.rs @@ -57,6 +57,11 @@ pub struct Backend { pub diagnostics_store: DiagnosticsStoreFactory, /// Creates the assembled server components for `serve`. pub server_components: ServerComponentsFactory, + /// The backend's catalog migrations on the shared runner, which `init`, + /// `migrate`, `verify` and `serve` use; `None` for a backend that still + /// migrates through its bootstrapper and checks its catalog in its own + /// server factory. + pub catalog_migrations: Option<&'static dyn crate::migrations::CatalogMigrations>, } static BACKEND: OnceLock = OnceLock::new(); diff --git a/crates/storage/src/bootstrapper.rs b/crates/storage/src/bootstrapper.rs index 5e3ca8dd2..354a85eb7 100755 --- a/crates/storage/src/bootstrapper.rs +++ b/crates/storage/src/bootstrapper.rs @@ -179,6 +179,14 @@ pub trait Bootstrapper: Send + Sync { /// Return the catalog connection URL for config file generation. fn catalog_connection_url(&self) -> String; + /// The deployment `init` runs the catalog migrations on, with the values + /// this bootstrapper resolved from the command line and the config file, + /// so the backend derives none of them again. The default names the + /// catalog's connection URL alone. + fn init_target(&self) -> crate::migrations::Target { + crate::migrations::Target::at(self.catalog_connection_url()) + } + /// Generate the backend-specific configuration subsection for extenddb.toml. /// Returns only the [`storage.backend_name`] section content, not the [storage] header. fn generate_backend_config_section(&self) -> String; @@ -311,10 +319,21 @@ pub mod helpers { Ok(()) } - /// Extract a CLI argument value by flag name. + /// Extract a CLI argument value by flag name, given as `--flag value` or + /// `--flag=value`, as clap accepts both. The first occurrence wins; clap + /// refuses a flag given twice before a bootstrapper reads the arguments. #[must_use] pub fn extract_arg(args: &[String], flag: &str) -> Option { - args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone()) + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + if arg == flag { + return iter.next().cloned(); + } + if let Some(value) = arg.strip_prefix(flag).and_then(|r| r.strip_prefix('=')) { + return Some(value.to_owned()); + } + } + None } #[cfg(test)] @@ -568,6 +587,19 @@ pub mod helpers { // Should return first occurrence assert_eq!(extract_arg(&args, "--host"), Some("first".to_string())); } + + /// clap accepts `--data-db=items`, so the bootstrapper reads it too; + /// a longer flag that starts with the name is not a match. + #[test] + fn test_extract_arg_equals_form() { + let args: Vec = ["init", "--data-db=items", "--pg-user", "admin"] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + assert_eq!(extract_arg(&args, "--data-db"), Some("items".to_owned())); + assert_eq!(extract_arg(&args, "--pg-user"), Some("admin".to_owned())); + assert_eq!(extract_arg(&args, "--data"), None); + } } } diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index 8fe3ac287..c936675a5 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -77,12 +77,6 @@ pub enum StorageError { /// nothing read so far may be combined with a fresh snapshot. #[error("snapshot expired: {0}")] SnapshotExpired(String), - #[error( - "Catalog version mismatch: expected {expected}, found {found}. Run 'extenddb migrate' to update." - )] - CatalogVersionMismatch { expected: String, found: String }, - #[error("Catalog not initialized. Run 'extenddb init' to set up the catalog.")] - CatalogNotInitialized, #[error("Connection error: {0}")] Connection(String), /// The backend does not implement the requested feature. Distinct from diff --git a/crates/storage/src/hooks.rs b/crates/storage/src/hooks.rs index 82259e7e6..a4a130930 100644 --- a/crates/storage/src/hooks.rs +++ b/crates/storage/src/hooks.rs @@ -63,4 +63,67 @@ pub trait ServerRuntimeHooks: Send + Sync { fn backend_info(&self) -> Option { None } + + /// The serving registration, for a backend whose database can end it + /// while the server runs: PostgreSQL ends the session that holds the + /// serving key on a restart, a failover or `pg_terminate_backend`, and a + /// registration row is lost when its renewals fail. `None` (the + /// default) for a backend whose registration lasts as long as the + /// process, such as a file lock. + /// + /// `serve` calls this once, before [`ServerRuntimeHooks::spawn_workers`]. + /// When [`ServingRegistrar::lost`] returns, the server answers every new + /// request with `ServiceUnavailable` and calls + /// [`ServingRegistrar::register_again`] at once and then every second. + /// It serves again on [`Reregistration::Registered`]. On + /// [`Reregistration::Refused`], or when 30 s pass without a + /// registration, it stops as a shutdown signal stops it (the listener + /// drains, [`WorkerContext::shutdown`] fires, the PID file is removed) + /// and exits 1. + fn serving_registrar(&self) -> Option> { + None + } +} + +/// A backend's side of the serving registration: it reports the loss of +/// the registration the server holds, and makes one attempt to register +/// again when asked. The server decides the rest: it refuses requests while +/// it holds no registration, chooses how often and how long to try, and +/// stops. +#[async_trait] +pub trait ServingRegistrar: Send + Sync { + /// Resolve once the registration the server holds now is lost, that is + /// once a migration could take the catalog, with the reason in words + /// for the log. The server calls it again only after + /// [`ServingRegistrar::register_again`] returned + /// [`Reregistration::Registered`], and the call then waits on the new + /// registration. + async fn lost(&self) -> String; + + /// One attempt to register again. Take a new registration without + /// waiting: a migration that holds the migration lock, or waits for the + /// servers to stop, makes the attempt [`Reregistration::Refused`]. Then + /// read the catalog's status under the new registration, and return + /// [`Reregistration::Registered`] only when it equals the status the + /// server started with; the new registration then replaces the lost + /// one. The server bounds each attempt with a timeout and drops it when + /// the timeout passes, so a dropped attempt must leave no registration + /// behind. + async fn register_again(&self) -> Reregistration; +} + +/// The outcome of one [`ServingRegistrar::register_again`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Reregistration { + /// The server holds a new registration, and the catalog's status is the + /// one it started with. + Registered, + /// No registration was taken, and a later attempt may take one: the + /// database did not answer or refused the connection, or the status + /// could not be read. The server tries again while its window lasts. + Unavailable(String), + /// The server must stop: a migration holds the migration lock or waits + /// for the servers to stop, or the catalog's status changed while the + /// server held no registration. + Refused(String), } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index b7aec12ff..b3c68fb98 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -18,6 +18,7 @@ pub mod error; pub mod hooks; pub mod management_store; pub mod management_store_factory; +pub mod migrations; pub mod operations; pub mod policy_migration; pub mod server_components; diff --git a/crates/storage/src/management_store/mod.rs b/crates/storage/src/management_store/mod.rs index c0b753845..958515254 100755 --- a/crates/storage/src/management_store/mod.rs +++ b/crates/storage/src/management_store/mod.rs @@ -44,6 +44,9 @@ pub type RoleListEntry = ( serde_json::Value, time::OffsetDateTime, ); +/// A stored policy document as the resource-tag migration reads it: the +/// decoded document, or the text of the error the store met decoding it. +pub type StoredDocument = Result; // ── Settings store ───────────────────────────────────────────────────── @@ -350,6 +353,36 @@ pub trait ManagementStore: Send + Sync { principal_name: &str, ) -> BoxFuture<'_, OpResult>>; + /// List a principal's policies for the resource-tag migration as + /// `(policy_name, document)`, decoding each document on its own: a + /// document the store cannot decode comes back as its decode error, and + /// the principal's other policies are still read. + /// + /// The default reads [`list_policies`](Self::list_policies), which fails + /// the whole list on a document it cannot decode. A backend that stores + /// documents as text overrides this. + fn list_policy_documents( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let (account_id, principal_type, principal_name) = ( + account_id.to_owned(), + principal_type.to_owned(), + principal_name.to_owned(), + ); + Box::pin(async move { + let policies = self + .list_policies(&account_id, &principal_type, &principal_name) + .await?; + Ok(policies + .into_iter() + .map(|(name, document, _created)| (name, Ok(document))) + .collect()) + }) + } + // ── Permissions boundaries ───────────────────────────────────── /// Set (upsert) a permissions boundary on a user. @@ -396,6 +429,38 @@ pub trait ManagementStore: Send + Sync { role_name: &str, ) -> BoxFuture<'_, OpResult<()>>; + /// The permissions boundary of a user or a role (`principal_type` + /// `user` or `role`) for the resource-tag migration: `None` when it has + /// none, else the decoded document or the text of the error the store + /// met decoding it. + /// + /// The default reads [`get_user_boundary`](Self::get_user_boundary) or + /// [`get_role_boundary`](Self::get_role_boundary), which fail on a + /// document they cannot decode. A backend that stores documents as text + /// overrides this. + fn boundary_document( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let (account_id, principal_type, principal_name) = ( + account_id.to_owned(), + principal_type.to_owned(), + principal_name.to_owned(), + ); + Box::pin(async move { + let boundary = match principal_type.as_str() { + "user" => self.get_user_boundary(&account_id, &principal_name).await, + "role" => self.get_role_boundary(&account_id, &principal_name).await, + other => Err(OpError::Validation(format!( + "a permissions boundary belongs to a user or a role, not a {other}" + ))), + }?; + Ok(boundary.map(Ok)) + }) + } + // ── Access keys ──────────────────────────────────────────────── /// Create an access key for an IAM user. The implementation handles diff --git a/crates/storage/src/management_store/types.rs b/crates/storage/src/management_store/types.rs index a8aab735a..26ba653b7 100755 --- a/crates/storage/src/management_store/types.rs +++ b/crates/storage/src/management_store/types.rs @@ -6,7 +6,7 @@ // ── Error types ──────────────────────────────────────────────────────── /// Error from a management or operational storage operation. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum OpError { /// Input validation failed (caller should have caught this). Validation(String), @@ -18,6 +18,39 @@ pub enum OpError { HasDependents(String), /// Internal storage error. Internal(String), + /// A database statement failed with this error code (PostgreSQL's + /// SQLSTATE). `message` is the driver's text, for logs and the + /// operator's commands; API responses never carry it. + Database { message: String, sqlstate: String }, +} + +impl OpError { + /// The database error code of the statement that failed, when the error + /// carries one. + #[must_use] + pub fn sqlstate(&self) -> Option<&str> { + match self { + Self::Database { sqlstate, .. } => Some(sqlstate), + Self::Validation(_) + | Self::AlreadyExists(_) + | Self::NotFound(_) + | Self::HasDependents(_) + | Self::Internal(_) => None, + } + } + + /// The error's text, without the variant name that `{:?}` prints. + #[must_use] + pub fn message(&self) -> &str { + match self { + Self::Validation(m) + | Self::AlreadyExists(m) + | Self::NotFound(m) + | Self::HasDependents(m) + | Self::Internal(m) + | Self::Database { message: m, .. } => m, + } + } } /// Result alias for management operations. diff --git a/crates/storage/src/migrations/advice.rs b/crates/storage/src/migrations/advice.rs new file mode 100644 index 000000000..5f0b6c949 --- /dev/null +++ b/crates/storage/src/migrations/advice.rs @@ -0,0 +1,517 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The messages `serve`, `migrate` and `verify` print when they stop on the +//! catalog: what the status is, and the command that resolves it, written +//! without a password in it. + +use super::catalog::{CatalogMigrations, Target}; +use super::registry::{MigrationId, RELEASED_THROUGH}; +use super::status::{CatalogStatus, MigrationError, Pending}; +use super::table::Kind; + +/// What a message needs beyond the status. +#[derive(Debug, Clone, Copy)] +pub struct Advice<'a> { + /// The ExtendDB release of this binary. + pub release: &'a str, + /// The deployment's location, for display (a SQLite path, a redacted URL). + pub location: &'a str, + /// The backend's [`backup_advice`](super::CatalogMigrations::backup_advice). + pub backup: &'a str, + /// The config file the commands were given, as an absolute path; `None` + /// when the command ran without one (a dev-mode server configured by + /// `EXTENDDB__` variables), so the advised command names none either. + /// The advised command quotes it for a POSIX shell. + pub config: Option<&'a str>, + /// The backend's + /// [`location_assignment`](super::CatalogMigrations::location_assignment) + /// when there is no config file, when the environment named the store + /// over the config file's location, or when the config file does not + /// name the store by its absolute location (a relative location, or + /// none, resolves against the directory the command runs in): the + /// advised command starts with it, so it acts on this deployment's + /// store wherever it runs and whatever `extenddb.toml` the directory + /// holds. + pub assignment: Option<&'a str>, +} + +impl Advice<'_> { + fn command(&self, flags: &str) -> String { + let assignment = self + .assignment + .map_or_else(String::new, |a| format!("{a} ")); + match self.config { + Some(config) => format!( + " {assignment}extenddb migrate --config {} {flags}", + shell_word(config) + ), + None => format!(" {assignment}extenddb migrate {flags}"), + } + } +} + +/// `word` as one word of a POSIX shell command: as it is when every +/// character is one a shell leaves alone, else in single quotes, with each +/// single quote in it written as `'\''`. +#[must_use] +pub fn shell_word(word: &str) -> String { + let plain = !word.is_empty() + && word + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"/._-+:@%".contains(&b)); + if plain { + word.to_owned() + } else { + format!("'{}'", word.replace('\'', "'\\''")) + } +} + +/// Whether the config file at `path` names the store by its absolute +/// `location`: a value in one of its `[storage.]` sections is +/// `location`, and `location` is absolute. A relative location in the +/// file, or none (the backend's default), resolves against the directory a +/// command runs in, and a file this cannot read or parse is treated the +/// same way. +fn names_location(path: &str, location: &str) -> bool { + if !std::path::Path::new(location).is_absolute() { + return false; + } + let Ok(text) = std::fs::read_to_string(path) else { + return false; + }; + let Ok(file) = text.parse::() else { + return false; + }; + file.get("storage") + .and_then(toml::Value::as_table) + .is_some_and(|storage| { + storage + .values() + .filter_map(toml::Value::as_table) + .any(|section| section.values().any(|v| v.as_str() == Some(location))) + }) +} + +/// The values an [`Advice`] borrows, built once by each command that prints +/// one (`serve`, `migrate`, `verify`). +#[derive(Debug, Clone)] +pub struct AdviceFor { + release: String, + location: String, + backup: String, + config: Option, + assignment: Option, +} + +impl AdviceFor { + /// The advice for the deployment at `target`, shown as `location` (the + /// caller redacts any password), with the backup clause of `migrations`. + /// Reads the config file `config` to see whether it names the store by + /// its absolute location. + #[must_use] + pub fn new( + release: &str, + location: String, + migrations: Option<&dyn CatalogMigrations>, + target: &Target, + config: Option<&str>, + ) -> Self { + Self { + release: release.to_owned(), + location, + backup: migrations.map_or_else( + || "back up the catalog".to_owned(), + |m| m.backup_advice(target), + ), + config: config.map(|c| { + std::path::absolute(c).map_or_else(|_| c.to_owned(), |p| p.display().to_string()) + }), + assignment: migrations + .filter(|m| { + config.is_none_or(|c| { + m.location_from_environment() || !names_location(c, &target.location) + }) + }) + .and_then(|m| m.location_assignment(target)), + } + } + + /// The advice, borrowing these values. + #[must_use] + pub fn advice(&self) -> Advice<'_> { + Advice { + release: &self.release, + location: &self.location, + backup: &self.backup, + config: self.config.as_deref(), + assignment: self.assignment.as_deref(), + } + } +} + +/// The message for a status a command does not proceed on. `Current` and +/// `AheadCompatible` get the status line alone. +#[must_use] +pub fn status_message(status: &CatalogStatus, advice: &Advice<'_>) -> String { + let latest = MigrationId::LATEST.number(); + match status { + CatalogStatus::Behind { version, pending } => { + let (at, stop) = match version { + Some(v) => ( + format!("the catalog is at migration {v}"), + "Stop the server.", + ), + None => ( + "the catalog has no migration ledger (a release before 1.0 created it)" + .to_owned(), + "Stop every server that uses this catalog: a server of a release before 1.0 \ + is not detected while it runs.", + ), + }; + format!( + "{at}, and ExtendDB {release} needs migration {latest}.\n{}\n{stop}\n{backup}, \ + then run:\n{}", + pending_line(pending), + advice.command("--yes"), + release = advice.release, + backup = capitalized(advice.backup), + ) + } + // `init --config ` refuses a config file that exists, and this + // catalog was found through one, so the command is named alone. + CatalogStatus::Uninitialized => format!( + "the catalog at {} is not initialized; create it with 'extenddb init'", + advice.location + ), + CatalogStatus::NeedsAttention { items } => format!( + "a pending migration needs a person before it can run:\n{}\nFix each item, then run:\n{}", + items + .iter() + .map(|item| format!(" {item}")) + .collect::>() + .join("\n"), + advice.command("--yes") + ), + CatalogStatus::Changed { id, .. } if *id > RELEASED_THROUGH => format!( + "{status}.\nMigration {id} is unreleased: clear its rows, then apply it again:\n{}\n{}", + advice.command(&format!("--forget {id}")), + advice.command("--yes") + ), + CatalogStatus::Changed { .. } => format!( + "{status}.\nA released migration never changes; run the ExtendDB release that \ + recorded it, or restore the backup taken before it ran." + ), + CatalogStatus::WrittenByOlderRelease { trace } => format!( + "a release before 1.0 wrote to this catalog after it was migrated ({trace}).\nStop \ + every server of a release before 1.0, then run:\n{}", + advice.command("--yes --reconverge") + ), + CatalogStatus::AheadIncompatible { .. } => format!( + "{status}.\nExtendDB {} cannot use it: run the release that applied those \ + migrations, or restore the backup taken before they ran.", + advice.release + ), + CatalogStatus::Current { .. } | CatalogStatus::AheadCompatible { .. } => status.to_string(), + } +} + +/// The message for an error a command stopped on. +#[must_use] +pub fn error_message(error: &MigrationError, advice: &Advice<'_>) -> String { + match error { + MigrationError::Status(status) => status_message(status, advice), + MigrationError::Locked(m) => format!("{m}. Retry once it has finished."), + MigrationError::Store(m) + | MigrationError::Database { message: m, .. } + | MigrationError::Refused(m) + | MigrationError::Blocked(m) => m.clone(), + MigrationError::Stopped { error, report } => { + let mut message = error_message(error, advice); + if !report.applied.is_empty() { + let steps: Vec = report + .applied + .iter() + .map(|s| format!("{} {} in {}", s.id, s.name, s.store_name)) + .collect(); + message.push_str(&format!( + "\nBefore it stopped, this run committed {}; the next run goes on from there.", + steps.join(", ") + )); + } + for note in &report.notes { + message.push('\n'); + message.push_str(note); + } + message + } + } +} + +fn capitalized(text: &str) -> String { + let mut chars = text.chars(); + chars.next().map_or_else(String::new, |first| { + first.to_uppercase().chain(chars).collect() + }) +} + +/// `Pending (all breaking): 2 name (catalog), ...`, or each kind beside its +/// migration when the kinds differ. +fn pending_line(pending: &[Pending]) -> String { + let kinds: Vec = pending.iter().map(|p| p.kind).collect(); + let same = kinds.windows(2).all(|w| w[0] == w[1]); + let items: Vec = pending + .iter() + .map(|p| { + if same { + format!("{} {} ({})", p.id, p.name, p.store.as_str()) + } else { + format!( + "{} {} ({}, {})", + p.id, + p.name, + p.store.as_str(), + p.kind.as_str() + ) + } + }) + .collect(); + match kinds.first() { + Some(kind) if same => format!("Pending (all {}): {}.", kind.as_str(), items.join(", ")), + _ => format!("Pending: {}.", items.join(", ")), + } +} + +#[cfg(test)] +#[path = "advice_tests.rs"] +mod quoting_tests; + +#[cfg(test)] +mod tests { + use super::{Advice, error_message, status_message}; + use crate::migrations::{ + AppliedStep, CatalogStatus, Kind, MigrationError, MigrationId, Pending, RunReport, + StoreKind, + }; + + const ADVICE: Advice<'static> = Advice { + release: "1.0.1", + location: "/var/lib/extenddb/extenddb.sqlite", + backup: "back up the SQLite database file /var/lib/extenddb/extenddb.sqlite", + config: Some("/etc/extenddb/extenddb.toml"), + assignment: None, + }; + + fn pending(kind: Kind) -> Vec { + MigrationId::ALL + .iter() + .skip(1) + .map(|id| Pending { + id: id.number(), + name: id.name(), + store: StoreKind::Catalog, + store_name: "db".to_owned(), + kind, + }) + .collect() + } + + #[test] + fn a_catalog_behind_names_each_migration_the_backup_and_the_command() { + let message = status_message( + &CatalogStatus::Behind { + version: Some(1), + pending: pending(Kind::Breaking), + }, + &ADVICE, + ); + let latest = MigrationId::LATEST.number(); + let first = message.lines().next().unwrap_or_default(); + assert_eq!( + first, + format!("the catalog is at migration 1, and ExtendDB 1.0.1 needs migration {latest}.") + ); + assert!(message.contains("Pending (all breaking): 2 "), "{message}"); + assert!( + message.contains( + "Stop the server.\nBack up the SQLite database file \ + /var/lib/extenddb/extenddb.sqlite, then run:" + ), + "{message}" + ); + assert!( + message.ends_with(" extenddb migrate --config /etc/extenddb/extenddb.toml --yes"), + "{message}" + ); + } + + #[test] + fn a_catalog_with_no_ledger_says_to_stop_every_server() { + let message = status_message( + &CatalogStatus::Behind { + version: None, + pending: pending(Kind::Breaking), + }, + &ADVICE, + ); + assert!(message.starts_with("the catalog has no migration ledger")); + assert!(message.contains("Stop every server that uses this catalog")); + } + + #[test] + fn mixed_kinds_are_named_per_migration() { + let mut list = pending(Kind::Breaking); + list[0].kind = Kind::Additive; + let message = status_message( + &CatalogStatus::Behind { + version: Some(1), + pending: list, + }, + &ADVICE, + ); + assert!(message.contains("Pending: 2 "), "{message}"); + assert!(message.contains("(catalog, additive)"), "{message}"); + assert!(message.contains("(catalog, breaking)"), "{message}"); + } + + #[test] + fn each_refusal_names_its_command() { + let cases = [ + ( + CatalogStatus::Uninitialized, + "create it with 'extenddb init'", + ), + ( + CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + }, + "--yes --reconverge", + ), + ( + CatalogStatus::NeedsAttention { + items: vec!["migration 2 resource_tag_condition_keys: a collision".to_owned()], + }, + " migration 2 resource_tag_condition_keys: a collision", + ), + ( + CatalogStatus::Changed { + id: 4, + store_name: "db".to_owned(), + recorded: "a".to_owned(), + expected: "b".to_owned(), + }, + "--forget 4", + ), + ( + CatalogStatus::AheadIncompatible { + unknown: vec![(9, "later".to_owned())], + }, + "ExtendDB 1.0.1 cannot use it", + ), + ]; + for (status, expected) in cases { + let message = status_message(&status, &ADVICE); + assert!(message.contains(expected), "{status:?}: {message}"); + assert!( + !message.contains("--pg-pass") && !message.contains("password"), + "{message}" + ); + } + } + + #[test] + fn a_held_lock_says_to_retry() { + let message = error_message(&MigrationError::Locked("held".to_owned()), &ADVICE); + assert_eq!(message, "held. Retry once it has finished."); + } + + /// Sessions that never finish by themselves carry their own remedy. + #[test] + fn a_blocked_run_prints_its_own_remedy_alone() { + let error = MigrationError::Blocked("end them, then run again".to_owned()); + assert_eq!(error_message(&error, &ADVICE), "end them, then run again"); + assert_eq!(error.exit_code(), crate::migrations::exit::LOCKED); + } + + /// A dev server configured by `EXTENDDB__` variables has no config file, + /// and `migrate` finds the same database from the same variables. + #[test] + fn without_a_config_file_the_command_names_none() { + let advice = Advice { + config: None, + ..ADVICE + }; + let message = status_message( + &CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + }, + &advice, + ); + assert!( + message.ends_with("\n extenddb migrate --yes --reconverge"), + "{message}" + ); + let advice = Advice { + config: None, + assignment: Some("EXTENDDB__STORAGE__SQLITE__PATH=/srv/x.sqlite"), + ..ADVICE + }; + let message = status_message( + &CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + }, + &advice, + ); + assert!( + message.ends_with( + "\n EXTENDDB__STORAGE__SQLITE__PATH=/srv/x.sqlite extenddb migrate --yes \ + --reconverge" + ), + "{message}" + ); + // A store the environment named over the config file's location: the + // command carries both, as the server found its store. + let advice = Advice { + assignment: Some("EXTENDDB__STORAGE__SQLITE__PATH=/srv/x.sqlite"), + ..ADVICE + }; + let message = status_message( + &CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + }, + &advice, + ); + assert!( + message.ends_with( + "\n EXTENDDB__STORAGE__SQLITE__PATH=/srv/x.sqlite extenddb migrate --config \ + /etc/extenddb/extenddb.toml --yes --reconverge" + ), + "{message}" + ); + } + + #[test] + fn a_stopped_run_names_what_it_committed_and_its_copy() { + let error = MigrationError::Stopped { + error: Box::new(MigrationError::Store("migration 3 failed".to_owned())), + report: RunReport { + from: None, + to: Some(2), + applied: vec![AppliedStep { + id: 1, + name: "baseline", + store_name: "db".to_owned(), + notes: Vec::new(), + }], + notes: vec!["copied the database to db.pre-migration-legacy-to-4-x".to_owned()], + }, + }; + assert_eq!(error.exit_code(), crate::migrations::exit::ERROR); + let message = error_message(&error, &ADVICE); + assert_eq!( + message, + "migration 3 failed\nBefore it stopped, this run committed 1 baseline in db; the \ + next run goes on from there.\ncopied the database to db.pre-migration-legacy-to-4-x" + ); + } +} diff --git a/crates/storage/src/migrations/advice_tests.rs b/crates/storage/src/migrations/advice_tests.rs new file mode 100644 index 000000000..a9746bce9 --- /dev/null +++ b/crates/storage/src/migrations/advice_tests.rs @@ -0,0 +1,225 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The advised command runs in a POSIX shell as printed, from any +//! directory: every path in it is quoted, and a store that the config file +//! names by a relative location is named by its absolute location. + +use std::path::PathBuf; +use std::time::Duration; + +use async_trait::async_trait; + +use super::{Advice, AdviceFor, shell_word, status_message}; +use crate::migrations::{ + CatalogMigrations, CatalogStatus, HeldLock, LedgerRow, Listed, MigrationError, MigrationId, + RunKind, RunReport, Target, +}; + +/// A backend whose location is a file, named by an assignment the way the +/// SQLite backend names it. The advice reads nothing else of it. +struct FileBackend; + +#[async_trait] +impl CatalogMigrations for FileBackend { + fn listed(&self, _id: MigrationId) -> Listed { + Listed::NotApplicable("test") + } + fn backup_advice(&self, target: &Target) -> String { + format!("back up {}", target.location) + } + fn location_assignment(&self, target: &Target) -> Option { + Some(format!( + "EXTENDDB__STORAGE__SQLITE__PATH={}", + shell_word(&target.location) + )) + } + async fn status(&self, _: &Target) -> Result { + Err(MigrationError::Store("not read".to_owned())) + } + async fn catalog_ledger(&self, _: &Target) -> Result, MigrationError> { + Err(MigrationError::Store("not read".to_owned())) + } + async fn lock(&self, _: &Target, _: Duration) -> Result { + Err(MigrationError::Store("not taken".to_owned())) + } + async fn run(&self, _: &HeldLock, _: &str, _: RunKind) -> Result { + Err(MigrationError::Store("not run".to_owned())) + } + async fn forget(&self, _: &HeldLock, _: u32) -> Result<(), MigrationError> { + Err(MigrationError::Store("not run".to_owned())) + } +} + +/// A directory of its own under the temporary directory, removed when the +/// value is dropped. +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Self { + let dir = + std::env::temp_dir().join(format!("extenddb-advice-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create the scratch directory"); + Self(dir) + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A config file in a directory whose name holds a space and a single +/// quote, with `section` as its `[storage.sqlite]` body. +fn config_file(scratch: &Scratch, section: &str) -> PathBuf { + let dir = scratch.0.join("sp ace's dir"); + std::fs::create_dir_all(&dir).expect("create the config directory"); + let path = dir.join("extenddb.toml"); + std::fs::write(&path, format!("[storage.sqlite]\n{section}")).expect("write the config"); + path +} + +/// The command `status` advises, as one line without its indent. +fn advised(status: &CatalogStatus, advice: &Advice<'_>) -> String { + let message = status_message(status, advice); + message + .lines() + .last() + .map(str::trim_start) + .unwrap_or_default() + .to_owned() +} + +fn written_by_older_release() -> CatalogStatus { + CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + } +} + +#[test] +fn a_word_with_shell_characters_is_single_quoted() { + assert_eq!(shell_word("/srv/x.sqlite"), "/srv/x.sqlite"); + assert_eq!(shell_word("/a b/it's"), "'/a b/it'\\''s'"); + assert_eq!(shell_word("/a/$HOME/`x`"), "'/a/$HOME/`x`'"); + assert_eq!(shell_word(""), "''"); +} + +#[test] +fn the_config_path_is_quoted_for_a_posix_shell() { + let advice = Advice { + release: "1.0.1", + location: "/srv/x.sqlite", + backup: "back up /srv/x.sqlite", + config: Some("/srv/sp ace's dir/extenddb.toml"), + assignment: None, + }; + assert_eq!( + advised(&written_by_older_release(), &advice), + "extenddb migrate --config '/srv/sp ace'\\''s dir/extenddb.toml' --yes --reconverge" + ); +} + +/// A config file that names the store by a relative location, or not at +/// all, gets the assignment with the absolute location; one that names it +/// by that absolute location does not. +#[test] +fn a_store_the_config_file_names_relatively_is_named_by_its_absolute_path() { + let scratch = Scratch::new("relative"); + let store = scratch.0.join("served").join("extenddb.sqlite"); + let location = store.to_string_lossy().into_owned(); + let assignment = |section: &str| { + let config = config_file(&scratch, section); + AdviceFor::new( + "1.0.1", + location.clone(), + Some(&FileBackend), + &Target::at(location.as_str()), + Some(&config.to_string_lossy()), + ) + .assignment + }; + let pinned = Some(format!( + "EXTENDDB__STORAGE__SQLITE__PATH={}", + shell_word(&location) + )); + assert_eq!(assignment("path = \"extenddb.sqlite\"\n"), pinned); + assert_eq!(assignment("pool_size = 10\n"), pinned); + assert_eq!( + assignment(&format!( + "path = {}\n", + toml::Value::from(location.as_str()) + )), + None + ); + assert_eq!(assignment("not toml at all ]["), pinned); +} + +/// The advised command, pasted into `sh` in another directory, passes the +/// config path as one argument and the store's absolute path, whatever the +/// directory holds. A stand-in `extenddb` prints what it received. +#[cfg(unix)] +#[test] +fn the_advised_command_runs_as_printed_from_another_directory() { + use std::os::unix::fs::PermissionsExt; + + let scratch = Scratch::new("shell"); + let config = config_file(&scratch, "path = \"extenddb.sqlite\"\n"); + // Where the command that printed the advice resolved the relative path. + let location = config + .parent() + .map(|dir| dir.join("extenddb.sqlite")) + .expect("the config directory"); + let location = location.to_string_lossy().into_owned(); + let bin = scratch.0.join("bin"); + std::fs::create_dir_all(&bin).expect("create the bin directory"); + let fake = bin.join("extenddb"); + std::fs::write( + &fake, + "#!/bin/sh\nprintf '%s\\n' \"${EXTENDDB__STORAGE__SQLITE__PATH-unset}\" \"$(pwd -P)\" \ + \"$@\"\n", + ) + .expect("write the stand-in"); + std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)) + .expect("make the stand-in executable"); + let elsewhere = scratch.0.join("elsewhere"); + std::fs::create_dir_all(&elsewhere).expect("create another directory"); + + let config_shown = config.to_string_lossy().into_owned(); + let values = AdviceFor::new( + "1.0.1", + location.clone(), + Some(&FileBackend), + &Target::at(location.as_str()), + Some(&config_shown), + ); + let command = advised(&written_by_older_release(), &values.advice()); + let path = std::env::var("PATH").unwrap_or_default(); + let output = std::process::Command::new("sh") + .arg("-c") + .arg(&command) + .current_dir(&elsewhere) + .env("PATH", format!("{}:{path}", bin.display())) + .env_remove("EXTENDDB__STORAGE__SQLITE__PATH") + .output() + .expect("run sh"); + assert!(output.status.success(), "{command}: {output:?}"); + let printed = String::from_utf8_lossy(&output.stdout); + let elsewhere = std::fs::canonicalize(&elsewhere).expect("canonicalize"); + let elsewhere = elsewhere.to_string_lossy(); + assert_eq!( + printed.lines().collect::>(), + [ + location.as_str(), + &*elsewhere, + "migrate", + "--config", + config_shown.as_str(), + "--yes", + "--reconverge", + ], + "{command}" + ); + assert!(std::path::Path::new(&location).is_absolute(), "{location}"); +} diff --git a/crates/storage/src/migrations/backend.rs b/crates/storage/src/migrations/backend.rs new file mode 100644 index 000000000..e936f6661 --- /dev/null +++ b/crates/storage/src/migrations/backend.rs @@ -0,0 +1,104 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The trait a backend implements for the runner. + +use async_trait::async_trait; + +use super::checksum::Released; +use super::code::{CodeBody, Done}; +use super::ledger::{CatalogTraces, LedgerRow}; +use super::registry::MigrationId; +use super::status::MigrationError; +use super::table::{Migration, StoreKind}; + +/// One store instance: its kind and the name a backend knows it by (the +/// SQLite file, a database name, a keyspace). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct StoreRef { + pub kind: StoreKind, + pub name: String, +} + +/// What the runner asks a backend to apply in one store. +pub enum Step<'a, H: 'static> { + /// Execute a file body. + File { + name: &'static str, + text: &'static str, + }, + /// Run a code body. + Code(&'a dyn CodeBody), + /// Run nothing and record the row only: a `NotApplicable` entry, or the + /// catalog row of an id whose bodies all run in other stores. + Record, +} + +/// What the runner needs from a backend. +/// +/// A backend owns its connections and its ledgers. The runner owns the order, +/// the version, the checksum rules and the status. +#[async_trait] +pub trait MigrationBackend: Send + Sync { + /// The handle code bodies receive, which reaches every store. + type Stores: Send + Sync + 'static; + + /// The backend's name, for messages. + fn backend_name(&self) -> &'static str; + + /// The backend's entry for `id`: an exhaustive `match` with no wildcard. + fn migration(&self, id: MigrationId) -> Migration; + + /// The backend's released checksums (see [`Released`]). + fn released(&self) -> &'static [Released]; + + /// The handle passed to code bodies. + fn stores(&self) -> &Self::Stores; + + /// The instances of one kind of store. Exactly one for + /// [`StoreKind::Catalog`]. + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError>; + + /// A store's ledger rows. A ledger table that does not exist reads as no + /// rows; any other failure is an error, never "not applied". + async fn read_ledger(&self, store: &StoreRef) -> Result, MigrationError>; + + /// The pre-1.0 traces in the catalog's settings. + async fn read_traces(&self) -> Result; + + /// Whether the catalog store holds no tables (or collections) at all, so + /// a code body's `detect` has nothing to read in it. + async fn catalog_is_empty(&self) -> Result; + + /// Called once, before the first write of a run that applies something. + /// `from` is `None` for a catalog with no ledger. Returns lines for the + /// operator (the SQLite copy's path). + async fn before_first_write( + &self, + from: Option, + to: u32, + ) -> Result, MigrationError>; + + /// Apply one step in one store. `row` is the ledger row to commit with + /// it, for a store that has no row for `id`; `None` means the store has + /// one (`--reconverge`), which is left as it is, notes and `applied_at` + /// included. When `catalog_version` is given (the catalog store), write + /// it to the `catalog_version` settings row in the same commit. The + /// backend appends its own notes (a post-check's) to the body's and + /// returns all of them. + async fn apply( + &self, + store: &StoreRef, + id: MigrationId, + step: Step<'_, Self::Stores>, + row: Option, + catalog_version: Option, + ) -> Result; + + /// Write the `catalog_version` row alone, after an id whose last body ran + /// outside the catalog store. + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError>; + + /// Delete one id's row from a store's ledger (`forget`). + async fn delete_ledger_row(&self, store: &StoreRef, id: u32) -> Result<(), MigrationError>; +} diff --git a/crates/storage/src/migrations/catalog.rs b/crates/storage/src/migrations/catalog.rs new file mode 100644 index 000000000..846794caf --- /dev/null +++ b/crates/storage/src/migrations/catalog.rs @@ -0,0 +1,479 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The installed backend's catalog migrations, as `init`, `migrate`, +//! `verify` and `serve` use them. +//! +//! The app crate does not name a backend crate, so a backend that migrates +//! through the shared runner installs a [`CatalogMigrations`] in its +//! [`Backend`](crate::Backend). A backend without one still migrates through +//! its [`Bootstrapper`](crate::bootstrapper::Bootstrapper), and the commands +//! take that path for it. + +use std::any::Any; +use std::time::Duration; + +use async_trait::async_trait; + +use super::ledger::LedgerRow; +use super::registry::MigrationId; +use super::status::{CatalogStatus, MigrationError, RunReport}; +use super::table::{Kind, Migration, StoreKind}; + +/// One migration of a backend's table, as `migrate --list` prints it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Listed { + /// The migration has bodies on this backend. + Applies { + kind: Kind, + convergent: bool, + stores: Vec, + }, + /// The migration changes nothing on this backend, for this reason. + NotApplicable(&'static str), +} + +impl Listed { + /// The listing of one entry of a backend's table. + #[must_use] + pub fn of(migration: &Migration) -> Self { + match migration { + Migration::NotApplicable(reason) => Self::NotApplicable(reason), + Migration::Applies { .. } => Self::Applies { + kind: migration.kind(), + convergent: migration.convergent(), + stores: migration.store_kinds(), + }, + } + } +} + +/// What a locked run does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunKind { + /// Create the catalog (`init`): apply every migration to a store whose + /// status is `Uninitialized`. + Create, + /// Apply every pending migration ([`Runner::migrate`](super::Runner::migrate)). + Migrate, + /// Run every convergent migration again + /// ([`Runner::reconverge`](super::Runner::reconverge)). + Reconverge, +} + +/// The migration lock of one deployment, held until it is dropped. What it +/// holds is the backend's own: the backend that took it reads it back with +/// [`HeldLock::get`]. +pub struct HeldLock { + location: String, + held: Box, +} + +impl HeldLock { + /// A lock on the deployment at `location`. + pub fn new(location: impl Into, held: impl Any + Send + Sync) -> Self { + Self { + location: location.into(), + held: Box::new(held), + } + } + + /// The location the lock was taken for. + #[must_use] + pub fn location(&self) -> &str { + &self.location + } + + /// The backend's lock value, if it has type `T`. + #[must_use] + pub fn get(&self) -> Option<&T> { + self.held.downcast_ref() + } +} + +impl std::fmt::Debug for HeldLock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HeldLock") + .field("location", &self.location) + .finish_non_exhaustive() + } +} + +/// The deployment a command works on, with the values the command's flags +/// gave (parsed by the command, or resolved by `init`'s bootstrapper). A +/// backend reads these fields and never the command line. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Target { + /// The deployment's storage location as the config file gives it + /// (`[storage.]`): on SQLite the database file, on PostgreSQL + /// the catalog database's connection string. + pub location: String, + /// The data store the command names (`--data-db` on PostgreSQL); `None` + /// lets the backend read it from the catalog. + pub data_store: Option, + /// The admin role for the backend's admin connection: on PostgreSQL the + /// role of the pgvector attempt (`--pg-user` and `--pg-pass`, or the + /// role `init` created the databases with). + pub admin: Option, + /// The data store's connection string, which `init` records with + /// migration 1. Only `init` sets it. + pub data_connection: Option, +} + +impl Target { + /// A target at `location` with no other values. + #[must_use] + pub fn at(location: impl Into) -> Self { + Self { + location: location.into(), + ..Self::default() + } + } +} + +/// An admin role and its password. `Debug` prints no password. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct AdminRole { + pub user: String, + pub password: Option, +} + +impl std::fmt::Debug for AdminRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminRole") + .field("user", &self.user) + .field("password", &self.password.as_ref().map(|_| "")) + .finish() + } +} + +/// A backend's catalog migrations, for the deployment a [`Target`] names. +#[async_trait] +pub trait CatalogMigrations: Send + Sync { + /// This backend's entry for `id`. Opens nothing. + fn listed(&self, id: MigrationId) -> Listed; + + /// What an operator backs up before migrating the deployment, as one + /// clause: "back up ...". Opens nothing. + fn backup_advice(&self, target: &Target) -> String; + + /// The name of the data store `target` resolves to, which the commands + /// put in the target they give [`backup_advice`](Self::backup_advice): + /// the one the target names, else the one the catalog records, or + /// `None` when it cannot be read. Read-only. The target's by default. + async fn data_store_name(&self, target: &Target) -> Option { + target.data_store.clone() + } + + /// The environment assignment that names this deployment's store by its + /// absolute location, such as + /// `EXTENDDB__STORAGE__SQLITE__PATH=`, with the value + /// quoted for a POSIX shell, for an advised command that must reach the + /// store from any directory: one run without a config file, or with a + /// config file that names the store by a relative location. `None` (the + /// default) when the backend's location cannot be given that way + /// without a secret in it. + fn location_assignment(&self, target: &Target) -> Option { + let _ = target; + None + } + + /// Whether this process's environment names the store, in the variable + /// [`location_assignment`](Self::location_assignment) sets, which + /// overrides a config file's location. The advised command then carries + /// that assignment beside the config file, so it acts on the store the + /// command or server used. `false` (the default) when the backend has no + /// such variable. + fn location_from_environment(&self) -> bool { + false + } + + /// The catalog status. Takes no lock and writes nothing, so it runs + /// beside a server. + async fn status(&self, target: &Target) -> Result; + + /// The catalog store's ledger rows, in id order, for `verify` to print + /// their notes. Read-only; empty for a store with no ledger. + async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError>; + + /// Take the migration lock, retrying for up to `timeout` while another + /// process holds it ([`MigrationError::Locked`] after, exit 6). The lock + /// carries what [`CatalogMigrations::run`] needs of `target`. + async fn lock(&self, target: &Target, timeout: Duration) -> Result; + + /// Run the migrations under a lock this backend took. + async fn run( + &self, + lock: &HeldLock, + release: &str, + kind: RunKind, + ) -> Result; + + /// Delete the ledger rows of an unreleased id under a lock this backend + /// took ([`Runner::forget`](super::Runner::forget)). + async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError>; + + /// Notes about the deployment that its status does not show, for the + /// preview and `verify` to print and `migrate --yes` to check before it + /// takes the lock ([`before_lock`]). On PostgreSQL: tables and sequences + /// the application role does not own. Read-only, like the status. None + /// by default. + async fn status_notes(&self, target: &Target) -> Result, MigrationError> { + let _ = target; + Ok(Vec::new()) + } + + /// Lines saying what the pending migrations of a `Behind` `status` + /// change, for the preview and `verify` to print: while migration 2 is + /// pending in the catalog store, the stored policies it rewrites + /// ([`pending_rewrite_lines`](crate::policy_migration::pending_rewrite_lines)). + /// Read-only. Empty for any other status. The default reads the + /// policies through the installed backend's management store; a backend + /// whose management store writes when it opens reads them another way. + async fn pending_changes( + &self, + target: &Target, + status: &CatalogStatus, + ) -> Result, MigrationError> { + if !policy_rewrite_pending(status) { + return Ok(Vec::new()); + } + let store = crate::management_store_factory::create_management_store(&target.location) + .await + .map_err(|e| { + MigrationError::store( + "open the catalog to list the policies migration 2 rewrites", + e, + ) + })?; + policy_rewrite_lines(store.as_ref()).await + } +} + +/// Whether migration 2 is pending in the catalog store of a `Behind` +/// `status`: when [`CatalogMigrations::pending_changes`] lists the policies +/// it rewrites. +#[must_use] +pub fn policy_rewrite_pending(status: &CatalogStatus) -> bool { + let CatalogStatus::Behind { pending, .. } = status else { + return false; + }; + let rewrite = MigrationId::ResourceTagConditionKeys.number(); + pending + .iter() + .any(|p| p.id == rewrite && p.store == StoreKind::Catalog) +} + +/// [`CatalogMigrations::pending_changes`]'s lines for a pending migration +/// 2, from a pass over the policies stored in `store` that writes nothing. +/// +/// # Errors +/// +/// A stored policy could not be read. +pub async fn policy_rewrite_lines( + store: &dyn crate::management_store::ManagementStore, +) -> Result, MigrationError> { + let report = crate::policy_migration::migrate_resource_tag_condition_keys(store, false).await; + if report.errors > 0 { + return Err(MigrationError::Store( + crate::policy_migration::failure_message(&report), + )); + } + Ok(crate::policy_migration::pending_rewrite_lines(&report)) +} + +/// A note from [`CatalogMigrations::status_notes`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatusNote { + /// The line the preview and `verify` print. + pub text: String, + /// Whether `migrate --yes` refuses before it takes the lock while the + /// note holds and the run would apply something. + pub refuses_runs: bool, +} + +/// What `migrate --yes` checks before it takes the lock: the backend's +/// status notes, returned for the operator to read, or a refusal (exit 1) +/// when a note refuses runs and the run would apply something, which is +/// every `--reconverge` and a `migrate` whose status is not `Current` or +/// `AheadCompatible`. `init` creates its objects and checks nothing. +/// +/// # Errors +/// +/// A note refuses the run, or as [`CatalogMigrations::status_notes`] and +/// [`CatalogMigrations::status`]. +pub async fn before_lock( + migrations: &dyn CatalogMigrations, + target: &Target, + kind: RunKind, +) -> Result, MigrationError> { + if kind == RunKind::Create { + return Ok(Vec::new()); + } + let notes = migrations.status_notes(target).await?; + let refusing: Vec<&str> = notes + .iter() + .filter(|n| n.refuses_runs) + .map(|n| n.text.as_str()) + .collect(); + if refusing.is_empty() { + return Ok(notes); + } + if kind == RunKind::Migrate { + let status = migrations.status(target).await?; + if matches!( + status, + CatalogStatus::Current { .. } | CatalogStatus::AheadCompatible { .. } + ) { + return Ok(notes); + } + } + // One note per line, as the preview prints them, so the statements + // each note ends with can be copied on their own. + Err(MigrationError::Refused(format!( + "migrate --yes refuses to apply migrations while these notes hold:\n{}\nNothing was \ + changed and no lock was taken.", + refusing + .iter() + .map(|n| format!("Note: {n}")) + .collect::>() + .join("\n") + ))) +} + +/// Run `kind` under `lock`, which `migrations` took for `target`. The +/// status is read under the lock first, and a status the run would refuse +/// stops it before the backend opens anything to write: `migrate --yes` +/// refuses a store that is not a catalog (`Uninitialized`, exit 1, naming +/// `extenddb init`), and `init` refuses a store that is one. +/// +/// # Errors +/// +/// The status refuses the run, or as [`CatalogMigrations::run`]. +pub async fn run_locked( + migrations: &dyn CatalogMigrations, + target: &Target, + lock: &HeldLock, + release: &str, + kind: RunKind, +) -> Result { + let status = migrations.status(target).await?; + if let Some(refusal) = refusal(&status, kind) { + return Err(refusal); + } + migrations.run(lock, release, kind).await +} + +/// Delete the ledger rows of `id` under `lock`, which `migrations` took for +/// `target`, after reading the status under the lock as [`run_locked`] +/// does: a store that is not a catalog (`Uninitialized`) is refused, naming +/// `extenddb init`, before the backend opens anything to write. +/// +/// # Errors +/// +/// The store is not a catalog, or as [`CatalogMigrations::forget`]. +pub async fn forget_locked( + migrations: &dyn CatalogMigrations, + target: &Target, + lock: &HeldLock, + id: u32, +) -> Result<(), MigrationError> { + let status = migrations.status(target).await?; + if status == CatalogStatus::Uninitialized { + return Err(MigrationError::Status(status)); + } + migrations.forget(lock, id).await +} + +/// The error a run of `kind` stops with on `status`, if any. The runner +/// checks the same statuses again under its own reads. +fn refusal(status: &CatalogStatus, kind: RunKind) -> Option { + match (kind, status) { + (RunKind::Create, CatalogStatus::Uninitialized) => None, + (RunKind::Create, status) => Some(MigrationError::Refused(format!( + "the catalog is already initialized ({status}); use 'extenddb migrate' for pending \ + migrations" + ))), + ( + _, + CatalogStatus::Uninitialized + | CatalogStatus::Changed { .. } + | CatalogStatus::AheadIncompatible { .. } + | CatalogStatus::NeedsAttention { .. }, + ) + | (RunKind::Migrate, CatalogStatus::WrittenByOlderRelease { .. }) => { + Some(MigrationError::Status(status.clone())) + } + _ => None, + } +} + +/// The installed backend's catalog migrations, or `None` when no backend is +/// installed or the installed one still migrates through its bootstrapper. +#[must_use] +pub fn installed() -> Option<&'static dyn CatalogMigrations> { + crate::backend::try_backend().and_then(|backend| backend.catalog_migrations) +} + +/// What `extenddb version`, `verify` and the `serve` banner call the catalog +/// this binary needs: `migration `, the highest id it carries, on a +/// backend on the runner; else the backend's catalog version string. +#[must_use] +pub fn binary_catalog_label() -> String { + if installed().is_some() { + return format!("migration {}", MigrationId::LATEST.number()); + } + crate::operations::catalog_version().unwrap_or_else(|_| "unknown".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::{AdminRole, RunKind, Target, refusal}; + use crate::migrations::{CatalogStatus, MigrationError, exit}; + + #[test] + fn an_admin_password_is_not_printed() { + let target = Target { + admin: Some(AdminRole { + user: "admin".to_owned(), + password: Some("s3cret".to_owned()), + }), + ..Target::at("db") + }; + let shown = format!("{target:?}"); + assert!(shown.contains("admin"), "{shown}"); + assert!(!shown.contains("s3cret"), "{shown}"); + } + + #[test] + fn a_run_refuses_what_its_kind_cannot_apply() { + let current = CatalogStatus::Current { version: 4 }; + let older = CatalogStatus::WrittenByOlderRelease { + trace: "catalog_version is 0.0.4".to_owned(), + }; + let attention = CatalogStatus::NeedsAttention { + items: vec!["collision".to_owned()], + }; + let code = |status: &CatalogStatus, kind| refusal(status, kind).map(|e| e.exit_code()); + // migrate --yes and --reconverge refuse a store that is not a catalog + // with the status, so the message names `extenddb init` (exit 1). + for kind in [RunKind::Migrate, RunKind::Reconverge] { + assert_eq!( + refusal(&CatalogStatus::Uninitialized, kind), + Some(MigrationError::Status(CatalogStatus::Uninitialized)) + ); + assert_eq!(code(&attention, kind), Some(exit::ATTENTION)); + assert_eq!(refusal(¤t, kind), None); + } + assert_eq!(code(&older, RunKind::Migrate), Some(exit::ATTENTION)); + assert_eq!(refusal(&older, RunKind::Reconverge), None); + // init creates only what is not a catalog yet. + assert_eq!( + refusal(&CatalogStatus::Uninitialized, RunKind::Create), + None + ); + let again = refusal(¤t, RunKind::Create).expect("refused"); + assert_eq!(again.exit_code(), exit::ERROR); + assert!(again.to_string().contains("already initialized"), "{again}"); + } +} diff --git a/crates/storage/src/migrations/checksum.rs b/crates/storage/src/migrations/checksum.rs new file mode 100644 index 000000000..f7b73cb52 --- /dev/null +++ b/crates/storage/src/migrations/checksum.rs @@ -0,0 +1,592 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Checksums of migration bodies, and the test every backend runs on its +//! table to keep released bodies frozen. + +use sha2::{Digest, Sha256}; + +use super::registry::MigrationId; +use super::table::{Body, Kind, Migration, StoreBody, StoreKind}; + +/// SHA-256 of a file body with CRLF normalised to LF, as lower-case hex. The +/// checksum of a code body's source and of a `NotApplicable` reason are +/// computed the same way. +#[must_use] +pub fn file_checksum(text: &str) -> String { + let normalized = text.replace("\r\n", "\n"); + Sha256::digest(normalized.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// A released body's pinned checksum, kind and convergent flag, as a +/// backend's `RELEASED` table lists them. The first entry for an `(id, store)` +/// is the released body; each later one is a deliberate change and carries a +/// `rebless` reason. A file body is recorded with the first entry's checksum +/// and the runtime accepts any entry's, so every release reads the same +/// ledger row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Released { + pub id: MigrationId, + pub store: StoreKind, + /// SHA-256 of the file, of the code body's source, or of the + /// `NotApplicable` reason. + pub checksum: &'static str, + /// The fingerprint a code body records in the ledger; `None` for a file + /// body and a `NotApplicable` entry. It never changes after release. + pub fingerprint: Option<&'static str>, + pub kind: Kind, + pub convergent: bool, + pub rebless: Option<&'static str>, +} + +/// The checksum the test pins for a body: of the file, or of the code +/// body's source. +#[cfg(any(test, feature = "test-util"))] +pub(crate) fn pinned_checksum(body: &Body) -> String { + match body { + Body::File { text, .. } => file_checksum(text), + Body::Code { source, .. } => file_checksum(source), + } +} + +/// The checksum this binary computes for a body's ledger row, before the +/// released entries are consulted: the file's, or the code body's +/// fingerprint. +pub(crate) fn computed_checksum(body: &Body) -> String { + match body { + Body::File { text, .. } => file_checksum(text), + Body::Code { body, .. } => body.fingerprint().to_owned(), + } +} + +/// The released checksums of `id` in a `store` kind, first entry first. +fn released_checksums( + released: &[Released], + id: MigrationId, + store: StoreKind, +) -> impl Iterator + '_ { + released + .iter() + .filter(move |r| r.id == id && r.store == store) + .map(|r| r.checksum.to_owned()) +} + +/// The body a row of an id in a `kind` store records: the store's own body, +/// or, for the catalog row of an id whose bodies all run in other stores, +/// the id's first body. +fn recorded_body(bodies: &[StoreBody], kind: StoreKind) -> Option<&StoreBody> { + bodies + .iter() + .find(|b| b.store == kind) + .or_else(|| bodies.first().filter(|_| kind == StoreKind::Catalog)) +} + +/// The checksum this binary records for a row of `id` in a `kind` store. A +/// file body or a `NotApplicable` reason that has been released records its +/// first released checksum, so a reblessed body writes the value every +/// release accepts; a code body records its fingerprint. +pub(crate) fn record_checksum( + released: &[Released], + id: MigrationId, + migration: &Migration, + kind: StoreKind, +) -> Option { + match migration { + Migration::NotApplicable(reason) => (kind == StoreKind::Catalog).then(|| { + released_checksums(released, id, StoreKind::Catalog) + .next() + .unwrap_or_else(|| file_checksum(reason)) + }), + Migration::Applies { bodies, .. } => recorded_body(bodies, kind).map(|b| match &b.body { + Body::File { .. } => released_checksums(released, id, b.store) + .next() + .unwrap_or_else(|| computed_checksum(&b.body)), + Body::Code { .. } => computed_checksum(&b.body), + }), + } +} + +/// The checksums this binary accepts for a row of `id` in a `kind` store: +/// the one it records, and for a file body or a `NotApplicable` reason every +/// released one and the one its current text has. +pub(crate) fn accepted_checksums( + released: &[Released], + id: MigrationId, + migration: &Migration, + kind: StoreKind, +) -> Vec { + let Some(recorded) = record_checksum(released, id, migration, kind) else { + return Vec::new(); + }; + let mut accepted = vec![recorded]; + match migration { + Migration::NotApplicable(reason) => { + accepted.extend(released_checksums(released, id, StoreKind::Catalog)); + accepted.push(file_checksum(reason)); + } + Migration::Applies { bodies, .. } => { + if let Some(b) = recorded_body(bodies, kind) + && let Body::File { .. } = &b.body + { + accepted.extend(released_checksums(released, id, b.store)); + accepted.push(computed_checksum(&b.body)); + } + } + } + accepted +} + +/// The line a release PR pastes into a backend's `RELEASED` table. +#[cfg(any(test, feature = "test-util"))] +fn released_line( + id: MigrationId, + store: StoreKind, + checksum: &str, + fingerprint: Option<&str>, + kind: Kind, + convergent: bool, +) -> String { + let fingerprint = fingerprint.map_or_else(|| "None".to_owned(), |f| format!("Some(\"{f}\")")); + format!( + " Released {{ id: MigrationId::{id:?}, store: StoreKind::{store:?}, \ + checksum: \"{checksum}\", fingerprint: {fingerprint}, kind: Kind::{kind:?}, \ + convergent: {convergent}, rebless: None }}," + ) +} + +/// One body as the checksum test sees it. +#[cfg(any(test, feature = "test-util"))] +struct Pinned { + store: StoreKind, + checksum: String, + fingerprint: Option<&'static str>, +} + +/// Check a backend's migration table against its released checksums. +/// +/// - Every released body (`id <= released_through`) must match the last +/// `RELEASED` entry for its id and store, every entry after the first must +/// carry a `rebless` reason, and the kind and convergent flag must equal +/// the table's. A code body's fingerprint must equal every entry's, +/// because deployed ledgers record it. A `NotApplicable` reason counts as +/// a shipped body. +/// - No entry may name an id above `released_through`, or a store the +/// table does not declare. A `NotApplicable` reason must not be empty. +/// - A file's name must be `NNNN_.sql` or `.cql` with `NNNN` the id. +/// No statement of a file may begin with `BEGIN`, `COMMIT`, `END`, +/// `ROLLBACK`, `SAVEPOINT` or `RELEASE` (the statements inside a +/// `CREATE TRIGGER` body are not statements of the file), the text may not +/// mention `catalog_version` outside comments, and a CQL comment may not +/// hold a `;`. +/// - A code body's fingerprint must start with `NNNN_/`. +/// +/// Returns the lines to paste for every id the table carries, for the +/// release PR that raises `RELEASED_THROUGH`; on failure the error lists +/// every problem followed by the same lines. +/// +/// # Errors +/// +/// One message naming every problem found. +#[cfg(any(test, feature = "test-util"))] +pub fn check_released( + backend: &str, + table: impl Fn(MigrationId) -> Migration, + released: &[Released], + released_through: u32, +) -> Result { + let mut problems = Vec::new(); + let mut lines = Vec::new(); + for id in MigrationId::ALL { + let migration = table(id); + let mut bodies: Vec = Vec::new(); + match &migration { + Migration::NotApplicable(reason) => { + if reason.trim().is_empty() { + problems.push(format!("{backend} {id}: a NotApplicable reason is empty")); + } + bodies.push(Pinned { + store: StoreKind::Catalog, + checksum: file_checksum(reason), + fingerprint: None, + }); + } + Migration::Applies { bodies: list, .. } => { + if list.is_empty() { + problems.push(format!("{backend} {id}: declares no body")); + } + for store_body in list { + let fingerprint = match &store_body.body { + Body::File { name, text } => { + check_file(backend, id, name, text, &mut problems); + None + } + Body::Code { body, .. } => { + check_fingerprint(backend, id, body.fingerprint(), &mut problems); + Some(body.fingerprint()) + } + }; + bodies.push(Pinned { + store: store_body.store, + checksum: pinned_checksum(&store_body.body), + fingerprint, + }); + } + } + } + for body in &bodies { + lines.push(released_line( + id, + body.store, + &body.checksum, + body.fingerprint, + migration.kind(), + migration.convergent(), + )); + if id.number() <= released_through { + check_pins(backend, id, &migration, body, released, &mut problems); + } + } + for entry in released.iter().filter(|r| r.id == id) { + if id.number() > released_through { + problems.push(format!( + "{backend} {id}: pins a checksum but is above RELEASED_THROUGH \ + ({released_through}); unreleased bodies carry no literal" + )); + } else if !bodies.iter().any(|b| b.store == entry.store) { + problems.push(format!( + "{backend} {id}: pins a checksum for the {} store, which the table does not \ + declare", + entry.store.as_str() + )); + } + } + } + let paste = lines.join("\n"); + if problems.is_empty() { + Ok(paste) + } else { + Err(format!( + "{}\n\nLines for the release PR that raises RELEASED_THROUGH:\n{paste}", + problems.join("\n") + )) + } +} + +/// The rules for one released body and its `RELEASED` entries. +#[cfg(any(test, feature = "test-util"))] +fn check_pins( + backend: &str, + id: MigrationId, + migration: &Migration, + body: &Pinned, + released: &[Released], + problems: &mut Vec, +) { + let store = body.store.as_str(); + let entries: Vec<&Released> = released + .iter() + .filter(|r| r.id == id && r.store == body.store) + .collect(); + let Some(last) = entries.last() else { + problems.push(format!( + "{backend} {id} ({store}): released but has no pinned checksum" + )); + return; + }; + if last.checksum != body.checksum { + problems.push(format!( + "{backend} {id} ({store}): the body changed after release (pinned {}, now {}). A \ + released body is frozen; if the change is deliberate, add an entry with the new \ + checksum and a `rebless: Some(\"\")` note", + last.checksum, body.checksum + )); + } + for later in entries.iter().skip(1) { + if later.rebless.is_none_or(|r| r.trim().is_empty()) { + problems.push(format!( + "{backend} {id} ({store}): checksum {} replaces a released one without a \ + rebless reason", + later.checksum + )); + } + } + for entry in &entries { + if entry.kind != migration.kind() || entry.convergent != migration.convergent() { + problems.push(format!( + "{backend} {id} ({store}): released as {} convergent={}, now {} \ + convergent={}; older binaries read the released values", + entry.kind.as_str(), + entry.convergent, + migration.kind().as_str(), + migration.convergent() + )); + } + if entry.fingerprint != body.fingerprint { + problems.push(format!( + "{backend} {id} ({store}): released with fingerprint {:?}, now {:?}; deployed \ + ledgers record the fingerprint, so a released code body keeps it", + entry.fingerprint, body.fingerprint + )); + } + } +} + +/// A code body's fingerprint starts with its id's file stem. +#[cfg(any(test, feature = "test-util"))] +fn check_fingerprint( + backend: &str, + id: MigrationId, + fingerprint: &str, + problems: &mut Vec, +) { + let stem = format!("{:04}_{}/", id.number(), id.name()); + if !fingerprint.starts_with(&stem) { + problems.push(format!( + "{backend} {id}: code body fingerprint {fingerprint:?} should start with {stem:?}" + )); + } +} + +/// Statements a file body may not hold, because the runner owns the +/// transaction: each of them commits it, ends it, or nests in it. +#[cfg(any(test, feature = "test-util"))] +const TRANSACTION_KEYWORDS: [&str; 6] = + ["BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE"]; + +/// The file rules of [`check_released`]. +#[cfg(any(test, feature = "test-util"))] +fn check_file(backend: &str, id: MigrationId, name: &str, text: &str, problems: &mut Vec) { + let expected_stem = format!("{:04}_{}", id.number(), id.name()); + let (stem, extension) = name.rsplit_once('.').unwrap_or((name, "")); + let number = stem.split('_').next().unwrap_or_default(); + if number.parse::().ok() != Some(id.number()) { + problems.push(format!( + "{backend} {id}: file {name} does not start with the id's number" + )); + } else if stem != expected_stem { + problems.push(format!( + "{backend} {id}: file {name} should be named {expected_stem}.{extension}" + )); + } + let cql = match extension { + "sql" => false, + "cql" => true, + _ => { + problems.push(format!( + "{backend} {id}: file {name} is neither .sql nor .cql" + )); + false + } + }; + let (code, comments) = split_comments(text, cql); + for keyword in statement_keywords(&code) { + if TRANSACTION_KEYWORDS.contains(&keyword.as_str()) { + problems.push(format!( + "{backend} {id}: file {name} has a {keyword} statement; the runner owns the \ + transaction" + )); + } + } + if code.to_ascii_uppercase().contains("CATALOG_VERSION") { + problems.push(format!( + "{backend} {id}: file {name} names catalog_version; the runner writes the version" + )); + } + if cql && comments.contains(';') { + problems.push(format!( + "{backend} {id}: file {name} has a ';' inside a comment, which splits a statement" + )); + } +} + +/// The words of SQL or CQL with comments removed, upper-cased, and `;` as +/// its own token. A quoted string or identifier is one `''` token, so the +/// words inside it are not read as keywords. +#[cfg(any(test, feature = "test-util"))] +fn tokens(code: &str) -> Vec { + let mut out = Vec::new(); + let mut chars = code.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\'' | '"' | '`' => { + while let Some(d) = chars.next() { + if d == c { + if chars.peek() == Some(&c) { + chars.next(); + } else { + break; + } + } + } + out.push("''".to_owned()); + } + '[' => { + for d in chars.by_ref() { + if d == ']' { + break; + } + } + out.push("''".to_owned()); + } + // A PostgreSQL dollar-quoted string, `$tag$ ... $tag$` with an + // empty or identifier tag, such as a `DO` block's body. `$1` is a + // parameter, not a quote. + '$' => { + let mut probe = chars.clone(); + let mut tag = String::new(); + let mut opens = false; + for d in probe.by_ref() { + if d == '$' { + opens = true; + break; + } + if !(d.is_ascii_alphanumeric() || d == '_') + || (tag.is_empty() && d.is_ascii_digit()) + { + break; + } + tag.push(d); + } + if opens { + chars = probe; + let close = format!("${tag}$"); + let mut body = String::new(); + for d in chars.by_ref() { + body.push(d); + if body.ends_with(&close) { + break; + } + } + out.push("''".to_owned()); + } + } + ';' => out.push(";".to_owned()), + c if c.is_ascii_alphanumeric() || c == '_' => { + let mut word = c.to_ascii_uppercase().to_string(); + while let Some(&d) = chars.peek() { + if !(d.is_ascii_alphanumeric() || d == '_') { + break; + } + word.push(d.to_ascii_uppercase()); + chars.next(); + } + out.push(word); + } + _ => {} + } + } + out +} + +/// The first keyword of every statement. A `CREATE TRIGGER` statement runs +/// from `CREATE` to the `;` after the `END` that closes its `BEGIN` (a +/// `CASE` inside the body has its own `END`), so the statements of its body +/// are not counted. +#[cfg(any(test, feature = "test-util"))] +fn statement_keywords(code: &str) -> Vec { + let mut keywords = Vec::new(); + let mut statement: Vec = Vec::new(); + let mut in_body = false; + let mut case_depth = 0usize; + for token in tokens(code) { + if in_body { + match token.as_str() { + "CASE" => case_depth += 1, + "END" if case_depth > 0 => case_depth -= 1, + "END" => in_body = false, + _ => {} + } + continue; + } + if token == ";" { + if let Some(first) = statement.first() { + keywords.push(first.clone()); + } + statement.clear(); + continue; + } + if token == "BEGIN" && is_create_trigger(&statement) { + in_body = true; + case_depth = 0; + } + statement.push(token); + } + if let Some(first) = statement.first() { + keywords.push(first.clone()); + } + keywords +} + +#[cfg(any(test, feature = "test-util"))] +fn is_create_trigger(words: &[String]) -> bool { + let mut words = words.iter().map(String::as_str); + words.next() == Some("CREATE") + && match words.next() { + Some("TEMP" | "TEMPORARY") => words.next() == Some("TRIGGER"), + Some("TRIGGER") => true, + _ => false, + } +} + +/// Split SQL or CQL into its text outside comments and the comments' text. +/// String literals (`'...'` with `''` escapes) and quoted identifiers are +/// kept whole in the first part. +#[must_use] +pub fn split_comments(text: &str, cql: bool) -> (String, String) { + let mut code = String::with_capacity(text.len()); + let mut comments = String::new(); + let mut chars = text.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\'' | '"' => { + code.push(c); + while let Some(d) = chars.next() { + code.push(d); + if d == c { + if chars.peek() == Some(&c) { + code.push(c); + chars.next(); + } else { + break; + } + } + } + } + '-' if chars.peek() == Some(&'-') => line_comment(&mut chars, &mut comments, &mut code), + '/' if cql && chars.peek() == Some(&'/') => { + line_comment(&mut chars, &mut comments, &mut code); + } + '/' if chars.peek() == Some(&'*') => { + chars.next(); + let mut last = ' '; + for d in chars.by_ref() { + if last == '*' && d == '/' { + break; + } + comments.push(d); + last = d; + } + code.push(' '); + } + _ => code.push(c), + } + } + (code, comments) +} + +fn line_comment( + chars: &mut std::iter::Peekable>, + comments: &mut String, + code: &mut String, +) { + for d in chars.by_ref() { + if d == '\n' { + code.push('\n'); + break; + } + comments.push(d); + } +} diff --git a/crates/storage/src/migrations/checksum_tests.rs b/crates/storage/src/migrations/checksum_tests.rs new file mode 100644 index 000000000..5921b9fbf --- /dev/null +++ b/crates/storage/src/migrations/checksum_tests.rs @@ -0,0 +1,314 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The checksum test helper on the fake table, with a non-zero released +//! bound. + +use super::tests::{FakeCode, FakeStores, table}; +use super::*; + +fn pins(through: u32) -> Vec { + let mut out = Vec::new(); + for id in MigrationId::ALL + .into_iter() + .filter(|id| id.number() <= through) + { + let m = table(id); + let bodies: Vec<(StoreKind, String, Option<&'static str>)> = match &m { + Migration::NotApplicable(r) => vec![(StoreKind::Catalog, file_checksum(r), None)], + Migration::Applies { bodies, .. } => bodies + .iter() + .map(|b| match &b.body { + Body::File { text, .. } => (b.store, file_checksum(text), None), + Body::Code { source, body } => { + (b.store, file_checksum(source), Some(body.fingerprint())) + } + }) + .collect(), + }; + for (store, sum, fingerprint) in bodies { + out.push(Released { + id, + store, + checksum: Box::leak(sum.into_boxed_str()), + fingerprint, + kind: m.kind(), + convergent: m.convergent(), + rebless: None, + }); + } + } + out +} + +/// The fake table with one id replaced. +fn with( + replaced: MigrationId, + entry: impl Fn() -> Migration, +) -> impl Fn(MigrationId) -> Migration { + move |id| if id == replaced { entry() } else { table(id) } +} + +#[test] +fn the_checksum_helper_accepts_matching_pins_and_prints_the_lines() { + let paste = check_released("fake", table, &pins(3), 3).unwrap(); + assert!(paste.contains("Released { id: MigrationId::Baseline, store: StoreKind::Data,")); + assert!(paste.contains( + "fingerprint: Some(\"0002_resource_tag_condition_keys/fake/1\"), kind: Kind::Breaking" + )); + let bodies: usize = MigrationId::ALL + .into_iter() + .map(|id| table(id).bodies().len().max(1)) + .sum(); + assert_eq!(paste.lines().count(), bodies); + // Nothing released: nothing to compare. + check_released("fake", table, &[], 0).unwrap(); +} + +#[test] +fn the_checksum_helper_refuses_a_changed_released_body_without_a_rebless_note() { + let mut pinned = pins(3); + pinned[0].checksum = "0000"; + let err = check_released("fake", table, &pinned, 3).unwrap_err(); + assert!(err.contains("changed after release"), "{err}"); + assert!(err.contains("Lines for the release PR"), "{err}"); + + // A later entry with the current checksum and a reason is accepted. + let current = pins(3)[0]; + let mut reblessed = pins(3); + reblessed[0].checksum = "0000"; + reblessed.insert( + 1, + Released { + rebless: Some("trailing whitespace"), + ..current + }, + ); + check_released("fake", table, &reblessed, 3).unwrap(); + reblessed[1].rebless = None; + let err = check_released("fake", table, &reblessed, 3).unwrap_err(); + assert!(err.contains("without a rebless reason"), "{err}"); +} + +static CODE_2_RENAMED: FakeCode = FakeCode("0002_resource_tag_condition_keys/fake/2"); +static CODE_2_MISNAMED: FakeCode = FakeCode("0004_index_table_base_key/fake/1"); + +/// Deployed ledgers hold a code body's fingerprint, so a released body may +/// change its source with a rebless entry but never its fingerprint. +#[test] +fn the_checksum_helper_pins_code_body_fingerprints() { + let changed = with(MigrationId::ResourceTagConditionKeys, || { + Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &CODE_2_RENAMED, + "source 2, edited", + )], + } + }); + let mut pinned = pins(3); + let mut reblessed = pinned[2]; + reblessed.checksum = Box::leak(file_checksum("source 2, edited").into_boxed_str()); + reblessed.rebless = Some("a driver upgrade"); + pinned.insert(3, reblessed); + let err = check_released("fake", changed, &pinned, 3).unwrap_err(); + assert!(err.contains("released with fingerprint"), "{err}"); + assert!(err.contains("fake/2"), "{err}"); + + // The same rebless with the fingerprint kept is accepted. + let kept = with(MigrationId::ResourceTagConditionKeys, || { + Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &super::tests::CODE_2, + "source 2, edited", + )], + } + }); + check_released("fake", kept, &pinned, 3).unwrap(); + + // A fingerprint must name its own id. + let misnamed = with(MigrationId::ResourceTagConditionKeys, || { + Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &CODE_2_MISNAMED, + "source 2", + )], + } + }); + let err = check_released("fake", misnamed, &[], 0).unwrap_err(); + assert!( + err.contains("should start with \"0002_resource_tag_condition_keys/\""), + "{err}" + ); +} + +#[test] +fn the_checksum_helper_pins_kind_and_convergent_and_the_bound() { + let mut pinned = pins(3); + pinned[1].kind = Kind::Additive; + let err = check_released("fake", table, &pinned, 3).unwrap_err(); + assert!(err.contains("released as additive"), "{err}"); + + let mut pinned = pins(3); + pinned[3].convergent = false; + assert!(check_released("fake", table, &pinned, 3).is_err()); + + // Released bodies without pins, and pins above the bound. + let err = check_released("fake", table, &pins(1), 3).unwrap_err(); + assert!(err.contains("has no pinned checksum"), "{err}"); + let err = check_released("fake", table, &pins(3), 1).unwrap_err(); + assert!(err.contains("above RELEASED_THROUGH"), "{err}"); + + // A pin for a store the table does not declare. + let mut pinned = pins(3); + pinned.push(Released { + store: StoreKind::Account, + ..pinned[0] + }); + let err = check_released("fake", table, &pinned, 3).unwrap_err(); + assert!( + err.contains("for the account store, which the table does not declare"), + "{err}" + ); + + // An empty NotApplicable reason. + let empty = with(MigrationId::SortKeyDefinitions, || { + Migration::NotApplicable(" ") + }); + let err = check_released("fake", empty, &[], 0).unwrap_err(); + assert!(err.contains("a NotApplicable reason is empty"), "{err}"); +} + +fn file_problems(name: &'static str, text: &'static str) -> String { + let one = with(MigrationId::Baseline, move || Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::file(StoreKind::Catalog, name, text)], + }); + check_released("fake", one, &[], 0) + .err() + .unwrap_or_default() +} + +#[test] +fn the_checksum_helper_checks_file_names_and_contents() { + assert_eq!( + file_problems("0001_baseline.sql", "CREATE TABLE a (b TEXT);"), + "" + ); + assert!(file_problems("0002_baseline.sql", "SELECT 1;").contains("does not start with")); + assert!(file_problems("0001_other.sql", "SELECT 1;").contains("should be named")); + assert!( + file_problems( + "0001_baseline.sql", + "UPDATE settings SET value = '4' WHERE key = 'catalog_version';" + ) + .contains("names catalog_version") + ); + // Words and comments are not statements. + assert_eq!( + file_problems( + "0001_baseline.sql", + "-- BEGIN, COMMIT and catalog_version in a comment\nCREATE TABLE beginning (committed TEXT);" + ), + "" + ); + assert!( + file_problems("0001_baseline.cql", "-- one; two\nCREATE TABLE a (b text);") + .contains("inside a comment") + ); + assert!( + file_problems("0001_baseline.cql", "// one; two\nCREATE TABLE a (b text);") + .contains("inside a comment") + ); + assert_eq!( + file_problems( + "0001_baseline.cql", + "// BEGIN\nCREATE TABLE a (b text, c text);" + ), + "" + ); +} + +/// Every statement that commits, ends or nests in the runner's transaction +/// is refused, wherever it stands, and a trigger body's statements are not +/// statements of the file. +#[test] +fn the_checksum_helper_refuses_transaction_statements_and_allows_triggers() { + for (text, keyword) in [ + ("BEGIN;\nSELECT 1;", "BEGIN"), + ("SELECT 1;\ncommit;", "COMMIT"), + ("SELECT 1;\nEND;\nSELECT 2;", "END"), + ("SELECT 1; ROLLBACK;", "ROLLBACK"), + ("SAVEPOINT a; SELECT 1;", "SAVEPOINT"), + ("SELECT 1; RELEASE a;", "RELEASE"), + ("/* x */ BEGIN TRANSACTION;", "BEGIN"), + ] { + let problems = file_problems("0001_baseline.sql", text); + assert!( + problems.contains(&format!("has a {keyword} statement")), + "{text}: {problems}" + ); + } + assert_eq!( + file_problems( + "0001_baseline.sql", + "CREATE TRIGGER t AFTER INSERT ON a BEGIN \ + UPDATE a SET b = CASE WHEN new.b IS NULL THEN 0 ELSE new.b END; \ + SELECT RAISE(ROLLBACK, 'no'); END;\nCREATE TEMP TRIGGER u AFTER DELETE ON a BEGIN \ + DELETE FROM c; END;" + ), + "" + ); + assert!( + file_problems( + "0001_baseline.sql", + "CREATE TRIGGER t AFTER INSERT ON a BEGIN DELETE FROM c; END;\nCOMMIT;" + ) + .contains("has a COMMIT statement") + ); + // Inside a block comment or a string, nothing is a statement. + assert_eq!( + file_problems( + "0001_baseline.sql", + "/* BEGIN; COMMIT; catalog_version */ INSERT INTO a VALUES ('END; ROLLBACK;');" + ), + "" + ); + // Nor inside a dollar-quoted body, with or without a tag; `$1` is a + // parameter and quotes nothing. + assert_eq!( + file_problems( + "0001_baseline.sql", + "DO $$ BEGIN IF true THEN DELETE FROM a; END IF; END $$;\n\ + DO $body$ BEGIN PERFORM 1; END $body$;\nSELECT $1, 'x';" + ), + "" + ); + assert!( + file_problems("0001_baseline.sql", "DO $$ BEGIN END $$;\nBEGIN;") + .contains("has a BEGIN statement") + ); + assert!( + file_problems("0001_baseline.sql", "SELECT $1;\nCOMMIT;") + .contains("has a COMMIT statement") + ); +} + +#[test] +fn file_checksums_ignore_line_endings() { + assert_eq!(file_checksum("a\r\nb\n"), file_checksum("a\nb\n")); + assert_ne!(file_checksum("a\nb\n"), file_checksum("a\nb")); + assert_eq!( + file_checksum(""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); +} diff --git a/crates/storage/src/migrations/code.rs b/crates/storage/src/migrations/code.rs new file mode 100644 index 000000000..7bf1817cb --- /dev/null +++ b/crates/storage/src/migrations/code.rs @@ -0,0 +1,42 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Code bodies: migrations that DDL cannot express. + +use async_trait::async_trait; + +use crate::management_store::ManagementStore; + +use super::status::MigrationError; + +/// What a code body's `run` returns when it finishes. The notes are recorded +/// in the ledger row and printed by `migrate` and `verify`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Done { + pub notes: Vec, +} + +/// A migration written in Rust. `H` is the backend's handle to every one of +/// its stores, because a body recorded in one store may read another. +#[async_trait] +pub trait CodeBody: Send + Sync { + /// The checksum recorded in the ledger. A constant of the module, named + /// `NNNN_/...` after its id, that changes only with a new + /// migration, so a driver upgrade that edits a released body (with a + /// `rebless:` note) leaves deployed ledgers valid. + fn fingerprint(&self) -> &'static str; + + /// Read-only. Returns the items `run` would refuse to proceed past, each + /// a line for the operator. The runner calls it for every pending code + /// body before the first write of a run and stops on any item. + async fn detect(&self, stores: &H) -> Result, MigrationError>; + + /// Idempotent and safe to stop at any point. + async fn run(&self, stores: &H) -> Result; +} + +/// A backend handle that reaches the catalog through [`ManagementStore`], +/// which the resource-tag rewrite (migration 2) uses on every backend. +pub trait HasManagementStore { + fn management_store(&self) -> &dyn ManagementStore; +} diff --git a/crates/storage/src/migrations/ledger.rs b/crates/storage/src/migrations/ledger.rs new file mode 100644 index 000000000..201bd076d --- /dev/null +++ b/crates/storage/src/migrations/ledger.rs @@ -0,0 +1,239 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Ledger rows, the same columns in every store (`extenddb_migrations`). + +use super::status::MigrationError; +use super::table::Kind; + +/// What happened when the id was recorded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + Applied, + NotApplicable, +} + +impl Outcome { + /// The value recorded in the ledger's `outcome` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Applied => "applied", + Self::NotApplicable => "not_applicable", + } + } + + /// Parse a ledger value. + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "applied" => Some(Self::Applied), + "not_applicable" => Some(Self::NotApplicable), + _ => None, + } + } +} + +/// One row of a store's ledger. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LedgerRow { + pub id: u32, + pub name: String, + pub kind: Kind, + /// SHA-256 of a file body (CRLF normalised to LF), the fingerprint of a + /// code body, or SHA-256 of a `NotApplicable` reason. + pub checksum: String, + pub outcome: Outcome, + /// Notes a code body returned, one per line; empty for a file body. + pub notes: String, + /// Server time, set by the store when the row is written. Empty in a row + /// the runner hands to a backend to write. + pub applied_at: String, + /// The ExtendDB release that wrote the row. + pub applied_by: String, +} + +/// A ledger row as a store returns it, before this binary reads it: `id`, +/// `name`, `kind`, `checksum`, `outcome`, `notes`, `applied_at` and +/// `applied_by`, in that order. `I` is the integer type the store's driver +/// returns for `id`. +pub type LedgerColumns = (I, String, String, String, String, String, String, String); + +impl LedgerRow { + /// Read a row a store returned. `store` names the store in the error, for + /// a backend with more than one. + /// + /// # Errors + /// + /// An `id`, `kind` or `outcome` this binary cannot read. + pub fn from_columns>( + store: Option<&str>, + columns: LedgerColumns, + ) -> Result { + let (id, name, kind, checksum, outcome, notes, applied_at, applied_by) = columns; + let id: i64 = id.into(); + let bad = |what: &str, value: &str| { + MigrationError::Store(match store { + Some(store) => format!( + "extenddb_migrations row {id} in {store} has {what} {value:?}, which this \ + binary cannot read" + ), + None => format!( + "extenddb_migrations row {id} has {what} {value:?}, which this binary \ + cannot read" + ), + }) + }; + Ok(Self { + id: u32::try_from(id).map_err(|_| bad("id", &id.to_string()))?, + name, + kind: Kind::parse(&kind).ok_or_else(|| bad("kind", &kind))?, + checksum, + outcome: Outcome::parse(&outcome).ok_or_else(|| bad("outcome", &outcome))?, + notes, + applied_at, + applied_by, + }) + } +} + +/// What a release before 1.0 left in the catalog's settings, read beside the +/// ledger. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CatalogTraces { + /// The `catalog_version` settings row. A dotted value (`0.0.3`) comes from + /// a release before 1.0; the runner writes an integer. + pub catalog_version: Option, + /// Whether the `resource_tag_migration` row exists. Every released + /// baseline seeded it and migration 1 deletes it, so on a catalog with a + /// ledger it means a release before 1.0 wrote to the catalog. + pub resource_tag_marker: bool, +} + +impl CatalogTraces { + /// The traces in the catalog's `catalog_version` and + /// `resource_tag_migration` settings rows, given as `(key, value)`. + #[must_use] + pub fn from_settings(rows: &[(String, String)]) -> Self { + Self { + catalog_version: rows + .iter() + .find(|(k, _)| k == "catalog_version") + .map(|(_, v)| v.clone()), + resource_tag_marker: rows.iter().any(|(k, _)| k == "resource_tag_migration"), + } + } + + /// The version row holds a dotted pre-1.0 version. + #[must_use] + pub fn dotted_version(&self) -> bool { + self.catalog_version + .as_deref() + .is_some_and(|v| v.contains('.')) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn columns(id: I, kind: &str, outcome: &str) -> LedgerColumns { + ( + id, + "baseline".to_owned(), + kind.to_owned(), + "c".to_owned(), + outcome.to_owned(), + "a note".to_owned(), + "2026-10-29T14:00:00Z".to_owned(), + "1.0.0".to_owned(), + ) + } + + fn refusal(result: Result) -> String { + match result { + Err(MigrationError::Store(message)) => message, + other => panic!("{other:?}"), + } + } + + #[test] + fn a_ledger_row_is_read_from_its_columns() { + let row = LedgerRow::from_columns(Some("extenddb"), columns(4_i32, "breaking", "applied")) + .expect("row"); + assert_eq!( + row, + LedgerRow { + id: 4, + name: "baseline".to_owned(), + kind: Kind::Breaking, + checksum: "c".to_owned(), + outcome: Outcome::Applied, + notes: "a note".to_owned(), + applied_at: "2026-10-29T14:00:00Z".to_owned(), + applied_by: "1.0.0".to_owned(), + } + ); + let row = LedgerRow::from_columns(None, columns(2_i64, "additive", "not_applicable")) + .expect("row"); + assert_eq!( + (row.kind, row.outcome), + (Kind::Additive, Outcome::NotApplicable) + ); + } + + #[test] + fn a_value_this_binary_cannot_read_is_an_error_naming_it() { + assert_eq!( + refusal(LedgerRow::from_columns( + None, + columns(-1_i64, "breaking", "applied") + )), + "extenddb_migrations row -1 has id \"-1\", which this binary cannot read" + ); + assert_eq!( + refusal(LedgerRow::from_columns( + None, + columns(4_294_967_296_i64, "breaking", "applied") + )), + "extenddb_migrations row 4294967296 has id \"4294967296\", which this binary cannot \ + read" + ); + assert_eq!( + refusal(LedgerRow::from_columns( + Some("extenddb_catalog"), + columns(2_i32, "sideways", "applied") + )), + "extenddb_migrations row 2 in extenddb_catalog has kind \"sideways\", which this \ + binary cannot read" + ); + assert_eq!( + refusal(LedgerRow::from_columns( + None, + columns(3_i64, "breaking", "maybe") + )), + "extenddb_migrations row 3 has outcome \"maybe\", which this binary cannot read" + ); + } + + #[test] + fn the_traces_are_read_from_their_two_settings_rows() { + let rows = |pairs: &[(&str, &str)]| -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect() + }; + assert_eq!(CatalogTraces::from_settings(&[]), CatalogTraces::default()); + let traces = CatalogTraces::from_settings(&rows(&[ + ("other", "x"), + ("catalog_version", "0.0.3"), + ("resource_tag_migration", "done"), + ])); + assert_eq!(traces.catalog_version.as_deref(), Some("0.0.3")); + assert!(traces.resource_tag_marker && traces.dotted_version()); + let traces = CatalogTraces::from_settings(&rows(&[("catalog_version", "4")])); + assert_eq!(traces.catalog_version.as_deref(), Some("4")); + assert!(!traces.resource_tag_marker && !traces.dotted_version()); + } +} diff --git a/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs b/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs new file mode 100644 index 000000000..861c39c8f --- /dev/null +++ b/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs @@ -0,0 +1,295 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 2, `resource_tag_condition_keys`: rewrite stored IAM policies +//! from `dynamodb:ResourceTag/` to `aws:ResourceTag/`. +//! +//! The live service evaluates resource tags only as `aws:ResourceTag/` +//! and leaves `dynamodb:ResourceTag/` absent, so ExtendDB no longer +//! resolves the `dynamodb:` form. Before the evaluator change both forms +//! resolved from the same table tags, so replacing the first with the second +//! keeps every policy's decision. +//! +//! The pure transform, [`rewrite_policy_document`], defines this migration and +//! is frozen here: this file's checksum is pinned once the migration is +//! released, and the file is excluded from rustfmt so formatting cannot +//! change it. The body reaches the stored policies through +//! `ManagementStore` on every backend, with the driver in +//! `crate::policy_migration`. + +use serde_json::Value; + +use async_trait::async_trait; + +use super::code::{CodeBody, Done, HasManagementStore}; +use super::status::{CatalogStatus, MigrationError}; +use crate::management_store::OpError; +use crate::policy_migration::{MigrationReport, attention_items, failure_message, migrate_resource_tag_condition_keys}; + +/// The condition-key prefix the live service does not populate. +const DYNAMODB_PREFIX: &str = "dynamodb:ResourceTag/"; +/// The condition-key prefix the live service evaluates resource tags as. +const AWS_PREFIX: &str = "aws:ResourceTag/"; + +/// ASCII-case-insensitive prefix strip, returning the untouched remainder. +fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix) + .then(|| &s[prefix.len()..]) +} + +/// Outcome of rewriting one policy document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RewriteOutcome { + /// The rewritten document. Equal to the input when nothing changed or a + /// collision was found. + pub document: Value, + /// Number of rewrites applied (condition-key renames plus policy-variable + /// prefix rewrites). Zero when nothing changed or on a collision. + pub rewrites: usize, + /// A `Condition` operator block held both `aws:ResourceTag/k` and a + /// `dynamodb:ResourceTag/k` that would rewrite onto the same JSON key. + /// The two cannot share one JSON object, so the document is left unchanged + /// and the caller reports it for manual attention. + pub collision: bool, +} + +impl RewriteOutcome { + /// Whether the document was rewritten (changed and no collision). + #[must_use] + pub fn changed(&self) -> bool { + self.rewrites > 0 && !self.collision + } +} + +/// Rewrite every `dynamodb:ResourceTag/` condition key and +/// `${dynamodb:ResourceTag/...}` policy variable (any ASCII case) in a policy +/// document to the `aws:ResourceTag/` form, inside `Condition` blocks only. +/// +/// Idempotent: a document with no `dynamodb:ResourceTag` reference is returned +/// unchanged with `rewrites == 0`. On a collision the input is returned +/// unchanged with `collision == true`. +#[must_use] +pub fn rewrite_policy_document(document: &Value) -> RewriteOutcome { + let mut doc = document.clone(); + let mut rewrites = 0usize; + let mut collision = false; + + // Statement may be a single object or an array of objects. + match doc.get_mut("Statement") { + Some(Value::Array(statements)) => { + for stmt in statements.iter_mut() { + rewrite_statement(stmt, &mut rewrites, &mut collision); + } + } + Some(stmt @ Value::Object(_)) => { + rewrite_statement(stmt, &mut rewrites, &mut collision); + } + _ => {} + } + + if collision { + // Leave the stored policy exactly as it was. + return RewriteOutcome { + document: document.clone(), + rewrites: 0, + collision: true, + }; + } + + RewriteOutcome { + document: if rewrites > 0 { doc } else { document.clone() }, + rewrites, + collision: false, + } +} + +/// Rewrite the `Condition` block of one statement in place. +fn rewrite_statement(stmt: &mut Value, rewrites: &mut usize, collision: &mut bool) { + let Some(condition) = stmt.get_mut("Condition").and_then(Value::as_object_mut) else { + return; + }; + for (_operator, block) in condition.iter_mut() { + let Some(map) = block.as_object_mut() else { + continue; + }; + rewrite_operator_block(map, rewrites, collision); + } +} + +/// Rewrite one operator block: its condition keys (object keys) and the policy +/// variables in its values. +fn rewrite_operator_block( + block: &mut serde_json::Map, + rewrites: &mut usize, + collision: &mut bool, +) { + // First rewrite the variables inside the values, keyed by the current + // (pre-rename) condition key, so a value rewrite is not lost by the rename. + for value in block.values_mut() { + rewrite_variables_in_json(value, rewrites); + } + + // Collect the condition-key renames. + let mut renames: Vec<(String, String)> = Vec::new(); + for key in block.keys() { + if let Some(rest) = strip_prefix_ci(key, DYNAMODB_PREFIX) { + renames.push((key.clone(), format!("{AWS_PREFIX}{rest}"))); + } + } + if renames.is_empty() { + return; + } + // A rename onto a key the block already has (an aws:ResourceTag key, or + // an earlier rename) is a duplicate when the two values are equal: both + // prefixes resolve from the same table tags, so the two conditions are + // the same condition and the dynamodb: one is dropped. Different values + // cannot share one JSON key, so the block is a collision. + let mut moves: Vec<(String, String)> = Vec::new(); + let mut drops: Vec = Vec::new(); + for (old_key, new_key) in &renames { + let value = &block[old_key]; + let existing = block.get(new_key).or_else(|| { + moves + .iter() + .find(|(_, n)| n == new_key) + .map(|(o, _)| &block[o]) + }); + match existing { + Some(v) if v == value => drops.push(old_key.clone()), + Some(_) => { + *collision = true; + return; + } + None => moves.push((old_key.clone(), new_key.clone())), + } + } + for old_key in drops { + block.remove(&old_key); + *rewrites += 1; + } + for (old_key, new_key) in moves { + if let Some(value) = block.remove(&old_key) { + block.insert(new_key, value); + *rewrites += 1; + } + } +} + +/// Rewrite `${dynamodb:ResourceTag/...}` policy variables inside a value, which +/// is a string or an array of strings. +fn rewrite_variables_in_json(value: &mut Value, rewrites: &mut usize) { + match value { + Value::String(s) => { + if let Some(new) = rewrite_variables_in_str(s, rewrites) { + *s = new; + } + } + Value::Array(items) => { + for item in items.iter_mut() { + rewrite_variables_in_json(item, rewrites); + } + } + _ => {} + } +} + +/// Rewrite the prefix of every `${dynamodb:ResourceTag/...}` variable in `s`, +/// leaving all other text (including a literal `dynamodb:ResourceTag/...` not +/// wrapped in `${...}`) untouched. Returns `None` when nothing changed. +fn rewrite_variables_in_str(s: &str, rewrites: &mut usize) -> Option { + if !s.contains("${") { + return None; + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + let mut changed = false; + while let Some(open) = rest.find("${") { + out.push_str(&rest[..open + 2]); + let after_open = &rest[open + 2..]; + let Some(close) = after_open.find('}') else { + // No closing brace: copy the remainder and stop. + out.push_str(after_open); + rest = ""; + break; + }; + let inner = &after_open[..close]; + if let Some(tag_rest) = strip_prefix_ci(inner, DYNAMODB_PREFIX) { + out.push_str(AWS_PREFIX); + out.push_str(tag_rest); + *rewrites += 1; + changed = true; + } else { + out.push_str(inner); + } + out.push('}'); + rest = &after_open[close + 1..]; + } + out.push_str(rest); + changed.then_some(out) +} + +/// This file, whose SHA-256 each backend's checksum test pins. +pub const SOURCE: &str = include_str!("m0002_resource_tag_condition_keys.rs"); + +/// Recorded as this migration's checksum in every ledger. Changes only with a +/// new migration. +pub const FINGERPRINT: &str = "0002_resource_tag_condition_keys/1"; + +/// Migration 2's body on every backend. +pub struct ResourceTagConditionKeys; + +#[async_trait] +impl CodeBody for ResourceTagConditionKeys { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// The policies whose two resource-tag forms name the same key with + /// different values in one operator block, and the policies whose stored + /// documents cannot be decoded. Neither can be rewritten, and leaving a + /// `dynamodb:` form in place can turn a Deny off. + async fn detect(&self, stores: &H) -> Result, MigrationError> { + let report = migrate_resource_tag_condition_keys(stores.management_store(), false).await; + refuse_lookup_errors(&report)?; + Ok(attention_items(&report)) + } + + async fn run(&self, stores: &H) -> Result { + let report = migrate_resource_tag_condition_keys(stores.management_store(), true).await; + refuse_lookup_errors(&report)?; + let items = attention_items(&report); + if !items.is_empty() { + return Err(MigrationError::Status(CatalogStatus::NeedsAttention { items })); + } + let notes = report + .rewritten + .iter() + .map(|r| { + format!( + "rewrote account {} {} policy {} ({} key(s))", + r.account_id, r.principal, r.policy_name, r.rewrites + ) + }) + .collect(); + Ok(Done { notes }) + } +} + +/// A pass that could not read or write a policy must not be recorded: the +/// next run finishes it. The error names the failed lookups and writes and +/// carries the first failure's database error code, so the runner runs the +/// body again after a lock timeout. +fn refuse_lookup_errors(report: &MigrationReport) -> Result<(), MigrationError> { + if report.errors == 0 { + return Ok(()); + } + let message = failure_message(report); + Err(match &report.first_error { + Some(OpError::Database { sqlstate, .. }) => MigrationError::Database { + message, + sqlstate: sqlstate.clone(), + }, + _ => MigrationError::Store(message), + }) +} diff --git a/crates/storage/src/migrations/mod.rs b/crates/storage/src/migrations/mod.rs new file mode 100644 index 000000000..d84fb6fb7 --- /dev/null +++ b/crates/storage/src/migrations/mod.rs @@ -0,0 +1,64 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Catalog migrations: one numbered list for every backend, a ledger in every +//! store, and the runner that applies the list. +//! +//! - [`MigrationId`] numbers every migration once, for all backends. +//! - Each backend declares, per id, the body it runs in each of its stores or +//! why the id does not apply to it ([`Migration`]), in an exhaustive `match` +//! with no wildcard arm, so a backend that lacks an entry does not compile. +//! - [`Runner`] applies the pending bodies in order, catalog store first, and +//! commits each body with its ledger row. The catalog version is read from +//! the ledgers, never from a constant or a file. +//! - [`Runner::status`] computes the [`CatalogStatus`] that `serve`, +//! `migrate` and `verify` act on, and [`CatalogStatus::exit_code`] maps it +//! to the process exit status. +//! +//! A backend implements [`MigrationBackend`]: reading and writing its ledgers, +//! running one body inside one store's commit, and whatever its store needs +//! before the first write of a run (the SQLite copy). + +pub mod advice; +mod backend; +mod catalog; +mod checksum; +mod code; +mod ledger; +mod record; +mod registry; +mod runner; +mod status; +mod table; + +// Frozen once released: neither rustfmt nor a later clippy lint may require +// an edit that changes its checksum. +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +pub mod m0002_resource_tag_condition_keys; + +pub use backend::{MigrationBackend, Step, StoreRef}; +pub use catalog::{ + AdminRole, CatalogMigrations, HeldLock, Listed, RunKind, StatusNote, Target, before_lock, + binary_catalog_label, forget_locked, installed, policy_rewrite_lines, policy_rewrite_pending, + run_locked, +}; +#[cfg(any(test, feature = "test-util"))] +pub use checksum::check_released; +pub use checksum::{Released, file_checksum, split_comments}; +pub use code::{CodeBody, Done, HasManagementStore}; +pub use ledger::{CatalogTraces, LedgerColumns, LedgerRow, Outcome}; +pub use record::{LedgerWrites, in_transaction, record_rows, sql_ledger_ddl}; +pub use registry::{MigrationId, RELEASED_THROUGH}; +pub use runner::Runner; +pub use status::{AppliedStep, CatalogStatus, MigrationError, Pending, RunReport, exit}; +#[cfg(any(test, feature = "test-util"))] +pub use table::refuse_additive; +pub use table::{Body, Kind, Migration, StoreBody, StoreKind}; + +#[cfg(test)] +mod checksum_tests; +#[cfg(test)] +mod reconverge_tests; +#[cfg(test)] +mod tests; diff --git a/crates/storage/src/migrations/reconverge_tests.rs b/crates/storage/src/migrations/reconverge_tests.rs new file mode 100644 index 000000000..cc57b1b50 --- /dev/null +++ b/crates/storage/src/migrations/reconverge_tests.rs @@ -0,0 +1,307 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `forget`, `reconverge` and reblessed released bodies on the fake backend. + +use super::tests::{CATALOG_FILE, CODE_2, Fake, LATEST, current, runner, steps_after}; +use super::*; + +#[tokio::test] +async fn forget_refuses_released_ids() { + let fake = Fake::new(); + current(&fake).await; + let all: Vec = (1..=LATEST).collect(); + let err = runner(&fake) + .with_released_through(2) + .forget(2) + .await + .unwrap_err(); + assert!(matches!(err, MigrationError::Refused(_)), "{err:?}"); + assert_eq!(fake.rows("cat"), all); + runner(&fake) + .with_released_through(2) + .forget(3) + .await + .unwrap(); + let without_3: Vec = all.iter().copied().filter(|id| *id != 3).collect(); + assert_eq!(fake.rows("cat"), without_3); + assert_eq!(fake.version_row().as_deref(), Some("2")); + runner(&fake).migrate().await.unwrap(); + assert_eq!(fake.rows("cat"), all); +} + +#[tokio::test] +async fn forget_refuses_an_id_it_cannot_forget() { + let fake = Fake::new(); + runner(&fake) + .migrate_through(MigrationId::ResourceTagConditionKeys) + .await + .unwrap(); + let before = fake.rows("cat"); + // An id this binary does not carry. + let err = runner(&fake).forget(LATEST + 3).await.unwrap_err(); + assert_eq!(err.exit_code(), exit::ERROR); + assert!(err.to_string().contains("carries no migration"), "{err}"); + // An id no store has a row for. + let err = runner(&fake).forget(LATEST).await.unwrap_err(); + assert_eq!(err.exit_code(), exit::ERROR); + assert!(err.to_string().contains("nothing to forget"), "{err}"); + // 0 is no migration id, whatever the released bound. + for released in [0, 2] { + let err = runner(&fake) + .with_released_through(released) + .forget(0) + .await + .unwrap_err(); + assert_eq!(err.exit_code(), exit::ERROR); + assert!(err.to_string().contains("carries no migration 0"), "{err}"); + } + assert_eq!(fake.rows("cat"), before); +} + +/// No migration may be declared additive before CI runs the previous +/// release against a catalog this binary migrated. +#[test] +fn an_additive_declaration_is_refused() { + let table = |additive: MigrationId| { + move |id: MigrationId| -> Migration<()> { + Migration::Applies { + kind: if id == additive { + Kind::Additive + } else { + Kind::Breaking + }, + convergent: true, + bodies: Vec::new(), + } + } + }; + let err = refuse_additive("fake", table(MigrationId::LATEST)).unwrap_err(); + assert!( + err.contains(&format!( + "migration {} is declared Kind::Additive on fake", + MigrationId::LATEST + )), + "{err}" + ); + assert!(err.contains("step 4"), "{err}"); + // A migration that does not apply has no kind to declare. + assert_eq!( + refuse_additive("fake", |_| Migration::<()>::NotApplicable("no store")), + Ok(()) + ); +} + +/// After its row for migration 1 is forgotten the catalog has a ledger at +/// migration 0, which is not a catalog without a ledger. +#[tokio::test] +async fn a_ledger_without_migration_1_is_at_migration_0() { + let fake = Fake::new(); + current(&fake).await; + runner(&fake) + .with_released_through(0) + .forget(1) + .await + .unwrap(); + let CatalogStatus::Behind { version, .. } = runner(&fake).status().await.unwrap() else { + panic!("expected Behind"); + }; + assert_eq!(version, Some(0)); + assert_eq!(fake.version_row().as_deref(), Some("0")); + let report = runner(&fake).migrate().await.unwrap(); + assert_eq!((report.from, report.to), (Some(0), Some(LATEST))); +} + +/// Id 4 is not convergent on the fake. While no store has a row for it, a +/// reconverge applies it like any pending id. Once it is recorded, a +/// reconverge leaves it as it is and names it in the report's notes, runs +/// the convergent ids again, and repairs the catalog. +#[tokio::test] +async fn reconverge_skips_a_recorded_id_that_is_not_convergent() { + let fake = Fake::new(); + runner(&fake) + .migrate_through(MigrationId::SortKeyDefinitions) + .await + .unwrap(); + let an_older_release_writes = || { + fake.with_state(|s| { + s.traces.catalog_version = Some("0.0.3".into()); + s.traces.resource_tag_marker = true; + }); + }; + an_older_release_writes(); + let report = runner(&fake).reconverge().await.unwrap(); + assert_eq!( + report.applied.len(), + steps_after(0), + "every id ran: {report:?}" + ); + assert!( + report.notes.iter().all(|n| !n.starts_with("Skipped")), + "{report:?}" + ); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); + + an_older_release_writes(); + let four = MigrationId::IndexTableBaseKey; + let steps_of_four = steps_after(four.number() - 1) - steps_after(four.number()); + let report = runner(&fake).reconverge().await.unwrap(); + assert!( + report.applied.iter().all(|s| s.id != four.number()), + "{report:?}" + ); + assert_eq!(report.applied.len(), steps_after(0) - steps_of_four); + assert!( + report.notes.contains(&format!( + "Skipped {four}: it is not convergent on fake, so it does not run again over \ + data written since it was applied" + )), + "{report:?}" + ); + assert_eq!(fake.version_row(), Some(LATEST.to_string())); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); +} + +/// A store without a row for an id that is not convergent gets one, as in +/// `migrate`; a store that has a row is left as it is. +#[tokio::test] +async fn reconverge_applies_an_id_that_is_not_convergent_where_it_has_no_row() { + let fake = Fake::new(); + current(&fake).await; + let four = MigrationId::IndexTableBaseKey.number(); + fake.with_state(|s| { + if let Some(rows) = s.ledgers.get_mut("data-b") { + rows.retain(|r| r.id != four); + } + s.traces.resource_tag_marker = true; + }); + let report = runner(&fake).reconverge().await.unwrap(); + let ran: Vec<&str> = report + .applied + .iter() + .filter(|s| s.id == four) + .map(|s| s.store_name.as_str()) + .collect(); + assert_eq!(ran, ["data-b"], "{report:?}"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with(&format!("Skipped {four} "))), + "{report:?}" + ); + assert!(fake.rows("data-b").contains(&four)); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); +} + +/// A reconverge reruns the bodies and writes the version row again, and +/// leaves every ledger row as it was, so the first application's notes and +/// `applied_at` survive. The bodies' new notes are in the report. +#[tokio::test] +async fn reconverge_leaves_the_ledger_rows_as_they_are() { + let fake = Fake::new(); + runner(&fake) + .migrate_through(MigrationId::SortKeyDefinitions) + .await + .unwrap(); + fake.with_state(|s| { + for rows in s.ledgers.values_mut() { + for row in rows { + row.applied_at = "2026-01-01T00:00:00.000Z".into(); + row.applied_by = "1.0.0".into(); + } + } + s.traces.catalog_version = Some("0.0.3".into()); + s.traces.resource_tag_marker = true; + }); + let before: Vec> = ["cat", "data-a", "data-b"] + .iter() + .map(|s| fake.ledger(s)) + .collect(); + let report = runner(&fake).reconverge().await.unwrap(); + for (store, rows) in ["cat", "data-a", "data-b"].iter().zip(&before) { + let after = fake.ledger(store); + for row in rows { + assert!(after.contains(row), "{store}: {row:?} changed: {after:?}"); + } + } + let two = report.applied.iter().find(|s| s.id == 2).expect("id 2 ran"); + assert_eq!(two.notes, [format!("{} ran", CODE_2.0)]); + assert_eq!(fake.version_row(), Some(LATEST.to_string())); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); +} + +#[tokio::test] +async fn reconverge_refuses_an_id_this_binary_does_not_carry() { + let fake = Fake::new(); + runner(&fake) + .migrate_through(MigrationId::SortKeyDefinitions) + .await + .unwrap(); + fake.add_catalog_row(LATEST + 1, Kind::Additive); + fake.with_state(|s| s.traces.resource_tag_marker = true); + let err = runner(&fake).reconverge().await.unwrap_err(); + let MigrationError::Refused(message) = &err else { + panic!("{err:?}"); + }; + assert!(message.contains("does not carry"), "{message}"); +} + +/// A released file body changed later with a rebless entry: a new catalog +/// records the first released checksum, so every release reads it, and the +/// runtime accepts the released checksum and the current one, and nothing +/// else. +#[tokio::test] +async fn a_reblessed_file_body_records_its_first_release_and_accepts_both() { + let current_checksum = file_checksum(CATALOG_FILE); + let released: &'static [Released] = Box::leak(Box::new([ + Released { + id: MigrationId::Baseline, + store: StoreKind::Catalog, + checksum: "released-1", + fingerprint: None, + kind: Kind::Breaking, + convergent: true, + rebless: None, + }, + Released { + id: MigrationId::Baseline, + store: StoreKind::Catalog, + checksum: Box::leak(current_checksum.clone().into_boxed_str()), + fingerprint: None, + kind: Kind::Breaking, + convergent: true, + rebless: Some("a comment reworded"), + }, + ])); + let fake = Fake::released(released); + current(&fake).await; + let recorded = |fake: &Fake| fake.ledger("cat")[0].checksum.clone(); + assert_eq!(recorded(&fake), "released-1"); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); + let set = |checksum: &str| { + fake.with_state(|s| s.ledgers.get_mut("cat").unwrap()[0].checksum = checksum.into()); + }; + set(¤t_checksum); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); + set("another"); + assert_eq!(runner(&fake).status().await.unwrap().label(), "Changed"); +} diff --git a/crates/storage/src/migrations/record.rs b/crates/storage/src/migrations/record.rs new file mode 100644 index 000000000..8618d717d --- /dev/null +++ b/crates/storage/src/migrations/record.rs @@ -0,0 +1,228 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How a backend records a body: the ledger row and the version row, written +//! in the same order on every backend, and the transaction they commit in. +//! +//! A backend implements [`LedgerWrites`] for a store with its own statements +//! and driver calls, and calls [`record_rows`] after the body. A store with +//! transactions that are statements on one connection (the SQL stores) runs +//! the body and the rows through [`in_transaction`]; a store without them +//! (Cassandra), or a body that runs outside one (PostgreSQL migration 4), +//! records the rows after the body. + +use std::future::Future; + +use async_trait::async_trait; + +use super::ledger::LedgerRow; + +/// The writes that record a body in one store, in the store's own statements. +#[async_trait] +pub trait LedgerWrites: Sync { + /// The error the store's driver calls return. + type Error: Send; + + /// Create the ledger if it does not exist. The runner creates it with + /// the first row it writes, so no migration body can change it. + async fn create_ledger(&self) -> Result<(), Self::Error>; + + /// Insert `row`, whose notes are set. + async fn insert_row(&self, row: &LedgerRow) -> Result<(), Self::Error>; + + /// Set the catalog's `catalog_version` settings row to `version`. + async fn write_version(&self, version: u32) -> Result<(), Self::Error>; +} + +/// Record a body that ran: `row` with `notes` one per line, after creating +/// the ledger, then the version, each when given. +/// +/// # Errors +/// +/// The first write that fails. +pub async fn record_rows( + store: &W, + row: Option, + notes: &[String], + catalog_version: Option, +) -> Result<(), W::Error> { + if let Some(mut row) = row { + row.notes = notes.join("\n"); + store.create_ledger().await?; + store.insert_row(&row).await?; + } + if let Some(version) = catalog_version { + store.write_version(version).await?; + } + Ok(()) +} + +/// The statement that creates the ledger in a SQL store. `applied_at` is +/// that column's type and default, the one part SQL dialects write +/// differently. +#[must_use] +pub fn sql_ledger_ddl(applied_at: &str) -> String { + format!( + "CREATE TABLE IF NOT EXISTS extenddb_migrations (id INTEGER PRIMARY KEY, name TEXT NOT \ + NULL, kind TEXT NOT NULL, checksum TEXT NOT NULL, outcome TEXT NOT NULL, notes TEXT NOT \ + NULL DEFAULT '', applied_at {applied_at}, applied_by TEXT NOT NULL)" + ) +} + +/// Run `work` in a transaction: `begin`, `work`, then `commit`. When `work` +/// or `commit` fails, `rollback` runs and the first error is returned. When +/// `begin` fails, its error is returned and nothing else runs. Each future +/// runs only when awaited here, so the caller builds all four up front. +/// +/// # Errors +/// +/// The error of `begin`, `work` or `commit`. +pub async fn in_transaction( + begin: impl Future>, + work: impl Future>, + commit: impl Future>, + rollback: impl Future, +) -> Result { + begin.await?; + let result = match work.await { + Ok(value) => commit.await.map(|()| value), + Err(e) => Err(e), + }; + if result.is_err() { + rollback.await; + } + result +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + /// The calls a fake store received, in order. + #[derive(Default)] + struct Calls(Mutex>); + + impl Calls { + fn push(&self, call: impl Into) { + self.0.lock().unwrap().push(call.into()); + } + + fn take(&self) -> Vec { + std::mem::take(&mut self.0.lock().unwrap()) + } + } + + #[async_trait] + impl LedgerWrites for Calls { + type Error = String; + + async fn create_ledger(&self) -> Result<(), String> { + self.push("create"); + Ok(()) + } + + async fn insert_row(&self, row: &LedgerRow) -> Result<(), String> { + self.push(format!("insert {} {:?}", row.id, row.notes)); + Ok(()) + } + + async fn write_version(&self, version: u32) -> Result<(), String> { + self.push(format!("version {version}")); + Ok(()) + } + } + + fn row() -> LedgerRow { + LedgerRow { + id: 2, + name: "resource_tag_condition_keys".to_owned(), + kind: super::super::Kind::Breaking, + checksum: "c".to_owned(), + outcome: super::super::Outcome::Applied, + notes: String::new(), + applied_at: String::new(), + applied_by: "test".to_owned(), + } + } + + /// SQLite stores the statement's text in `sqlite_master`, and the schema + /// files record it, so the text is pinned. + #[test] + fn the_sql_ledger_statement_is_the_one_the_stores_record() { + assert_eq!( + sql_ledger_ddl("TIMESTAMPTZ NOT NULL DEFAULT now()"), + "CREATE TABLE IF NOT EXISTS extenddb_migrations (id INTEGER PRIMARY KEY, name TEXT \ + NOT NULL, kind TEXT NOT NULL, checksum TEXT NOT NULL, outcome TEXT NOT NULL, notes \ + TEXT NOT NULL DEFAULT '', applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), applied_by \ + TEXT NOT NULL)" + ); + } + + #[tokio::test] + async fn the_rows_are_written_in_order_with_the_notes() { + let calls = Calls::default(); + let notes = ["one".to_owned(), "two".to_owned()]; + record_rows(&calls, Some(row()), ¬es, Some(2)) + .await + .unwrap(); + assert_eq!( + calls.take(), + ["create", "insert 2 \"one\\ntwo\"", "version 2"] + ); + record_rows(&calls, None, ¬es, Some(3)).await.unwrap(); + assert_eq!(calls.take(), ["version 3"]); + record_rows(&calls, Some(row()), &[], None).await.unwrap(); + assert_eq!(calls.take(), ["create", "insert 2 \"\""]); + } + + #[tokio::test] + async fn a_failed_body_or_commit_rolls_back_and_a_failed_begin_runs_nothing() { + let calls = Calls::default(); + let step = |name: &'static str, ok: bool| { + let calls = &calls; + async move { + calls.push(name); + if ok { Ok(()) } else { Err(name.to_owned()) } + } + }; + let rollback = || async { calls.push("rollback") }; + let done = in_transaction( + step("begin", true), + step("work", true), + step("commit", true), + rollback(), + ) + .await; + assert_eq!(done, Ok(())); + assert_eq!(calls.take(), ["begin", "work", "commit"]); + let failed = in_transaction( + step("begin", true), + step("work", false), + step("commit", true), + rollback(), + ) + .await; + assert_eq!(failed, Err("work".to_owned())); + assert_eq!(calls.take(), ["begin", "work", "rollback"]); + let failed = in_transaction( + step("begin", true), + step("work", true), + step("commit", false), + rollback(), + ) + .await; + assert_eq!(failed, Err("commit".to_owned())); + assert_eq!(calls.take(), ["begin", "work", "commit", "rollback"]); + let failed = in_transaction( + step("begin", false), + step("work", true), + step("commit", true), + rollback(), + ) + .await; + assert_eq!(failed, Err("begin".to_owned())); + assert_eq!(calls.take(), ["begin"]); + } +} diff --git a/crates/storage/src/migrations/registry.rs b/crates/storage/src/migrations/registry.rs new file mode 100644 index 000000000..99d581339 --- /dev/null +++ b/crates/storage/src/migrations/registry.rs @@ -0,0 +1,130 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The migrations this binary carries, numbered once for every backend. +//! +//! The list below is the only place an id is added: it generates the enum, +//! [`MigrationId::ALL`], the names and the descriptions, and +//! [`MigrationId::LATEST`] is the last entry of `ALL`. Every backend's table +//! then fails to compile until it has an arm for the new id. + +macro_rules! migration_ids { + ($( + $(#[$meta:meta])* + $variant:ident = $number:literal, $name:literal, $description:literal; + )+) => { + /// A migration's number, the same on every backend. The discriminant + /// is the id written to every ledger and never changes once released. + #[repr(u32)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub enum MigrationId { + $( $(#[$meta])* $variant = $number, )+ + } + + impl MigrationId { + /// Every id this binary carries, in order. + pub const ALL: [Self; [$($number),+].len()] = [$(Self::$variant),+]; + + /// The name recorded beside the number, and used in file names. + #[must_use] + pub const fn name(self) -> &'static str { + match self { + $( Self::$variant => $name, )+ + } + } + + /// One line for `migrate --list`. + #[must_use] + pub const fn description(self) -> &'static str { + match self { + $( Self::$variant => $description, )+ + } + } + } + }; +} + +migration_ids! { + /// Creates the v1.0 schema on an empty store and brings every released + /// catalog to it. + Baseline = 1, "baseline", + "create the v1.0 catalog schema, or bring a released catalog to it"; + /// Rewrites stored policies from `dynamodb:ResourceTag/` to + /// `aws:ResourceTag/`. + ResourceTagConditionKeys = 2, "resource_tag_condition_keys", + "rewrite dynamodb:ResourceTag condition keys in stored policies to aws:ResourceTag"; + /// Restores sort key attribute definitions that the pre-fix `UpdateTable` + /// deleted (#259). + SortKeyDefinitions = 3, "sort_key_definitions", + "restore sort key attribute definitions an UpdateTable before the #259 fix deleted"; + /// Gives every index table its base key index. + IndexTableBaseKey = 4, "index_table_base_key", + "give every secondary index table its base key index"; +} + +/// Highest id that has shipped in a release. Bodies at or below it are frozen: +/// each backend pins their checksums, kinds and convergent flags, and +/// `forget` refuses them. The v1.0 release PR raises it to 4. +pub const RELEASED_THROUGH: u32 = 0; + +impl MigrationId { + /// The highest id this binary carries. + pub const LATEST: Self = Self::ALL[Self::ALL.len() - 1]; + + /// The number recorded in the ledgers. + #[must_use] + pub const fn number(self) -> u32 { + self as u32 + } + + /// The id with this number, if this binary carries it. + #[must_use] + pub fn from_number(number: u32) -> Option { + Self::ALL.into_iter().find(|id| id.number() == number) + } +} + +impl std::fmt::Display for MigrationId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.number(), self.name()) + } +} + +#[cfg(test)] +mod tests { + use super::{MigrationId, RELEASED_THROUGH}; + + #[test] + fn ids_are_numbered_from_one_without_gaps() { + for (i, id) in MigrationId::ALL.iter().enumerate() { + assert_eq!(id.number() as usize, i + 1, "{id:?}"); + assert_eq!(MigrationId::from_number(id.number()), Some(*id)); + } + assert_eq!(MigrationId::ALL.last(), Some(&MigrationId::LATEST)); + assert_eq!(MigrationId::from_number(0), None); + assert_eq!( + MigrationId::from_number(MigrationId::LATEST.number() + 1), + None + ); + } + + #[test] + fn names_are_unique_snake_case() { + let mut names: Vec<&str> = MigrationId::ALL.iter().map(|id| id.name()).collect(); + for name in &names { + assert!( + name.chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'), + "{name}" + ); + } + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), MigrationId::ALL.len()); + } + + #[test] + fn the_released_bound_is_within_the_list() { + assert!(RELEASED_THROUGH == 0 || MigrationId::from_number(RELEASED_THROUGH).is_some()); + } +} diff --git a/crates/storage/src/migrations/runner.rs b/crates/storage/src/migrations/runner.rs new file mode 100644 index 000000000..a9c431c95 --- /dev/null +++ b/crates/storage/src/migrations/runner.rs @@ -0,0 +1,615 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The runner: applies a backend's pending bodies in order and computes the +//! catalog status from the ledgers. + +use super::backend::{MigrationBackend, Step, StoreRef}; +use super::checksum::{accepted_checksums, record_checksum}; +use super::code::Done; +use super::ledger::{CatalogTraces, LedgerRow, Outcome}; +use super::registry::{MigrationId, RELEASED_THROUGH}; +use super::status::{AppliedStep, CatalogStatus, MigrationError, Pending, RunReport}; +use super::table::{Body, Kind, Migration, StoreKind}; + +/// Every store's ledger, catalog first, and the catalog's pre-1.0 traces. +#[derive(Debug, Clone)] +struct Snapshot { + ledgers: Vec<(StoreRef, Vec)>, + traces: CatalogTraces, +} + +impl Snapshot { + fn catalog(&self) -> &StoreRef { + &self.ledgers[0].0 + } + + fn catalog_rows(&self) -> &[LedgerRow] { + &self.ledgers[0].1 + } + + fn has_row(&self, store: &StoreRef, id: u32) -> bool { + self.ledgers + .iter() + .any(|(s, rows)| s == store && rows.iter().any(|r| r.id == id)) + } + + fn instances(&self, kind: StoreKind) -> impl Iterator { + self.ledgers + .iter() + .map(|(s, _)| s) + .filter(move |s| s.kind == kind) + } + + fn put_row(&mut self, store: &StoreRef, row: LedgerRow) { + if let Some((_, rows)) = self.ledgers.iter_mut().find(|(s, _)| s == store) { + rows.retain(|r| r.id != row.id); + rows.push(row); + } + } + + /// Whether the store is a catalog: it has a ledger row, or a + /// `catalog_version` row a release before 1.0 wrote. + fn initialized(&self) -> bool { + !self.catalog_rows().is_empty() || self.traces.catalog_version.is_some() + } +} + +/// A step the run will apply. +struct Planned { + id: MigrationId, + store: StoreRef, + kind: Kind, + /// The store already has a row for the id (a reconverge reruns it). + recorded: bool, +} + +/// Applies a backend's migrations. See the module documentation of +/// [`crate::migrations`]. +pub struct Runner<'a, B: MigrationBackend> { + backend: &'a B, + applied_by: String, + released_through: u32, +} + +impl<'a, B: MigrationBackend> Runner<'a, B> { + /// A runner that records `applied_by` (the release) in every row. + pub fn new(backend: &'a B, applied_by: impl Into) -> Self { + Self { + backend, + applied_by: applied_by.into(), + released_through: RELEASED_THROUGH, + } + } + + /// Use another released bound than [`RELEASED_THROUGH`]. + #[cfg(test)] + #[must_use] + pub(crate) fn with_released_through(mut self, released_through: u32) -> Self { + self.released_through = released_through; + self + } + + /// The catalog status `serve`, `migrate` and `verify` act on. Read-only: + /// runs `detect` for pending code bodies and writes nothing. + /// + /// # Errors + /// + /// A ledger or a detect could not be read. + pub async fn status(&self) -> Result { + let snap = self.read().await?; + self.status_of(&snap).await + } + + /// Apply every pending migration this binary carries. + /// + /// # Errors + /// + /// The status forbids the run (`Changed`, `WrittenByOlderRelease`, + /// `AheadIncompatible`, `NeedsAttention`), or a body failed. A failed body + /// leaves the earlier ones committed with their rows; the next run + /// resumes at the first store without a row. + pub async fn migrate(&self) -> Result { + self.run(MigrationId::LATEST, false).await + } + + /// Apply pending migrations up to and including `id`, for tests that need + /// a catalog at an intermediate migration. + /// + /// # Errors + /// + /// As [`Runner::migrate`]. + #[cfg(any(test, feature = "test-util"))] + pub async fn migrate_through(&self, id: MigrationId) -> Result { + self.run(id, false).await + } + + /// Run the convergent migrations again, repairing a catalog a release + /// before 1.0 wrote to (`WrittenByOlderRelease`): each convergent body + /// reruns, the `catalog_version` row is written again, and the ledger + /// rows already there stay as they are, with their notes and + /// `applied_at`. A recorded migration that is not convergent does not + /// run again, and a line of the report's notes names it. A store without + /// a row for an id gets one, as in [`Runner::migrate`]. + /// + /// # Errors + /// + /// The catalog ledger holds an id that this binary does not carry, or as + /// [`Runner::migrate`]. + pub async fn reconverge(&self) -> Result { + self.run(MigrationId::LATEST, true).await + } + + /// Delete the ledger rows of an unreleased id, so a developer's database + /// whose body changed during review runs it again. + /// + /// # Errors + /// + /// This binary does not carry `id` (0 is no migration id), `id` is at or + /// below the released bound, no store has a row for it, or a store + /// failed. + pub async fn forget(&self, id: u32) -> Result<(), MigrationError> { + if MigrationId::from_number(id).is_none() { + return Err(MigrationError::Refused(format!( + "this binary carries no migration {id}; it carries migrations 1 to {}", + MigrationId::LATEST.number() + ))); + } + if id <= self.released_through { + return Err(MigrationError::Refused(format!( + "migration {id} has been released; only migrations above {} can be forgotten", + self.released_through + ))); + } + let mut snap = self.read().await?; + if !snap + .ledgers + .iter() + .any(|(_, rows)| rows.iter().any(|r| r.id == id)) + { + return Err(MigrationError::Refused(format!( + "no ledger has a row for migration {id}, so there is nothing to forget" + ))); + } + for (store, rows) in &mut snap.ledgers { + if rows.iter().any(|r| r.id == id) { + self.backend.delete_ledger_row(store, id).await?; + rows.retain(|r| r.id != id); + } + } + if let Some(version) = self.version_of(&snap) { + self.backend.write_catalog_version(version).await?; + } + Ok(()) + } + + /// The catalog store's ledger rows, in id order. Read-only. + /// + /// # Errors + /// + /// A ledger could not be read. + pub async fn catalog_ledger(&self) -> Result, MigrationError> { + let snap = self.read().await?; + let mut rows = snap.catalog_rows().to_vec(); + rows.sort_by_key(|r| r.id); + Ok(rows) + } + + async fn read(&self) -> Result { + let mut kinds = vec![StoreKind::Catalog]; + for id in MigrationId::ALL { + kinds.extend(self.backend.migration(id).store_kinds()); + } + kinds.sort_unstable(); + kinds.dedup(); + let mut ledgers = Vec::new(); + for kind in kinds { + let mut names = self.backend.store_names(kind).await?; + names.sort_unstable(); + if kind == StoreKind::Catalog && names.len() != 1 { + return Err(MigrationError::Store(format!( + "{} reported {} catalog stores; expected one", + self.backend.backend_name(), + names.len() + ))); + } + for name in names { + let store = StoreRef { kind, name }; + let rows = self.backend.read_ledger(&store).await?; + ledgers.push((store, rows)); + } + } + let traces = self.backend.read_traces().await?; + Ok(Snapshot { ledgers, traces }) + } + + /// The highest id such that it and every lower id have a catalog row and + /// a row in every store their bodies touch: `None` for a catalog store + /// with no ledger rows at all, `Some(0)` for a ledger without migration 1. + fn version_of(&self, snap: &Snapshot) -> Option { + if snap.catalog_rows().is_empty() { + return None; + } + let highest = snap.catalog_rows().iter().map(|r| r.id).max().unwrap_or(0); + let mut version = Some(0); + for number in 1..=highest { + if !snap.has_row(snap.catalog(), number) { + break; + } + if let Some(id) = MigrationId::from_number(number) { + let complete = self + .backend + .migration(id) + .store_kinds() + .into_iter() + .all(|kind| { + snap.instances(kind) + .collect::>() + .into_iter() + .all(|store| snap.has_row(store, number)) + }); + if !complete { + break; + } + } + version = Some(number); + } + version + } + + /// Every store without a row for an id up to `target`, in order: by id, + /// then catalog first. With `rerun`, also every store with a row for a + /// convergent id. + fn planned(&self, snap: &Snapshot, target: MigrationId, rerun: bool) -> Vec { + let mut steps = Vec::new(); + for id in MigrationId::ALL.into_iter().filter(|id| *id <= target) { + let migration = self.backend.migration(id); + let rerun = rerun && migration.convergent(); + for kind in migration.store_kinds() { + for store in snap.instances(kind) { + let recorded = snap.has_row(store, id.number()); + if rerun || !recorded { + steps.push(Planned { + id, + store: store.clone(), + kind: migration.kind(), + recorded, + }); + } + } + } + } + steps + } + + fn changed(&self, snap: &Snapshot) -> Option { + for (store, rows) in &snap.ledgers { + for row in rows { + let Some(id) = MigrationId::from_number(row.id) else { + continue; + }; + let accepted = accepted_checksums( + self.backend.released(), + id, + &self.backend.migration(id), + store.kind, + ); + if !accepted.contains(&row.checksum) { + return Some(CatalogStatus::Changed { + id: row.id, + store_name: store.name.clone(), + recorded: row.checksum.clone(), + expected: accepted.first().cloned().unwrap_or_else(|| { + format!("no body for the {} store", store.kind.as_str()) + }), + }); + } + } + } + None + } + + fn older_release_trace(&self, snap: &Snapshot) -> Option { + if !snap.has_row(snap.catalog(), MigrationId::Baseline.number()) { + return None; + } + if snap.traces.dotted_version() { + return Some(format!( + "catalog_version is {}", + snap.traces.catalog_version.as_deref().unwrap_or_default() + )); + } + snap.traces + .resource_tag_marker + .then(|| "the resource_tag_migration settings row exists".to_owned()) + } + + fn unknown(&self, snap: &Snapshot) -> Vec<(u32, String, Kind)> { + snap.catalog_rows() + .iter() + .filter(|r| r.id > MigrationId::LATEST.number()) + .map(|r| (r.id, r.name.clone(), r.kind)) + .collect() + } + + /// The first status that forbids a run, in the order the status is + /// decided: `Changed`, then `WrittenByOlderRelease` (which a reconverge + /// repairs), then `AheadIncompatible`. + fn forbidding(&self, snap: &Snapshot, reconverge: bool) -> Option { + if let Some(changed) = self.changed(snap) { + return Some(changed); + } + if !reconverge && let Some(trace) = self.older_release_trace(snap) { + return Some(CatalogStatus::WrittenByOlderRelease { trace }); + } + let unknown = self.unknown(snap); + unknown + .iter() + .any(|(_, _, kind)| *kind == Kind::Breaking) + .then(|| CatalogStatus::AheadIncompatible { + unknown: unknown + .into_iter() + .map(|(id, name, _)| (id, name)) + .collect(), + }) + } + + /// Run `detect` for every pending code body, once per id and kind of + /// store, and collect the items. + async fn detect(&self, steps: &[Planned]) -> Result, MigrationError> { + let mut seen: Vec<(MigrationId, StoreKind)> = Vec::new(); + let mut items = Vec::new(); + for step in steps { + if seen.contains(&(step.id, step.store.kind)) { + continue; + } + seen.push((step.id, step.store.kind)); + let migration = self.backend.migration(step.id); + let Some(store_body) = migration + .bodies() + .iter() + .find(|b| b.store == step.store.kind) + else { + continue; + }; + if let Body::Code { body, .. } = &store_body.body { + for item in body.detect(self.backend.stores()).await? { + items.push(format!("migration {}: {item}", step.id)); + } + } + } + Ok(items) + } + + /// The status, in this order: `Uninitialized`, then `Changed`, + /// `WrittenByOlderRelease`, `AheadIncompatible`, `NeedsAttention`, + /// `Behind`, `AheadCompatible` and `Current`. + async fn status_of(&self, snap: &Snapshot) -> Result { + if !snap.initialized() { + return Ok(CatalogStatus::Uninitialized); + } + if let Some(status) = self.forbidding(snap, false) { + return Ok(status); + } + let steps = self.planned(snap, MigrationId::LATEST, false); + let items = self.detect(&steps).await?; + if !items.is_empty() { + return Ok(CatalogStatus::NeedsAttention { items }); + } + let version = self.version_of(snap); + if !steps.is_empty() { + let pending = steps + .iter() + .map(|s| Pending { + id: s.id.number(), + name: s.id.name(), + store: s.store.kind, + store_name: s.store.name.clone(), + kind: s.kind, + }) + .collect(); + return Ok(CatalogStatus::Behind { version, pending }); + } + let unknown = self.unknown(snap); + if !unknown.is_empty() { + return Ok(CatalogStatus::AheadCompatible { + version: version.unwrap_or_default(), + unknown: unknown + .into_iter() + .map(|(id, name, _)| (id, name)) + .collect(), + }); + } + Ok(CatalogStatus::Current { + version: version.unwrap_or_default(), + }) + } + + async fn run( + &self, + target: MigrationId, + reconverge: bool, + ) -> Result { + let mut snap = self.read().await?; + let from = self.version_of(&snap); + if let Some(status) = self.forbidding(&snap, reconverge) { + return Err(MigrationError::Status(status)); + } + let skipped = if reconverge { + self.refuse_unknown(&snap)?; + self.skipped(&snap, target) + } else { + Vec::new() + }; + let steps = self.planned(&snap, target, reconverge); + if steps.is_empty() { + return Ok(RunReport { + from, + to: from, + notes: skipped, + ..RunReport::default() + }); + } + // A catalog store with no tables at all holds nothing a body could + // find, and the tables `detect` reads do not exist in it. + let items = if self.backend.catalog_is_empty().await? { + Vec::new() + } else { + self.detect(&steps).await? + }; + if !items.is_empty() { + return Err(MigrationError::Status(CatalogStatus::NeedsAttention { + items, + })); + } + let mut notes = self + .backend + .before_first_write(from, target.number()) + .await?; + let prepared = !notes.is_empty(); + notes.extend(skipped); + let mut applied = Vec::new(); + let result = self.apply_steps(&mut snap, &steps, &mut applied).await; + let report = RunReport { + from, + to: self.version_of(&snap), + applied, + notes, + }; + match result { + Ok(()) => Ok(report), + // Nothing was prepared or committed: the error says it all. + Err(error) if report.applied.is_empty() && !prepared => Err(error), + Err(error) => Err(MigrationError::Stopped { + error: Box::new(error), + report, + }), + } + } + + /// Apply `steps` in order, writing the version row after an id whose + /// last body ran outside the catalog store, and record each step applied. + async fn apply_steps( + &self, + snap: &mut Snapshot, + steps: &[Planned], + applied: &mut Vec, + ) -> Result<(), MigrationError> { + for (i, step) in steps.iter().enumerate() { + let done = self.apply_step(snap, step).await?; + applied.push(AppliedStep { + id: step.id.number(), + name: step.id.name(), + store_name: step.store.name.clone(), + notes: done.notes, + }); + let last_of_id = steps.get(i + 1).is_none_or(|next| next.id != step.id); + if last_of_id + && step.store.kind != StoreKind::Catalog + && let Some(version) = self.version_of(snap) + { + self.backend.write_catalog_version(version).await?; + } + } + Ok(()) + } + + /// Refuse a reconverge of a catalog whose ledger has an id this binary + /// does not carry, whose body it cannot run again. + fn refuse_unknown(&self, snap: &Snapshot) -> Result<(), MigrationError> { + match snap + .catalog_rows() + .iter() + .find(|row| MigrationId::from_number(row.id).is_none()) + { + Some(row) => Err(MigrationError::Refused(format!( + "the catalog has migration {} {}, which this binary does not carry; \ + reconverge with the binary that applied it", + row.id, row.name + ))), + None => Ok(()), + } + } + + /// A line for each id up to `target` that is not convergent and that a + /// store has a row for: a reconverge leaves those stores as they are. + fn skipped(&self, snap: &Snapshot, target: MigrationId) -> Vec { + MigrationId::ALL + .into_iter() + .filter(|id| *id <= target) + .filter(|id| { + let migration = self.backend.migration(*id); + !migration.convergent() + && migration.store_kinds().into_iter().any(|kind| { + snap.instances(kind) + .any(|store| snap.has_row(store, id.number())) + }) + }) + .map(|id| { + format!( + "Skipped {id}: it is not convergent on {}, so it does not run again \ + over data written since it was applied", + self.backend.backend_name() + ) + }) + .collect() + } + + async fn apply_step( + &self, + snap: &mut Snapshot, + step: &Planned, + ) -> Result { + let migration = self.backend.migration(step.id); + let checksum = record_checksum( + self.backend.released(), + step.id, + &migration, + step.store.kind, + ) + .ok_or_else(|| { + MigrationError::Store(format!( + "migration {} declares no body for the {} store", + step.id, + step.store.kind.as_str() + )) + })?; + let (outcome, body) = match &migration { + Migration::NotApplicable(_) => (Outcome::NotApplicable, None), + Migration::Applies { bodies, .. } => ( + Outcome::Applied, + bodies + .iter() + .find(|b| b.store == step.store.kind) + .map(|b| &b.body), + ), + }; + let row = (!step.recorded).then(|| LedgerRow { + id: step.id.number(), + name: step.id.name().to_owned(), + kind: migration.kind(), + checksum, + outcome, + notes: String::new(), + applied_at: String::new(), + applied_by: self.applied_by.clone(), + }); + let mut after = snap.clone(); + if let Some(row) = &row { + after.put_row(&step.store, row.clone()); + } + let catalog_version = (step.store.kind == StoreKind::Catalog) + .then(|| self.version_of(&after).unwrap_or_default()); + let action = match body { + None => Step::Record, + Some(Body::File { name, text }) => Step::File { name, text }, + Some(Body::Code { body, .. }) => Step::Code(*body), + }; + let done = self + .backend + .apply(&step.store, step.id, action, row, catalog_version) + .await?; + *snap = after; + Ok(done) + } +} diff --git a/crates/storage/src/migrations/status.rs b/crates/storage/src/migrations/status.rs new file mode 100644 index 000000000..c462c9c79 --- /dev/null +++ b/crates/storage/src/migrations/status.rs @@ -0,0 +1,284 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog status, the runner's errors, what a run did, and the process +//! exit codes. + +use super::table::{Kind, StoreKind}; + +/// One step a run applied. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppliedStep { + pub id: u32, + pub name: &'static str, + pub store_name: String, + /// The body's notes from this run. A reconverge leaves the row's notes + /// from the first application as they are, so these are printed only. + pub notes: Vec, +} + +/// What a run did. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RunReport { + /// The catalog version before the run; `None` without a ledger. + pub from: Option, + /// The catalog version after it. + pub to: Option, + pub applied: Vec, + /// Lines printed before the steps: the backend's preparation (the + /// SQLite copy's path), then one for each recorded migration a + /// reconverge skipped because it is not convergent. + pub notes: Vec, +} + +/// Process exit codes shared by `serve`, `migrate`, `migrate --yes` and +/// `verify`. +pub mod exit { + /// Current, or applied. `AheadCompatible` also exits 0, with a warning. + pub const OK: u8 = 0; + /// An error, and `Uninitialized` in a production build. + pub const ERROR: u8 = 1; + /// Migrations are pending that `migrate --yes` will apply. + pub const PENDING: u8 = 3; + /// The catalog has a breaking migration this binary does not know. + pub const AHEAD_INCOMPATIBLE: u8 = 4; + /// A person must act: `NeedsAttention`, `Changed`, `WrittenByOlderRelease`. + pub const ATTENTION: u8 = 5; + /// The lock or a running server outlasted the timeout; retry. + pub const LOCKED: u8 = 6; +} + +/// A migration this binary carries that a store has no ledger row for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pending { + pub id: u32, + pub name: &'static str, + pub store: StoreKind, + /// The store instance (the SQLite file, a keyspace name). + pub store_name: String, + pub kind: Kind, +} + +/// The status `serve`, `migrate` and `verify` act on. When several apply, it +/// is the first in this order: `Changed`, `WrittenByOlderRelease`, +/// `AheadIncompatible`, `NeedsAttention`, `Behind`, `AheadCompatible`, +/// `Current`. `Uninitialized` is decided before all of them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CatalogStatus { + /// Every migration this binary carries is recorded in every store it + /// touches. + Current { version: u32 }, + /// Migrations this binary carries are missing. `version` is `None` for a + /// catalog with no ledger (a release before 1.0, behind at migration 1). + Behind { + version: Option, + pending: Vec, + }, + /// The catalog ledger has ids above this binary's, all additive. + AheadCompatible { + version: u32, + unknown: Vec<(u32, String)>, + }, + /// The catalog ledger has an id above this binary's that is breaking. + AheadIncompatible { unknown: Vec<(u32, String)> }, + /// A pending code body's `detect` returned items. + NeedsAttention { items: Vec }, + /// A recorded checksum differs from this binary's body. + Changed { + id: u32, + store_name: String, + recorded: String, + expected: String, + }, + /// The catalog ledger has a row for migration 1 and a release before 1.0 + /// wrote to the catalog afterwards. + WrittenByOlderRelease { trace: String }, + /// Not a catalog: no ledger row and no `catalog_version` row. + Uninitialized, +} + +impl CatalogStatus { + /// The exit code for this status as read, for `serve`, `migrate` without + /// `--yes` and `verify` in a production build. `migrate --yes` exits 0 + /// once it has applied a `Behind` catalog. + #[must_use] + pub fn exit_code(&self) -> u8 { + match self { + Self::Current { .. } | Self::AheadCompatible { .. } => exit::OK, + Self::Behind { .. } => exit::PENDING, + Self::AheadIncompatible { .. } => exit::AHEAD_INCOMPATIBLE, + Self::NeedsAttention { .. } + | Self::Changed { .. } + | Self::WrittenByOlderRelease { .. } => exit::ATTENTION, + Self::Uninitialized => exit::ERROR, + } + } + + /// A short name for tests. + #[cfg(any(test, feature = "test-util"))] + #[must_use] + pub fn label(&self) -> &'static str { + match self { + Self::Current { .. } => "Current", + Self::Behind { .. } => "Behind", + Self::AheadCompatible { .. } => "AheadCompatible", + Self::AheadIncompatible { .. } => "AheadIncompatible", + Self::NeedsAttention { .. } => "NeedsAttention", + Self::Changed { .. } => "Changed", + Self::WrittenByOlderRelease { .. } => "WrittenByOlderRelease", + Self::Uninitialized => "Uninitialized", + } + } +} + +impl std::fmt::Display for CatalogStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Current { version } => write!(f, "the catalog is at migration {version}"), + Self::Behind { version, pending } => { + let at = version.map_or_else( + || "has no migration ledger (a release before 1.0)".to_owned(), + |v| format!("is at migration {v}"), + ); + let list: Vec = pending + .iter() + .map(|p| { + format!( + "{} {} ({}, {})", + p.id, + p.name, + p.store.as_str(), + p.kind.as_str() + ) + }) + .collect(); + write!(f, "the catalog {at}; pending: {}", list.join(", ")) + } + Self::AheadCompatible { version, unknown } => write!( + f, + "the catalog is at migration {version}, ahead of this binary by additive \ + migrations only: {}", + unknown_list(unknown) + ), + Self::AheadIncompatible { unknown } => write!( + f, + "the catalog has migrations this binary does not know, at least one of them \ + breaking: {}", + unknown_list(unknown) + ), + Self::NeedsAttention { items } => write!( + f, + "a pending migration needs attention before it can run: {}", + items.join("; ") + ), + Self::Changed { + id, + store_name, + recorded, + expected, + } => write!( + f, + "migration {id} in {store_name} was recorded with checksum {recorded}, and this \ + binary's body has {expected}" + ), + Self::WrittenByOlderRelease { trace } => write!( + f, + "a release before 1.0 wrote to this catalog after it was migrated ({trace}); \ + run 'extenddb migrate --yes --reconverge'" + ), + Self::Uninitialized => write!(f, "the catalog is not initialized"), + } + } +} + +fn unknown_list(unknown: &[(u32, String)]) -> String { + unknown + .iter() + .map(|(id, name)| format!("{id} {name}")) + .collect::>() + .join(", ") +} + +/// Why the runner stopped. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum MigrationError { + /// A connection or statement failed, or a body returned an error. + #[error("{0}")] + Store(String), + /// A statement failed with a database error code (PostgreSQL's + /// SQLSTATE). The runner reads the code, never the message, to run a + /// body again after a lock timeout or to name the owners of tables the + /// role may not alter. + #[error("{message}")] + Database { message: String, sqlstate: String }, + /// The catalog's status forbids the requested run. + #[error("{0}")] + Status(CatalogStatus), + /// `forget`, `reconverge` or a backend precondition refused the request. + #[error("{0}")] + Refused(String), + /// The lock was not free within the timeout. + #[error("{0}")] + Locked(String), + /// Sessions an earlier run left behind still hold what this run needs, + /// past the timeout, and do not end by themselves: the message names + /// them and the statement that ends them. Exits as [`Self::Locked`]. + #[error("{0}")] + Blocked(String), + /// A run stopped on `error` after its preparation: `report` holds what + /// it did first, the steps it committed and the backend's lines (the + /// SQLite copy's path). + #[error("{error}")] + Stopped { + error: Box, + report: RunReport, + }, +} + +impl MigrationError { + /// The process exit code for this error. + #[must_use] + pub fn exit_code(&self) -> u8 { + match self { + Self::Store(_) | Self::Database { .. } | Self::Refused(_) => exit::ERROR, + Self::Status(status) => status.exit_code(), + Self::Locked(_) | Self::Blocked(_) => exit::LOCKED, + Self::Stopped { error, .. } => error.exit_code(), + } + } + + /// Shorthand for a store error with context. + pub fn store(context: impl std::fmt::Display, err: impl std::fmt::Display) -> Self { + Self::Store(format!("{context}: {err}")) + } + + /// A store error with context that keeps the database's error code when + /// the driver reported one. Each backend's helper that maps its driver's + /// error calls this. + pub fn coded( + context: impl std::fmt::Display, + err: impl std::fmt::Display, + sqlstate: Option, + ) -> Self { + let message = format!("{context}: {err}"); + match sqlstate { + Some(sqlstate) => Self::Database { message, sqlstate }, + None => Self::Store(message), + } + } + + /// The database error code of the statement that failed, when the error + /// carries one. + #[must_use] + pub fn sqlstate(&self) -> Option<&str> { + match self { + Self::Database { sqlstate, .. } => Some(sqlstate), + Self::Stopped { error, .. } => error.sqlstate(), + Self::Store(_) + | Self::Status(_) + | Self::Refused(_) + | Self::Locked(_) + | Self::Blocked(_) => None, + } + } +} diff --git a/crates/storage/src/migrations/table.rs b/crates/storage/src/migrations/table.rs new file mode 100644 index 000000000..2d3cbb521 --- /dev/null +++ b/crates/storage/src/migrations/table.rs @@ -0,0 +1,217 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The types a backend declares its migration table with. + +use super::code::CodeBody; + +/// Whether the previous release reads and writes correctly against a catalog +/// with the migration applied. An older binary serves a catalog whose +/// unknown ids are all additive and refuses one with a breaking id. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// The previous release serves the result correctly. + Additive, + /// The previous release must not serve the result. + Breaking, +} + +impl Kind { + /// The value recorded in the ledger's `kind` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Additive => "additive", + Self::Breaking => "breaking", + } + } + + /// Parse a ledger value. + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "additive" => Some(Self::Additive), + "breaking" => Some(Self::Breaking), + _ => None, + } + } +} + +/// A kind of store a backend keeps. The runner applies an id's bodies in this +/// order, so the catalog store always comes first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum StoreKind { + /// The catalog database, keyspace or file. Its ledger has a row for every + /// id and is the deployment's record. + Catalog, + /// The data database (PostgreSQL, MongoDB). + Data, + /// One account keyspace (Cassandra): the body runs in each of them. + Account, +} + +impl StoreKind { + /// Lower-case name for messages and the checksum lines. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Catalog => "catalog", + Self::Data => "data", + Self::Account => "account", + } + } +} + +/// What the runner runs in one store. +pub enum Body { + /// SQL or CQL embedded with `include_str!`. `name` is the file name; its + /// leading number must equal the id. + File { + name: &'static str, + text: &'static str, + }, + /// A Rust body. `source` is its module's source, embedded with + /// `include_str!`, whose SHA-256 the checksum test pins; the ledger + /// records the body's fingerprint instead. + Code { + body: &'static dyn CodeBody, + source: &'static str, + }, +} + +/// One body and the kind of store it runs in. +pub struct StoreBody { + pub store: StoreKind, + pub body: Body, +} + +impl StoreBody { + /// A file body. + #[must_use] + pub fn file(store: StoreKind, name: &'static str, text: &'static str) -> Self { + Self { + store, + body: Body::File { name, text }, + } + } + + /// A code body. + #[must_use] + pub fn code(store: StoreKind, body: &'static dyn CodeBody, source: &'static str) -> Self { + Self { + store, + body: Body::Code { body, source }, + } + } +} + +/// A migration file of the crate that invokes the macro, named once. +/// +/// - `file_body!(store, "dir", "NNNN_.sql")` is the [`StoreBody`] that +/// executes `dir/NNNN_.sql` in `store`; the checksum rules read the +/// file name. +/// - `file_body!("dir", "NNNN_.sql")` is the file's text, for a code +/// body that executes a file and pins it together with its own source. +/// +/// `dir` starts at the invoking crate's root (`CARGO_MANIFEST_DIR`), so the +/// path is the same from every module of that crate. +#[macro_export] +macro_rules! file_body { + ($dir:literal, $name:literal) => { + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $dir, "/", $name)) + }; + ($store:expr, $dir:literal, $name:literal) => { + $crate::migrations::StoreBody::file($store, $name, $crate::file_body!($dir, $name)) + }; +} + +/// One backend's entry for one id. +pub enum Migration { + /// The migration changes this backend. + Applies { + kind: Kind, + /// Safe to run again on a store at any later id, over data that + /// clients wrote through the API since it was applied. A body that + /// rewrites such data is not convergent: migration 2 would rewrite a + /// policy written after it and change that policy's decision. + /// `--reconverge` runs the convergent migrations again and skips a + /// recorded one that is not. + convergent: bool, + /// One body per kind of store, catalog first. + bodies: Vec>, + }, + /// The change has no effect on this backend. Always additive and + /// convergent; recorded in the catalog ledger as `not_applicable`. + NotApplicable(&'static str), +} + +impl Migration { + /// The kind declared for this backend. + #[must_use] + pub fn kind(&self) -> Kind { + match self { + Self::Applies { kind, .. } => *kind, + Self::NotApplicable(_) => Kind::Additive, + } + } + + /// Whether every body is safe to run again. + #[must_use] + pub fn convergent(&self) -> bool { + match self { + Self::Applies { convergent, .. } => *convergent, + Self::NotApplicable(_) => true, + } + } + + /// The bodies, empty for `NotApplicable`. + #[must_use] + pub fn bodies(&self) -> &[StoreBody] { + match self { + Self::Applies { bodies, .. } => bodies, + Self::NotApplicable(_) => &[], + } + } + + /// The kinds of store this migration records rows in, catalog first. The + /// catalog ledger has a row for every id, including one whose bodies run + /// only in other stores, so an older binary reading the catalog sees + /// every id and its kind. + #[must_use] + pub fn store_kinds(&self) -> Vec { + let mut kinds: Vec = std::iter::once(StoreKind::Catalog) + .chain(self.bodies().iter().map(|b| b.store)) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + kinds + } +} + +/// Refuse a table with an entry declared [`Kind::Additive`]. A binary serves +/// a catalog that is ahead of it by additive migrations only, so an +/// additive declaration is safe only once CI runs the previous release +/// against a catalog this binary migrated (CONTRIBUTING.md, "Adding a +/// catalog migration", step 4). Until that job exists, each backend's tests +/// call this; the PR that adds the job deletes the calls. +/// +/// # Errors +/// +/// The first id declared additive, with what to do. +#[cfg(any(test, feature = "test-util"))] +pub fn refuse_additive( + backend: &str, + table: impl Fn(super::MigrationId) -> Migration, +) -> Result<(), String> { + for id in super::MigrationId::ALL { + let migration = table(id); + if matches!(migration, Migration::Applies { .. }) && migration.kind() == Kind::Additive { + return Err(format!( + "migration {id} is declared Kind::Additive on {backend}, and no CI job runs the \ + previous release against a catalog this binary migrated yet; declare it \ + Kind::Breaking (CONTRIBUTING.md, \"Adding a catalog migration\", step 4)" + )); + } + } + Ok(()) +} diff --git a/crates/storage/src/migrations/tests.rs b/crates/storage/src/migrations/tests.rs new file mode 100644 index 000000000..5f27a2ba2 --- /dev/null +++ b/crates/storage/src/migrations/tests.rs @@ -0,0 +1,560 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The runner, the status and the checksum rules on a fake backend with a +//! catalog store and two data stores. Expected sequences are built from +//! [`MigrationId::ALL`], so a new id needs no edit here beyond the fake +//! table's wildcard arm. + +use std::collections::BTreeMap; +use std::sync::Mutex; + +use async_trait::async_trait; + +use super::*; + +pub(super) const CATALOG_FILE: &str = "CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT);\n"; +const DATA_FILE: &str = "CREATE TABLE items (pk TEXT PRIMARY KEY);\n"; + +/// What code bodies see: the items `detect` reports, and a log. +#[derive(Default)] +pub(super) struct FakeStores { + attention: Mutex>, + runs: Mutex>, +} + +pub(super) struct FakeCode(pub(super) &'static str); + +#[async_trait] +impl CodeBody for FakeCode { + fn fingerprint(&self) -> &'static str { + self.0 + } + async fn detect(&self, stores: &FakeStores) -> Result, MigrationError> { + Ok(stores.attention.lock().unwrap().clone()) + } + async fn run(&self, stores: &FakeStores) -> Result { + stores.runs.lock().unwrap().push(self.0.to_owned()); + Ok(Done { + notes: vec![format!("{} ran", self.0)], + }) + } +} + +pub(super) static CODE_2: FakeCode = FakeCode("0002_resource_tag_condition_keys/fake/1"); +static CODE_4: FakeCode = FakeCode("0004_index_table_base_key/fake/1"); + +/// 1: a file in the catalog and in each data store. 2: code in the catalog. +/// 3: not applicable. 4: code in each data store, not convergent. Every +/// later id: not applicable. +#[allow(unreachable_patterns)] +pub(super) fn table(id: MigrationId) -> Migration { + match id { + MigrationId::Baseline => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![ + StoreBody::file(StoreKind::Catalog, "0001_baseline.sql", CATALOG_FILE), + StoreBody::file(StoreKind::Data, "0001_baseline.sql", DATA_FILE), + ], + }, + MigrationId::ResourceTagConditionKeys => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code(StoreKind::Catalog, &CODE_2, "source 2")], + }, + MigrationId::SortKeyDefinitions => Migration::NotApplicable("the fake has no sort keys"), + MigrationId::IndexTableBaseKey => Migration::Applies { + kind: Kind::Breaking, + convergent: false, + bodies: vec![StoreBody::code(StoreKind::Data, &CODE_4, "source 4")], + }, + _ => Migration::NotApplicable("a later id the fake does not model"), + } +} + +/// The fake's store instances of each kind, in the order the runner applies +/// them. +fn instances(kind: StoreKind) -> &'static [&'static str] { + match kind { + StoreKind::Catalog => &["cat"], + StoreKind::Data => &["data-a", "data-b"], + StoreKind::Account => &[], + } +} + +/// The writes a run makes for `id`: each store's body, catalog first, then +/// the version row when the id's last body ran outside the catalog. +fn writes_for(id: MigrationId) -> Vec { + let kinds = table(id).store_kinds(); + let mut out: Vec = kinds + .iter() + .flat_map(|kind| instances(*kind)) + .map(|store| format!("{} {store}", id.number())) + .collect(); + if kinds.last() != Some(&StoreKind::Catalog) { + out.push(format!("version {}", id.number())); + } + out +} + +/// The ids whose bodies touch `store`. +fn ids_in(store: &str) -> Vec { + MigrationId::ALL + .into_iter() + .filter(|id| { + table(*id) + .store_kinds() + .iter() + .any(|kind| instances(*kind).contains(&store)) + }) + .map(MigrationId::number) + .collect() +} + +/// The number of steps a run applies for the ids after `after`. +pub(super) fn steps_after(after: u32) -> usize { + MigrationId::ALL + .into_iter() + .filter(|id| id.number() > after) + .flat_map(|id| table(id).store_kinds()) + .map(|kind| instances(kind).len()) + .sum() +} + +pub(super) const LATEST: u32 = MigrationId::LATEST.number(); + +#[derive(Default)] +pub(super) struct State { + pub(super) ledgers: BTreeMap>, + pub(super) traces: CatalogTraces, + /// Whether the catalog store has tables. + tables: bool, + writes: Vec, + fail_apply: Option<(u32, String)>, + fail_read: bool, +} + +pub(super) struct Fake { + stores: FakeStores, + state: Mutex, + released: &'static [Released], +} + +impl Fake { + pub(super) fn new() -> Self { + Self::released(&[]) + } + pub(super) fn released(released: &'static [Released]) -> Self { + Self { + stores: FakeStores::default(), + state: Mutex::new(State::default()), + released, + } + } + fn writes(&self) -> Vec { + self.state.lock().unwrap().writes.clone() + } + pub(super) fn rows(&self, store: &str) -> Vec { + let state = self.state.lock().unwrap(); + let mut ids: Vec = state + .ledgers + .get(store) + .map(|rows| rows.iter().map(|r| r.id).collect()) + .unwrap_or_default(); + ids.sort_unstable(); + ids + } + pub(super) fn ledger(&self, store: &str) -> Vec { + self.state.lock().unwrap().ledgers[store].clone() + } + pub(super) fn version_row(&self) -> Option { + self.state.lock().unwrap().traces.catalog_version.clone() + } + pub(super) fn with_state(&self, f: impl FnOnce(&mut State)) { + f(&mut self.state.lock().unwrap()); + } + pub(super) fn add_catalog_row(&self, id: u32, kind: Kind) { + self.with_state(|s| { + s.ledgers.entry("cat".into()).or_default().push(LedgerRow { + id, + name: format!("future_{id}"), + kind, + checksum: "x".into(), + outcome: Outcome::Applied, + notes: String::new(), + applied_at: String::new(), + applied_by: "9.9.9".into(), + }); + }); + } +} + +#[async_trait] +impl MigrationBackend for Fake { + type Stores = FakeStores; + + fn backend_name(&self) -> &'static str { + "fake" + } + fn migration(&self, id: MigrationId) -> Migration { + table(id) + } + fn released(&self) -> &'static [Released] { + self.released + } + fn stores(&self) -> &FakeStores { + &self.stores + } + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError> { + let mut names: Vec = instances(kind).iter().map(|s| (*s).to_owned()).collect(); + names.reverse(); + Ok(names) + } + async fn read_ledger(&self, store: &StoreRef) -> Result, MigrationError> { + let state = self.state.lock().unwrap(); + if state.fail_read { + return Err(MigrationError::Store("permission denied".into())); + } + Ok(state.ledgers.get(&store.name).cloned().unwrap_or_default()) + } + async fn read_traces(&self) -> Result { + Ok(self.state.lock().unwrap().traces.clone()) + } + async fn catalog_is_empty(&self) -> Result { + Ok(!self.state.lock().unwrap().tables) + } + async fn before_first_write( + &self, + from: Option, + to: u32, + ) -> Result, MigrationError> { + let from = from.map_or("legacy".to_owned(), |v| v.to_string()); + let line = format!("copy {from}-{to}"); + self.with_state(|s| s.writes.push(line.clone())); + Ok(vec![line]) + } + async fn apply( + &self, + store: &StoreRef, + id: MigrationId, + step: Step<'_, FakeStores>, + row: Option, + catalog_version: Option, + ) -> Result { + let failing = self.state.lock().unwrap().fail_apply.clone(); + if failing == Some((id.number(), store.name.clone())) { + return Err(MigrationError::Store("killed".into())); + } + let done = match step { + Step::Code(body) => body.run(&self.stores).await?, + Step::File { .. } | Step::Record => Done::default(), + }; + self.with_state(|s| { + if let Some(mut row) = row { + row.notes = done.notes.join("\n"); + let rows = s.ledgers.entry(store.name.clone()).or_default(); + assert!( + rows.iter().all(|r| r.id != row.id), + "a row is written only where none exists" + ); + rows.push(row); + } + s.writes.push(format!("{} {}", id.number(), store.name)); + if let Some(v) = catalog_version { + s.traces.catalog_version = Some(v.to_string()); + } + if store.kind == StoreKind::Catalog { + s.tables = true; + if id == MigrationId::Baseline { + s.traces.resource_tag_marker = false; + } + } + }); + Ok(done) + } + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError> { + self.with_state(|s| { + s.traces.catalog_version = Some(version.to_string()); + s.writes.push(format!("version {version}")); + }); + Ok(()) + } + async fn delete_ledger_row(&self, store: &StoreRef, id: u32) -> Result<(), MigrationError> { + self.with_state(|s| { + if let Some(rows) = s.ledgers.get_mut(&store.name) { + rows.retain(|r| r.id != id); + } + }); + Ok(()) + } +} + +pub(super) fn runner(fake: &Fake) -> Runner<'_, Fake> { + Runner::new(fake, "1.0.0-test") +} + +#[tokio::test] +async fn a_fresh_store_gets_every_id_in_order_catalog_first() { + let fake = Fake::new(); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Uninitialized + ); + let report = runner(&fake).migrate().await.unwrap(); + let mut expected = vec![format!("copy legacy-{LATEST}")]; + expected.extend(MigrationId::ALL.into_iter().flat_map(writes_for)); + assert_eq!(fake.writes(), expected); + assert_eq!((report.from, report.to), (None, Some(LATEST))); + assert_eq!(report.notes, [format!("copy legacy-{LATEST}")]); + assert_eq!(fake.rows("cat"), (1..=LATEST).collect::>()); + assert_eq!(fake.rows("data-a"), ids_in("data-a")); + assert_eq!(fake.version_row(), Some(LATEST.to_string())); + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Current { version: LATEST } + ); + let again = runner(&fake).migrate().await.unwrap(); + assert!(again.applied.is_empty()); + assert_eq!( + fake.writes().len(), + expected.len(), + "a current catalog is not written" + ); +} + +#[tokio::test] +async fn a_crash_between_stores_resumes_at_the_first_store_without_a_row() { + let fake = Fake::new(); + fake.with_state(|s| s.fail_apply = Some((1, "data-b".into()))); + let err = runner(&fake).migrate().await.unwrap_err(); + assert_eq!(err.exit_code(), exit::ERROR); + // The run had taken its copy and committed two steps, and says so. + let MigrationError::Stopped { report, .. } = &err else { + panic!("expected Stopped: {err:?}"); + }; + assert_eq!(report.applied.len(), 2); + assert_eq!(report.notes, [format!("copy legacy-{LATEST}")]); + assert_eq!(fake.rows("cat"), [1]); + assert_eq!(fake.rows("data-a"), [1]); + // The catalog row alone does not count id 1 as done: the ledger exists + // and is at migration 0. + assert_eq!(fake.version_row().as_deref(), Some("0")); + let CatalogStatus::Behind { version, pending } = runner(&fake).status().await.unwrap() else { + panic!("expected Behind"); + }; + assert_eq!(version, Some(0)); + assert_eq!( + (pending[0].id, pending[0].store_name.as_str()), + (1, "data-b") + ); + + fake.with_state(|s| s.fail_apply = None); + runner(&fake).migrate().await.unwrap(); + let writes = fake.writes(); + let resumed: Vec<&str> = writes.iter().skip(3).map(String::as_str).collect(); + let mut expected = vec![ + format!("copy 0-{LATEST}"), + "1 data-b".to_owned(), + "version 1".to_owned(), + ]; + expected.extend(MigrationId::ALL.into_iter().skip(1).flat_map(writes_for)); + assert_eq!(resumed, expected); +} + +#[tokio::test] +async fn detect_runs_for_every_pending_code_body_before_any_write() { + let fake = Fake::new(); + fake.with_state(|s| { + s.traces.catalog_version = Some("0.0.3".into()); + s.tables = true; + }); + *fake.stores.attention.lock().unwrap() = vec!["policy p collides".into()]; + let err = runner(&fake).migrate().await.unwrap_err(); + let MigrationError::Status(CatalogStatus::NeedsAttention { items }) = &err else { + panic!("{err:?}"); + }; + // Both pending code bodies (migrations 2 and 4 of the fake) report, once + // each, before anything runs. + assert_eq!( + items, + &[ + "migration 2 resource_tag_condition_keys: policy p collides", + "migration 4 index_table_base_key: policy p collides" + ] + ); + assert_eq!(err.exit_code(), exit::ATTENTION); + assert!( + fake.writes().is_empty(), + "nothing is written, not even the copy" + ); + assert!(fake.stores.runs.lock().unwrap().is_empty()); +} + +/// A store with tables but neither a ledger nor a version row (a version +/// row deleted by hand) is not a catalog by the status's rule, and `detect` +/// still runs before the first write, because the store has tables. +#[tokio::test] +async fn detect_runs_on_a_store_with_tables_and_no_version_row() { + let fake = Fake::new(); + fake.with_state(|s| s.tables = true); + *fake.stores.attention.lock().unwrap() = vec!["policy p collides".into()]; + assert_eq!( + runner(&fake).status().await.unwrap(), + CatalogStatus::Uninitialized + ); + let err = runner(&fake).migrate().await.unwrap_err(); + assert_eq!(err.exit_code(), exit::ATTENTION, "{err:?}"); + assert!(fake.writes().is_empty()); + + // With no tables at all there is nothing to find: `detect` is not run, + // and the run applies every id. + let fake = Fake::new(); + *fake.stores.attention.lock().unwrap() = vec!["policy p collides".into()]; + let report = runner(&fake).migrate().await.unwrap(); + assert_eq!(report.to, Some(LATEST)); +} + +#[tokio::test] +async fn the_version_row_follows_the_ledger_through_a_partial_run() { + let fake = Fake::new(); + runner(&fake) + .migrate_through(MigrationId::ResourceTagConditionKeys) + .await + .unwrap(); + assert_eq!(fake.version_row().as_deref(), Some("2")); + let CatalogStatus::Behind { version, pending } = runner(&fake).status().await.unwrap() else { + panic!("expected Behind"); + }; + assert_eq!(version, Some(2)); + let ids: Vec = pending.iter().map(|p| p.id).collect(); + let expected: Vec = MigrationId::ALL + .into_iter() + .filter(|id| id.number() > 2) + .flat_map(|id| { + let n = table(id) + .store_kinds() + .iter() + .map(|kind| instances(*kind).len()) + .sum::(); + std::iter::repeat_n(id.number(), n) + }) + .collect(); + assert_eq!(ids, expected); + assert_eq!(pending[0].kind, Kind::Additive, "NotApplicable is additive"); +} + +#[tokio::test] +async fn a_ledger_read_error_stops_instead_of_counting_as_not_applied() { + let fake = Fake::new(); + fake.with_state(|s| s.fail_read = true); + assert!(runner(&fake).status().await.is_err()); + assert!(runner(&fake).migrate().await.is_err()); + assert!(fake.writes().is_empty()); +} + +pub(super) async fn current(fake: &Fake) { + runner(fake).migrate().await.unwrap(); +} + +#[tokio::test] +async fn every_status_and_its_exit_code() { + // Legacy: a dotted version and no ledger. + let fake = Fake::new(); + fake.with_state(|s| s.traces.catalog_version = Some("0.0.3".into())); + let status = runner(&fake).status().await.unwrap(); + assert!( + matches!(status, CatalogStatus::Behind { version: None, .. }), + "{status:?}" + ); + assert_eq!(status.exit_code(), exit::PENDING); + + // A pending code body with items is NeedsAttention, ahead of Behind. + *fake.stores.attention.lock().unwrap() = vec!["collision".into()]; + let status = runner(&fake).status().await.unwrap(); + assert_eq!(status.label(), "NeedsAttention"); + assert_eq!(status.exit_code(), exit::ATTENTION); + + // Current, then ahead by an additive id, then by a breaking one. + let fake = Fake::new(); + current(&fake).await; + fake.add_catalog_row(LATEST + 1, Kind::Additive); + let status = runner(&fake).status().await.unwrap(); + assert_eq!(status.label(), "AheadCompatible"); + assert_eq!(status.exit_code(), exit::OK); + assert!(runner(&fake).migrate().await.unwrap().applied.is_empty()); + fake.add_catalog_row(LATEST + 2, Kind::Breaking); + let status = runner(&fake).status().await.unwrap(); + assert_eq!(status.label(), "AheadIncompatible"); + assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE); + assert_eq!( + runner(&fake).migrate().await.unwrap_err().exit_code(), + exit::AHEAD_INCOMPATIBLE + ); + + // Behind wins over AheadCompatible. + let fake = Fake::new(); + current(&fake).await; + fake.add_catalog_row(LATEST + 1, Kind::Additive); + fake.with_state(|s| s.ledgers.get_mut("data-a").unwrap().retain(|r| r.id != 4)); + assert_eq!(runner(&fake).status().await.unwrap().label(), "Behind"); + + // A release before 1.0 wrote to it: a dotted version, or the marker. + let fake = Fake::new(); + current(&fake).await; + fake.with_state(|s| s.traces.catalog_version = Some("0.0.3".into())); + let status = runner(&fake).status().await.unwrap(); + assert_eq!(status.label(), "WrittenByOlderRelease"); + assert_eq!(status.exit_code(), exit::ATTENTION); + fake.with_state(|s| { + s.traces.catalog_version = Some(LATEST.to_string()); + s.traces.resource_tag_marker = true; + }); + assert_eq!( + runner(&fake).status().await.unwrap().label(), + "WrittenByOlderRelease" + ); + // ... which ranks above AheadIncompatible, and below Changed. + fake.add_catalog_row(LATEST + 2, Kind::Breaking); + assert_eq!( + runner(&fake).status().await.unwrap().label(), + "WrittenByOlderRelease" + ); + assert!(runner(&fake).migrate().await.is_err()); + fake.with_state(|s| s.ledgers.get_mut("data-b").unwrap()[0].checksum = "edited".into()); + let status = runner(&fake).status().await.unwrap(); + let CatalogStatus::Changed { id, store_name, .. } = &status else { + panic!("{status:?}"); + }; + assert_eq!((*id, store_name.as_str()), (1, "data-b")); + assert_eq!(status.exit_code(), exit::ATTENTION); + assert_eq!(CatalogStatus::Uninitialized.exit_code(), exit::ERROR); +} + +/// A breaking id ahead of the binary ranks above a pending code body whose +/// `detect` reports an item. +#[tokio::test] +async fn ahead_incompatible_ranks_above_needs_attention() { + let fake = Fake::new(); + current(&fake).await; + fake.with_state(|s| s.ledgers.get_mut("data-a").unwrap().retain(|r| r.id != 4)); + *fake.stores.attention.lock().unwrap() = vec!["collision".into()]; + assert_eq!( + runner(&fake).status().await.unwrap().label(), + "NeedsAttention" + ); + fake.add_catalog_row(LATEST + 1, Kind::Breaking); + let status = runner(&fake).status().await.unwrap(); + assert_eq!(status.label(), "AheadIncompatible", "{status:?}"); + let err = runner(&fake).migrate().await.unwrap_err(); + assert_eq!(err.exit_code(), exit::AHEAD_INCOMPATIBLE, "{err:?}"); +} + +#[tokio::test] +async fn a_changed_checksum_stops_the_run() { + let fake = Fake::new(); + current(&fake).await; + fake.with_state(|s| s.ledgers.get_mut("cat").unwrap()[0].checksum = "old".into()); + let err = runner(&fake).migrate().await.unwrap_err(); + assert_eq!(err.exit_code(), exit::ATTENTION); +} diff --git a/crates/storage/src/operations.rs b/crates/storage/src/operations.rs index be9576b39..c8497f934 100644 --- a/crates/storage/src/operations.rs +++ b/crates/storage/src/operations.rs @@ -33,6 +33,12 @@ pub trait OperationsEngine: Send + Sync { /// Check if a configuration key contains sensitive data that should be redacted. fn is_sensitive_key(&self, key: &str) -> bool; + + /// The value this backend's engine uses for the runtime setting `key` + /// when the catalog has no row for it, or `None` for a key the engine + /// does not read. Displays that print a default read it here: no + /// catalog baseline seeds these rows. + fn setting_default(&self, key: &str) -> Option; } /// Parsed connection string components (backend-agnostic). @@ -67,6 +73,16 @@ pub fn catalog_version() -> Result { get_operations_engine().map(OperationsEngine::catalog_version) } +/// The installed backend's default for the runtime setting `key` +/// ([`OperationsEngine::setting_default`]); `None` when no backend is +/// installed or its engine does not read the key. +#[must_use] +pub fn setting_default(key: &str) -> Option { + get_operations_engine() + .ok() + .and_then(|ops| ops.setting_default(key)) +} + /// Redact sensitive information from a connection string. pub fn redact_connection_string(s: &str) -> Result { get_operations_engine().map(|ops| ops.redact_connection_string(s)) diff --git a/crates/storage/src/policy_migration.rs b/crates/storage/src/policy_migration.rs index 3ba62d979..99eae8a11 100644 --- a/crates/storage/src/policy_migration.rs +++ b/crates/storage/src/policy_migration.rs @@ -15,216 +15,17 @@ //! //! This module has two layers: //! - [`rewrite_policy_document`], a pure transform over a single policy -//! document (unit-tested per operator form), and +//! document (unit-tested per operator form). It defines catalog migration 2 +//! and lives, frozen, in that migration's module; it is re-exported here. //! - [`migrate_resource_tag_condition_keys`], which drives it over every stored //! policy reachable through [`ManagementStore`], backend-independently. use serde_json::Value; -use crate::management_store::{ManagementStore, SessionPolicyEntry}; - -/// The condition-key prefix the live service does not populate. -const DYNAMODB_PREFIX: &str = "dynamodb:ResourceTag/"; -/// The condition-key prefix the live service evaluates resource tags as. -const AWS_PREFIX: &str = "aws:ResourceTag/"; - -/// ASCII-case-insensitive prefix strip, returning the untouched remainder. -fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { - let head = s.get(..prefix.len())?; - head.eq_ignore_ascii_case(prefix) - .then(|| &s[prefix.len()..]) -} - -/// Outcome of rewriting one policy document. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RewriteOutcome { - /// The rewritten document. Equal to the input when nothing changed or a - /// collision was found. - pub document: Value, - /// Number of rewrites applied (condition-key renames plus policy-variable - /// prefix rewrites). Zero when nothing changed or on a collision. - pub rewrites: usize, - /// A `Condition` operator block held both `aws:ResourceTag/k` and a - /// `dynamodb:ResourceTag/k` that would rewrite onto the same JSON key. - /// The two cannot share one JSON object, so the document is left unchanged - /// and the caller reports it for manual attention. - pub collision: bool, -} - -impl RewriteOutcome { - /// Whether the document was rewritten (changed and no collision). - #[must_use] - pub fn changed(&self) -> bool { - self.rewrites > 0 && !self.collision - } -} - -/// Rewrite every `dynamodb:ResourceTag/` condition key and -/// `${dynamodb:ResourceTag/...}` policy variable (any ASCII case) in a policy -/// document to the `aws:ResourceTag/` form, inside `Condition` blocks only. -/// -/// Idempotent: a document with no `dynamodb:ResourceTag` reference is returned -/// unchanged with `rewrites == 0`. On a collision the input is returned -/// unchanged with `collision == true`. -#[must_use] -pub fn rewrite_policy_document(document: &Value) -> RewriteOutcome { - let mut doc = document.clone(); - let mut rewrites = 0usize; - let mut collision = false; - - // Statement may be a single object or an array of objects. - match doc.get_mut("Statement") { - Some(Value::Array(statements)) => { - for stmt in statements.iter_mut() { - rewrite_statement(stmt, &mut rewrites, &mut collision); - } - } - Some(stmt @ Value::Object(_)) => { - rewrite_statement(stmt, &mut rewrites, &mut collision); - } - _ => {} - } - - if collision { - // Leave the stored policy exactly as it was. - return RewriteOutcome { - document: document.clone(), - rewrites: 0, - collision: true, - }; - } - - RewriteOutcome { - document: if rewrites > 0 { doc } else { document.clone() }, - rewrites, - collision: false, - } -} - -/// Rewrite the `Condition` block of one statement in place. -fn rewrite_statement(stmt: &mut Value, rewrites: &mut usize, collision: &mut bool) { - let Some(condition) = stmt.get_mut("Condition").and_then(Value::as_object_mut) else { - return; - }; - for (_operator, block) in condition.iter_mut() { - let Some(map) = block.as_object_mut() else { - continue; - }; - rewrite_operator_block(map, rewrites, collision); - } -} - -/// Rewrite one operator block: its condition keys (object keys) and the policy -/// variables in its values. -fn rewrite_operator_block( - block: &mut serde_json::Map, - rewrites: &mut usize, - collision: &mut bool, -) { - // First rewrite the variables inside the values, keyed by the current - // (pre-rename) condition key, so a value rewrite is not lost by the rename. - for value in block.values_mut() { - rewrite_variables_in_json(value, rewrites); - } - - // Collect the condition-key renames. - let mut renames: Vec<(String, String)> = Vec::new(); - for key in block.keys() { - if let Some(rest) = strip_prefix_ci(key, DYNAMODB_PREFIX) { - renames.push((key.clone(), format!("{AWS_PREFIX}{rest}"))); - } - } - if renames.is_empty() { - return; - } - // A rename onto a key the block already has (an aws:ResourceTag key, or - // an earlier rename) is a duplicate when the two values are equal: both - // prefixes resolve from the same table tags, so the two conditions are - // the same condition and the dynamodb: one is dropped. Different values - // cannot share one JSON key, so the block is a collision. - let mut moves: Vec<(String, String)> = Vec::new(); - let mut drops: Vec = Vec::new(); - for (old_key, new_key) in &renames { - let value = &block[old_key]; - let existing = block.get(new_key).or_else(|| { - moves - .iter() - .find(|(_, n)| n == new_key) - .map(|(o, _)| &block[o]) - }); - match existing { - Some(v) if v == value => drops.push(old_key.clone()), - Some(_) => { - *collision = true; - return; - } - None => moves.push((old_key.clone(), new_key.clone())), - } - } - for old_key in drops { - block.remove(&old_key); - *rewrites += 1; - } - for (old_key, new_key) in moves { - if let Some(value) = block.remove(&old_key) { - block.insert(new_key, value); - *rewrites += 1; - } - } -} - -/// Rewrite `${dynamodb:ResourceTag/...}` policy variables inside a value, which -/// is a string or an array of strings. -fn rewrite_variables_in_json(value: &mut Value, rewrites: &mut usize) { - match value { - Value::String(s) => { - if let Some(new) = rewrite_variables_in_str(s, rewrites) { - *s = new; - } - } - Value::Array(items) => { - for item in items.iter_mut() { - rewrite_variables_in_json(item, rewrites); - } - } - _ => {} - } -} - -/// Rewrite the prefix of every `${dynamodb:ResourceTag/...}` variable in `s`, -/// leaving all other text (including a literal `dynamodb:ResourceTag/...` not -/// wrapped in `${...}`) untouched. Returns `None` when nothing changed. -fn rewrite_variables_in_str(s: &str, rewrites: &mut usize) -> Option { - if !s.contains("${") { - return None; - } - let mut out = String::with_capacity(s.len()); - let mut rest = s; - let mut changed = false; - while let Some(open) = rest.find("${") { - out.push_str(&rest[..open + 2]); - let after_open = &rest[open + 2..]; - let Some(close) = after_open.find('}') else { - // No closing brace: copy the remainder and stop. - out.push_str(after_open); - rest = ""; - break; - }; - let inner = &after_open[..close]; - if let Some(tag_rest) = strip_prefix_ci(inner, DYNAMODB_PREFIX) { - out.push_str(AWS_PREFIX); - out.push_str(tag_rest); - *rewrites += 1; - changed = true; - } else { - out.push_str(inner); - } - out.push('}'); - rest = &after_open[close + 1..]; - } - out.push_str(rest); - changed.then_some(out) -} +use crate::management_store::{ManagementStore, OpError, SessionPolicyEntry, StoredDocument}; +pub use crate::migrations::m0002_resource_tag_condition_keys::{ + RewriteOutcome, rewrite_policy_document, +}; /// A policy that the migration could not rewrite automatically, with enough /// identity for an operator to find and fix it by hand. Carries no policy body. @@ -235,6 +36,17 @@ pub struct NeedsAttention { pub policy_name: String, } +/// A stored policy whose document the store could not decode, so the +/// migration cannot tell whether it names `dynamodb:ResourceTag`. Carries +/// the decode error's text and no policy body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Unreadable { + pub account_id: String, + pub principal: String, + pub policy_name: String, + pub error: String, +} + /// A policy the migration would rewrite (detect mode) or did rewrite (apply /// mode), with its identity and the number of rewrites. Carries no policy body. #[derive(Debug, Clone, PartialEq, Eq)] @@ -252,10 +64,22 @@ pub struct MigrationReport { pub rewritten: Vec, /// Policies left unchanged because of a collision. pub needs_attention: Vec, + /// Policies left unchanged because the store could not decode their + /// documents. The pass reads the principal's other policies. + pub unreadable: Vec, /// Count of storage errors encountered. The pass keeps going past an /// error so a single bad row does not hide the rest; the caller treats a /// non-zero count as a failed migration. pub errors: u64, + /// The first storage error. On a backend whose transaction rejects every + /// statement after a failed one, it is the one that says why, and its + /// database error code (a lock timeout) decides whether the runner runs + /// the body again. + pub first_error: Option, + /// One line per storage error, in the order the pass met them: what it + /// was reading or writing (the account, the principal, the policy) and + /// the error's text. + pub failures: Vec, } impl MigrationReport { @@ -264,6 +88,93 @@ impl MigrationReport { pub fn total_rewrites(&self) -> usize { self.rewritten.iter().map(|r| r.rewrites).sum() } + + /// Count a storage error met while doing `what`, keeping the first. + fn failed(&mut self, what: String, e: OpError) { + self.errors += 1; + self.failures.push(format!("{what}: {}", e.message())); + self.first_error.get_or_insert(e); + } +} + +/// The items a person must fix before migration 2 can run, one line each: +/// every policy whose two resource-tag forms collide, and every policy +/// whose stored document the store could not decode, with its decode error. +#[must_use] +pub fn attention_items(report: &MigrationReport) -> Vec { + let collisions = report.needs_attention.iter().map(|n| { + format!( + "account {} {} policy {} names aws:ResourceTag/ and dynamodb:ResourceTag/ \ + for the same key with different values in one condition operator; remove one of \ + the two forms", + n.account_id, n.principal, n.policy_name + ) + }); + let unreadable = report.unreadable.iter().map(|u| { + format!( + "account {} {} policy {} has a stored document that cannot be decoded ({}); \ + replace or delete the policy", + u.account_id, u.principal, u.policy_name, u.error + ) + }); + collisions.chain(unreadable).collect() +} + +/// How many failed lookups and writes [`failure_message`] names; the others +/// are counted. +const FAILURES_NAMED: usize = 5; + +/// The error of a pass that met storage errors: how many, and the first +/// [`FAILURES_NAMED`] of them with what each was reading or writing. +#[must_use] +pub fn failure_message(report: &MigrationReport) -> String { + let mut message = format!( + "resource-tag rewrite: {} policy lookup(s) or write(s) failed:", + report.errors + ); + for line in report.failures.iter().take(FAILURES_NAMED) { + message.push_str("\n "); + message.push_str(line); + } + if report.failures.len() > FAILURES_NAMED { + message.push_str(&format!( + "\n and {} more", + report.failures.len() - FAILURES_NAMED + )); + } + message +} + +/// How many of the policies a pending migration 2 rewrites the `migrate` +/// preview and `verify` name; the others are counted. +pub const PREVIEW_NAMED: usize = 20; + +/// What the `migrate` preview and `verify` print about a pending migration +/// 2 from a detect-mode `report`: the number of stored policies it rewrites, +/// the first [`PREVIEW_NAMED`] of them with their account, principal, policy +/// name and number of keys, and how many more there are. +#[must_use] +pub fn pending_rewrite_lines(report: &MigrationReport) -> Vec { + let total = report.rewritten.len(); + let mut lines = vec![match total { + 0 => "Migration 2 rewrites no stored policy.".to_owned(), + 1 => "Migration 2 rewrites 1 stored policy from dynamodb:ResourceTag to aws:ResourceTag:" + .to_owned(), + n => format!( + "Migration 2 rewrites {n} stored policies from dynamodb:ResourceTag to \ + aws:ResourceTag:" + ), + }]; + lines.extend(report.rewritten.iter().take(PREVIEW_NAMED).map(|r| { + format!( + " account {} {} policy {} ({} key(s))", + r.account_id, r.principal, r.policy_name, r.rewrites + ) + })); + if total > PREVIEW_NAMED { + lines.push(format!(" and {} more", total - PREVIEW_NAMED)); + } + lines } /// Rewrite `dynamodb:ResourceTag/` condition keys to `aws:ResourceTag/` across @@ -276,9 +187,11 @@ impl MigrationReport { /// /// With `apply` false nothing is written: the report lists what would change. /// With `apply` true each changed policy is written back. Every storage error -/// is counted and the pass continues, so an interrupted or partial run is -/// finished by the next `migrate`. Idempotent: a second pass finds nothing to -/// rewrite. No policy body is ever logged or returned. +/// is counted and named, and the pass continues, so an interrupted or partial +/// run is finished by the next `migrate`. A policy whose stored document the +/// store cannot decode is listed as unreadable, and the principal's other +/// policies are read. Idempotent: a second pass finds nothing to rewrite. No +/// policy body is ever logged or returned. pub async fn migrate_resource_tag_condition_keys( store: &dyn ManagementStore, apply: bool, @@ -287,8 +200,8 @@ pub async fn migrate_resource_tag_condition_keys( let accounts = match store.list_all_accounts().await { Ok(a) => a, - Err(_) => { - report.errors += 1; + Err(e) => { + report.failed("list the accounts".to_owned(), e); return report; } }; @@ -333,6 +246,32 @@ fn record_outcome( Some(outcome.document) } +/// [`record_outcome`] for a document as the store read it: one the store +/// could not decode is recorded as unreadable, with its decode error. +fn record_document( + report: &mut MigrationReport, + account_id: &str, + principal: &str, + policy_name: &str, + document: StoredDocument, +) -> Option { + match document { + Ok(document) => record_outcome(report, account_id, principal, policy_name, &document), + Err(error) => { + report.unreadable.push(Unreadable { + account_id: account_id.to_owned(), + principal: principal.to_owned(), + policy_name: policy_name.to_owned(), + error, + }); + None + } + } +} + +/// The name the report gives a permissions boundary. +const BOUNDARY: &str = ""; + async fn migrate_account( store: &dyn ManagementStore, account_id: &str, @@ -344,10 +283,10 @@ async fn migrate_account( Ok(users) => { for (_a, user_name, ..) in users { migrate_inline(store, account_id, "user", &user_name, apply, report).await; - migrate_user_boundary(store, account_id, &user_name, apply, report).await; + migrate_boundary(store, account_id, "user", &user_name, apply, report).await; } } - Err(_) => report.errors += 1, + Err(e) => report.failed(format!("account {account_id}: list its users"), e), } match store.list_groups(account_id).await { Ok(groups) => { @@ -355,16 +294,16 @@ async fn migrate_account( migrate_inline(store, account_id, "group", &group_name, apply, report).await; } } - Err(_) => report.errors += 1, + Err(e) => report.failed(format!("account {account_id}: list its groups"), e), } match store.list_roles(account_id).await { Ok(roles) => { for (_a, role_name, ..) in roles { migrate_inline(store, account_id, "role", &role_name, apply, report).await; - migrate_role_boundary(store, account_id, &role_name, apply, report).await; + migrate_boundary(store, account_id, "role", &role_name, apply, report).await; } } - Err(_) => report.errors += 1, + Err(e) => report.failed(format!("account {account_id}: list its roles"), e), } } @@ -376,102 +315,114 @@ async fn migrate_inline( apply: bool, report: &mut MigrationReport, ) { + let principal = format!("{principal_type}/{principal_name}"); let policies = match store - .list_policies(account_id, principal_type, principal_name) + .list_policy_documents(account_id, principal_type, principal_name) .await { Ok(p) => p, - Err(_) => { - report.errors += 1; + Err(e) => { + report.failed( + format!("account {account_id} {principal}: list its policies"), + e, + ); return; } }; - let principal = format!("{principal_type}/{principal_name}"); - for (policy_name, document, _created) in policies { - if let Some(new_doc) = - record_outcome(report, account_id, &principal, &policy_name, &document) - && apply - && store - .put_policy( - account_id, - principal_type, - principal_name, - &policy_name, - &new_doc, - ) - .await - .is_err() + let rewrites = record_documents(report, account_id, &principal, policies); + if !apply { + return; + } + for (policy_name, new_doc) in rewrites { + if let Err(e) = store + .put_policy( + account_id, + principal_type, + principal_name, + &policy_name, + &new_doc, + ) + .await { - report.errors += 1; + report.failed( + format!("account {account_id} {principal} policy {policy_name}: rewrite it"), + e, + ); } } } -async fn migrate_user_boundary( - store: &dyn ManagementStore, - account_id: &str, - user_name: &str, - apply: bool, +/// Record each of a principal's policies, as the store read them, and return +/// the rewritten documents to write back as `(policy_name, document)`. A +/// document the store could not decode is recorded as unreadable, and the +/// others are still read. +fn record_documents( report: &mut MigrationReport, -) { - match store.get_user_boundary(account_id, user_name).await { - Ok(Some(document)) => { - let principal = format!("user/{user_name}"); - if let Some(new_doc) = record_outcome( - report, - account_id, - &principal, - "", - &document, - ) && apply - && store - .set_user_boundary(account_id, user_name, &new_doc) - .await - .is_err() - { - report.errors += 1; - } - } - Ok(None) => {} - Err(_) => report.errors += 1, - } + account_id: &str, + principal: &str, + policies: Vec<(String, StoredDocument)>, +) -> Vec<(String, Value)> { + policies + .into_iter() + .filter_map(|(policy_name, document)| { + record_document(report, account_id, principal, &policy_name, document) + .map(|new_doc| (policy_name, new_doc)) + }) + .collect() } -async fn migrate_role_boundary( +/// The permissions boundary of a user or a role (`principal_type`). +async fn migrate_boundary( store: &dyn ManagementStore, account_id: &str, - role_name: &str, + principal_type: &str, + principal_name: &str, apply: bool, report: &mut MigrationReport, ) { - match store.get_role_boundary(account_id, role_name).await { - Ok(Some(document)) => { - let principal = format!("role/{role_name}"); - if let Some(new_doc) = record_outcome( - report, - account_id, - &principal, - "", - &document, - ) && apply - && store - .set_role_boundary(account_id, role_name, &new_doc) - .await - .is_err() - { - report.errors += 1; - } + let principal = format!("{principal_type}/{principal_name}"); + let document = match store + .boundary_document(account_id, principal_type, principal_name) + .await + { + Ok(Some(document)) => document, + Ok(None) => return, + Err(e) => { + report.failed( + format!("account {account_id} {principal}: read its permissions boundary"), + e, + ); + return; } - Ok(None) => {} - Err(_) => report.errors += 1, + }; + let Some(new_doc) = record_document(report, account_id, &principal, BOUNDARY, document) else { + return; + }; + if !apply { + return; + } + let written = if principal_type == "user" { + store + .set_user_boundary(account_id, principal_name, &new_doc) + .await + } else { + store + .set_role_boundary(account_id, principal_name, &new_doc) + .await + }; + if let Err(e) = written { + report.failed( + format!("account {account_id} {principal} policy {BOUNDARY}: rewrite it"), + e, + ); } } async fn migrate_sessions(store: &dyn ManagementStore, apply: bool, report: &mut MigrationReport) { let sessions = match store.list_session_policies().await { Ok(s) => s, - Err(_) => { - report.errors += 1; + Err(e) => { + report.failed("list the session policies".to_owned(), e); return; } }; @@ -487,18 +438,30 @@ async fn migrate_sessions(store: &dyn ManagementStore, apply: bool, report: &mut if let Some(new_doc) = record_outcome(report, &account_id, &principal, "", &policy) && apply - && store - .update_session_policy(&session_token, &new_doc) - .await - .is_err() + && let Err(e) = store.update_session_policy(&session_token, &new_doc).await { - report.errors += 1; + report.failed( + format!("account {account_id} {principal} policy : rewrite it"), + e, + ); } } } + #[cfg(test)] mod tests { use super::*; + + /// The prefixes the evaluator resolves resource tags under, before and + /// after the change migration 2 follows. + const DYNAMODB_PREFIX: &str = "dynamodb:ResourceTag/"; + const AWS_PREFIX: &str = "aws:ResourceTag/"; + + fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix) + .then(|| &s[prefix.len()..]) + } use serde_json::json; fn stmt_cond(cond: Value) -> Value { @@ -854,12 +817,134 @@ mod tests { rewrites: 3, }, ], - needs_attention: vec![], - errors: 0, + ..MigrationReport::default() }; assert_eq!(report.total_rewrites(), 5); } + // ── Documents the store cannot decode ─────────────────────────────── + + /// A document the store could not decode is named with its account, + /// principal, policy and decode error, and the principal's other + /// policies are still read and rewritten. + #[test] + fn an_undecodable_document_is_named_and_the_other_policies_are_read() { + let mut report = MigrationReport::default(); + let tagged = stmt_cond(json!({"StringEquals": {"dynamodb:ResourceTag/env": "prod"}})); + let plain = stmt_cond(json!({"StringEquals": {"aws:PrincipalTag/team": "eng"}})); + let policies = vec![ + ( + "broken".to_owned(), + Err("key must be a string at line 1 column 2".to_owned()), + ), + ("tagged".to_owned(), Ok(tagged)), + ("plain".to_owned(), Ok(plain)), + ]; + let writes = record_documents(&mut report, "111122223333", "user/alice", policies); + assert_eq!( + report.unreadable, + vec![Unreadable { + account_id: "111122223333".to_owned(), + principal: "user/alice".to_owned(), + policy_name: "broken".to_owned(), + error: "key must be a string at line 1 column 2".to_owned(), + }] + ); + assert_eq!( + writes + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(), + ["tagged"] + ); + assert_eq!(report.rewritten.len(), 1); + assert_eq!( + attention_items(&report), + [ + "account 111122223333 user/alice policy broken has a stored document that cannot \ + be decoded (key must be a string at line 1 column 2); replace or delete the policy" + ] + ); + } + + /// A pass that met storage errors names what each was reading, with the + /// error's text, and counts the ones past the fifth. + #[test] + fn a_failed_lookup_is_named_in_the_error() { + let mut report = MigrationReport::default(); + for i in 0..7 { + report.failed( + format!("account 111122223333 user/u{i}: list its policies"), + OpError::Internal(format!("failure {i}")), + ); + } + assert_eq!(report.errors, 7); + assert_eq!( + report.first_error, + Some(OpError::Internal("failure 0".to_owned())) + ); + let message = failure_message(&report); + let lines: Vec<&str> = message.lines().collect(); + assert_eq!( + lines[..3], + [ + "resource-tag rewrite: 7 policy lookup(s) or write(s) failed:", + " account 111122223333 user/u0: list its policies: failure 0", + " account 111122223333 user/u1: list its policies: failure 1", + ] + ); + assert_eq!(lines.len(), 7, "{message}"); + assert_eq!(lines[6], " and 2 more"); + } + + // ── What the preview prints ───────────────────────────────────────── + + fn rewritten(n: usize) -> MigrationReport { + MigrationReport { + rewritten: (0..n) + .map(|i| Rewritten { + account_id: "111122223333".to_owned(), + principal: format!("user/u{i}"), + policy_name: "p".to_owned(), + rewrites: 1, + }) + .collect(), + ..MigrationReport::default() + } + } + + /// The count, then the first 20 policies, then how many more. + #[test] + fn the_preview_names_the_first_twenty_rewrites_and_counts_the_rest() { + let lines = pending_rewrite_lines(&rewritten(23)); + assert_eq!( + lines[0], + "Migration 2 rewrites 23 stored policies from dynamodb:ResourceTag to \ + aws:ResourceTag:" + ); + assert_eq!( + lines[1], + " account 111122223333 user/u0 policy p (1 key(s))" + ); + assert_eq!( + lines[20], + " account 111122223333 user/u19 policy p (1 key(s))" + ); + assert_eq!(lines[21], " and 3 more"); + assert_eq!(lines.len(), 22); + let lines = pending_rewrite_lines(&rewritten(20)); + assert_eq!(lines.len(), 21); + assert!(lines.iter().all(|l| !l.contains("more")), "{lines:?}"); + assert_eq!( + pending_rewrite_lines(&rewritten(1))[0], + "Migration 2 rewrites 1 stored policy from dynamodb:ResourceTag to aws:ResourceTag:" + ); + assert_eq!( + pending_rewrite_lines(&rewritten(0)), + ["Migration 2 rewrites no stored policy."] + ); + } + // ── Equivalence: rewrite preserves the decision ────────────────────── // // On main both `aws:ResourceTag/*` and `dynamodb:ResourceTag/*` resolve diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 2ea59c9c9..f9115db15 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -48,14 +48,15 @@ pub enum BackendError { /// Failed to connect to backend database ConnectionFailed { backend: String, details: String }, - /// Catalog schema version mismatch - CatalogVersionMismatch { expected: String, found: String }, - /// Encryption key not found in settings table MissingEncryptionKey, /// Generic initialization failure InitializationFailed(String), + + /// The catalog's status refuses this server, or a migration the factory + /// ran failed; carries the exit code `serve` ends with. + Catalog(crate::migrations::MigrationError), } impl std::fmt::Display for BackendError { @@ -70,15 +71,12 @@ impl std::fmt::Display for BackendError { Self::ConnectionFailed { backend, details } => { write!(f, "Failed to connect to {backend}: {details}") } - Self::CatalogVersionMismatch { expected, found } => write!( - f, - "Catalog version mismatch: expected {expected}, found {found}. Run 'extenddb migrate'" - ), Self::MissingEncryptionKey => write!( f, "Encryption key not found in settings table. Run 'extenddb init'" ), Self::InitializationFailed(msg) => write!(f, "Backend initialization failed: {msg}"), + Self::Catalog(e) => write!(f, "{e}"), } } } @@ -95,6 +93,9 @@ pub struct ServerComponentsOptions { /// Backends whose bootstrap requires operator input ignore this and fail /// as usual; backends with a self-contained bootstrap (SQLite) honor it. pub bootstrap_if_uninitialized: bool, + /// The ExtendDB release that is starting, recorded in the ledger rows of + /// migrations the factory runs (a dev build, an in-memory database). + pub release: &'static str, } /// Factory function type for creating server components. diff --git a/devtools/catalog_checks.py b/devtools/catalog_checks.py new file mode 100644 index 000000000..a60f29970 --- /dev/null +++ b/devtools/catalog_checks.py @@ -0,0 +1,109 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""What the catalog check scripts share: running a command with a time limit, +checking its exit code, reporting each check, and starting and stopping a +server. + +devtools/check-sqlite-catalog-lock, check-sqlite-older-binary, +check-postgres-catalog-lock and check-postgres-older-binary import it. Each +check prints one line, `ok` or `FAIL` and what was checked, with the +command's output indented under a FAIL; `finish` prints the count of failed +checks or `all checks passed` and returns the script's exit code. +""" +import signal +import subprocess +import time + +failures = 0 + + +def report(ok: bool, what: str, output: str = "") -> bool: + """Print one check; a failed one prints `output` under it and is counted.""" + global failures + print(f"{'ok ' if ok else 'FAIL'} {what}", flush=True) + if not ok: + failures += 1 + for line in output.splitlines(): + print(f" | {line}", flush=True) + return ok + + +def _text(stream) -> str: + if stream is None: + return "" + if isinstance(stream, bytes): + return stream.decode("utf-8", "replace") + return stream + + +def run(cmd: list[str], env: dict | None = None, timeout: float = 90, cwd: str | None = None) -> tuple[int, str]: + """Run `cmd` and return its exit code and its output, stdout then stderr. + A command still running after `timeout` seconds is killed, and its code + is -1.""" + try: + done = subprocess.run( + cmd, env=env, cwd=cwd, capture_output=True, text=True, timeout=timeout, check=False + ) + except subprocess.TimeoutExpired as e: + return -1, f"still running after {timeout:g} s\n{_text(e.stdout)}{_text(e.stderr)}" + return done.returncode, done.stdout + done.stderr + + +def expect_exit( + want: int, + what: str, + cmd: list[str], + env: dict | None = None, + contains: str = "", + timeout: float = 90, + cwd: str | None = None, +) -> str: + """Run `cmd` and report whether it exited `want` and printed `contains`. + Returns its output.""" + code, output = run(cmd, env, timeout, cwd) + detail = f" (output names {contains!r})" if contains else "" + if code == want and contains in output: + report(True, f"exit {code}: {what}{detail}") + else: + report(False, f"exit {code}, expected {want}: {what}{detail}", output) + return output + + +def healthy(url: str, proc: subprocess.Popen | None = None, seconds: float = 60) -> bool: + """Whether `url` (a server's /health) answers `healthy` within `seconds`. + Stops waiting when `proc`, the server's process, has exited.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if proc is not None and proc.poll() is not None: + return False + probe = subprocess.run(["curl", "-sk", url], capture_output=True, text=True, check=False) + if "healthy" in probe.stdout: + return True + time.sleep(0.5) + return False + + +def start(cmd: list[str], env: dict | None, log: str, cwd: str | None = None) -> subprocess.Popen: + """Start `cmd` in the background with its output in the file `log`.""" + with open(log, "w", encoding="utf-8") as out: + return subprocess.Popen(cmd, env=env, cwd=cwd, stdout=out, stderr=subprocess.STDOUT) + + +def stop(proc: subprocess.Popen) -> None: + """Stop a process `start` started: SIGTERM, then SIGKILL after 30 s.""" + if proc.poll() is None: + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def finish() -> int: + """Print the outcome of every check reported, and return the exit code.""" + if failures: + print(f"{failures} check(s) failed") + return 1 + print("all checks passed") + return 0 diff --git a/devtools/catalog_checks_test.py b/devtools/catalog_checks_test.py new file mode 100644 index 000000000..a58488560 --- /dev/null +++ b/devtools/catalog_checks_test.py @@ -0,0 +1,69 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Tests of devtools/catalog_checks.py, which the catalog check scripts +report through: a check passes only when the command's exit code and its +output both match, every failure is counted, and finish returns 1 after one. + + python3 -m unittest discover -s devtools -p '*_test.py' +""" +import contextlib +import io +import unittest + +import catalog_checks + + +class CatalogChecksTest(unittest.TestCase): + def setUp(self) -> None: + catalog_checks.failures = 0 + + def check(self, *args, **kwargs) -> tuple[str, str]: + """Run expect_exit, returning its output and what it printed.""" + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + output = catalog_checks.expect_exit(*args, **kwargs) + return output, printed.getvalue() + + def test_an_expected_exit_code_passes(self) -> None: + for want, cmd in ((0, ["true"]), (1, ["false"]), (6, ["sh", "-c", "exit 6"])): + _, printed = self.check(want, "the code", cmd) + self.assertTrue(printed.startswith(f"ok exit {want}: the code"), printed) + self.assertEqual(catalog_checks.failures, 0) + + def test_another_exit_code_fails_with_no_expected_text(self) -> None: + _, printed = self.check(0, "the code", ["sh", "-c", "exit 6"]) + self.assertTrue(printed.startswith("FAIL exit 6, expected 0: the code"), printed) + _, printed = self.check(6, "the code", ["false"]) + self.assertTrue(printed.startswith("FAIL exit 1, expected 6: the code"), printed) + self.assertEqual(catalog_checks.failures, 2) + + def test_the_expected_text_must_be_in_the_output_as_well(self) -> None: + output, printed = self.check(6, "the text", ["sh", "-c", "echo held; exit 6"], contains="held") + self.assertEqual(output, "held\n") + self.assertTrue(printed.startswith("ok exit 6: the text"), printed) + _, printed = self.check(6, "the text", ["sh", "-c", "echo held; exit 6"], contains="free") + self.assertTrue(printed.startswith("FAIL exit 6, expected 6"), printed) + self.assertIn(" | held", printed) + _, printed = self.check(0, "the text", ["sh", "-c", "echo held; exit 6"], contains="held") + self.assertTrue(printed.startswith("FAIL exit 6, expected 0"), printed) + self.assertEqual(catalog_checks.failures, 2) + + def test_a_command_past_its_time_limit_is_killed_and_fails(self) -> None: + code, output = catalog_checks.run(["sleep", "5"], timeout=0.2) + self.assertEqual(code, -1) + self.assertIn("still running after 0.2 s", output) + + def test_finish_returns_1_after_a_failure_and_0_without(self) -> None: + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + self.assertEqual(catalog_checks.finish(), 0) + catalog_checks.report(False, "a check", "line one\nline two") + self.assertEqual(catalog_checks.finish(), 1) + text = printed.getvalue() + self.assertIn("all checks passed", text) + self.assertIn("FAIL a check\n | line one\n | line two\n", text) + self.assertIn("1 check(s) failed", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/devtools/check-postgres-catalog-lock b/devtools/check-postgres-catalog-lock new file mode 100755 index 000000000..e57eca247 --- /dev/null +++ b/devtools/check-postgres-catalog-lock @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Serve and migrate on one PostgreSQL catalog, with a release binary: + + devtools/check-postgres-catalog-lock [port] + + is a superuser's URL without a database, for example +postgresql://postgres:devpass@127.0.0.1:5432. The script creates its own +role and two databases with `init` and removes them at the end. The data +database's name is not the one `init` derives from the catalog's, and is +given as `--data-db=`, so `init` has to create the database it names. + +- verify names that data database; +- the migrate preview and verify run beside a server and exit 0; +- migrate --yes waits for the server's registration and exits 6, naming + "extenddb serve", when it outlasts --lock-timeout; +- while another session holds the migration lock (the advisory key + migrate takes, held here by the script), serve and a second migrate --yes + exit 6, migrate --yes --lock-timeout 0 exits 6 within 10 s, and the + preview exits 0; +- once the lock is free, migrate --yes and serve run; +- a server whose registration session the database ends registers again on + a new session and goes on serving, and migrate --yes exits 6 naming it; +- a server whose registration session ends while a migration holds both + keys stops through its shutdown path: it exits 1, says it could not + register again, and removes its PID file; +- with POOLER_URL set to a connection pooler in front of the same server + (for example postgresql://127.0.0.1:6432, a PgBouncer in transaction mode + that authenticates roles through auth_query), verify, the preview, + migrate --yes, a --reconverge, --forget 3 and its reapplication, and serve + run with the catalog reached through the pooler. + +init's HOME points into a temporary directory, so its TLS certificate is not +written to the caller's home. The server binds 127.0.0.1 only. +""" +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.parse + +import psycopg2 + +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop + +NAMESPACE = 0x0045_4442 # the advisory key namespace of crates/storage-postgres +MIGRATION_KEY = 1 +SERVING_KEY = 2 + + +def admin_query(admin_url: str, sql: str, args: tuple = ()) -> list: + """Run one statement on the server's postgres database and return its + rows.""" + conn = psycopg2.connect(f"{admin_url.rstrip('/')}/postgres") + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute(sql, args) + return cur.fetchall() + finally: + conn.close() + + +def serving_sessions(admin_url: str, stem: str) -> list[int]: + """The pids of the sessions named "extenddb serve" that hold the serving + key on the catalog.""" + rows = admin_query( + admin_url, + "SELECT l.pid FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid " + "WHERE l.locktype = 'advisory' AND l.classid = %s AND l.objid = %s AND l.objsubid = 2 " + "AND l.mode = 'ShareLock' AND l.granted AND a.datname = %s " + "AND a.application_name = 'extenddb serve' ORDER BY l.pid", + (NAMESPACE, SERVING_KEY, f"{stem}_catalog"), + ) + return [pid for (pid,) in rows] + + +def end_serving_sessions(admin_url: str, stem: str) -> list[int]: + """End every "extenddb serve" session on the catalog, as a database + restart does, waiting up to 5 s for each to exit; return their pids.""" + rows = admin_query( + admin_url, + "SELECT pid, pg_terminate_backend(pid, 5000) FROM pg_stat_activity " + "WHERE datname = %s AND application_name = 'extenddb serve'", + (f"{stem}_catalog",), + ) + return [pid for (pid, ended) in rows if ended] + + +def just_renewed(admin_url: str, stem: str, seconds: float = 40) -> bool: + """Wait until the registration session has run its renewal less than + 1 s ago, so the next one is about 14 s away.""" + conn = psycopg2.connect(f"{admin_url.rstrip('/')}/postgres") + conn.autocommit = True + deadline = time.monotonic() + seconds + try: + while time.monotonic() < deadline: + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_stat_activity WHERE datname = %s " + "AND application_name = 'extenddb serve' " + "AND state = 'idle in transaction' " + "AND clock_timestamp() - state_change < interval '1 second'", + (f"{stem}_catalog",), + ) + if cur.fetchone()[0] == 1: + return True + time.sleep(0.1) + finally: + conn.close() + return False + + +def main(binary: str, admin_url: str, port: int) -> int: + binary = os.path.realpath(binary) + admin = urllib.parse.urlsplit(admin_url) + stem = f"eddb_lock_{secrets.token_hex(4)}" + app_password = f"App{secrets.token_hex(8)}" + work = tempfile.mkdtemp(prefix=f"{stem}.") + env = dict(os.environ, HOME=os.path.join(work, "home")) + config = os.path.join(work, "extenddb.toml") + serve = [binary, "serve", "--config", config, "--foreground", "--port", str(port)] + servers: list[subprocess.Popen] = [] + holder = None + try: + code, output = run( + [ + binary, "init", "--config", config, + "--pg-host", admin.hostname, "--pg-port", str(admin.port or 5432), + "--pg-user", urllib.parse.unquote(admin.username or "postgres"), + f"--catalog-db={stem}_catalog", f"--data-db={stem}_items", + "--extenddb-user", stem, "--bind-addr", "127.0.0.1", + ], + dict( + env, + EXTENDDB_PG_PASSWORD=urllib.parse.unquote(admin.password or ""), + EXTENDDB_APP_PASSWORD=app_password, + ), + ) + if code != 0: + report(False, f"exit {code}: init", output) + return 1 + print("ok exit 0: init") + migrate = [binary, "migrate", "--config", config] + expect_exit( + 0, + "verify reaches the data database --data-db= named", + [binary, "verify", "--config", config], + env, + contains=f"Connected to data database '{stem}_items'", + ) + + log = os.path.join(work, "serve.log") + server = start(serve, env, log) + servers.append(server) + if not healthy(f"https://127.0.0.1:{port}/health", server): + stop(server) + with open(log, encoding="utf-8") as f: + report(False, "the server became healthy", f.read()) + return 1 + print("ok the server is healthy") + expect_exit(0, "migrate preview beside a server", migrate, env) + expect_exit(0, "verify beside a server", [binary, "verify", "--config", config], env) + expect_exit( + 6, + "migrate --yes while a server runs", + migrate + ["--yes", "--lock-timeout", "1"], + env, + contains='"extenddb serve"', + ) + stop(server) + + holder = psycopg2.connect(f"{admin_url.rstrip('/')}/{stem}_catalog") + with holder.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s, %s)", (NAMESPACE, MIGRATION_KEY)) + expect_exit(6, "serve while a migration holds the lock", serve, env) + expect_exit( + 6, + "a second migrate --yes while a migration holds the lock", + migrate + ["--yes", "--lock-timeout", "1"], + env, + ) + expect_exit( + 6, + "migrate --yes --lock-timeout 0 while a migration holds the lock, within 10 s", + migrate + ["--yes", "--lock-timeout", "0"], + env, + timeout=10, + ) + expect_exit(0, "migrate preview while a migration holds the lock", migrate, env) + holder.rollback() + holder.close() + holder = None + + expect_exit(0, "migrate --yes once the lock is free", migrate + ["--yes", "--lock-timeout", "5"], env) + server = start(serve, env, log) + servers.append(server) + report(healthy(f"https://127.0.0.1:{port}/health", server), "serve once the lock is free") + + # The database ends the registration's session, as a restart, a + # failover or pg_terminate_backend does. The server finds it gone at + # its next renewal and registers again on a new session. + ended = end_serving_sessions(admin_url, stem) + again: list[int] = [] + deadline = time.monotonic() + 45 + while time.monotonic() < deadline and server.poll() is None: + again = [pid for pid in serving_sessions(admin_url, stem) if pid not in ended] + if again: + break + time.sleep(0.5) + serving = server.poll() is None and healthy(f"https://127.0.0.1:{port}/health", server, 10) + with open(log, encoding="utf-8") as f: + report( + len(ended) == 1 and len(again) == 1 and serving, + f"a server whose registration session ended registers again and serves " + f"(ended {ended}, registered again as {again})", + f.read(), + ) + expect_exit( + 6, + "migrate --yes while the server is registered again", + migrate + ["--yes", "--lock-timeout", "1"], + env, + contains='"extenddb serve"', + ) + stop(server) + + # A migration takes both keys between the end of the session and the + # server's next renewal: the server cannot register again, and stops + # through its shutdown path. + pid_file = os.path.join(env["HOME"], ".extenddb", "run", f"extenddb-{port}.pid") + server = start(serve + ["--write-pid-file"], env, log) + servers.append(server) + up = healthy(f"https://127.0.0.1:{port}/health", server) + report(up and os.path.exists(pid_file), f"serve with a PID file at {pid_file}") + holder = psycopg2.connect(f"{admin_url.rstrip('/')}/{stem}_catalog") + renewed = just_renewed(admin_url, stem) + ended = end_serving_sessions(admin_url, stem) + with holder.cursor() as cur: + cur.execute( + "SELECT pg_try_advisory_xact_lock(%s, %s), pg_try_advisory_xact_lock(%s, %s)", + (NAMESPACE, MIGRATION_KEY, NAMESPACE, SERVING_KEY), + ) + took = cur.fetchone() + try: + code = server.wait(timeout=60) + except subprocess.TimeoutExpired: + code = None + holder.rollback() + holder.close() + holder = None + with open(log, encoding="utf-8") as f: + text = f.read() + report( + renewed + and len(ended) == 1 + and took == (True, True) + and code == 1 + and "could not register again" in text + and not os.path.exists(pid_file), + f"a server that cannot register again while a migration holds the keys stops with " + f"exit {code} and removes its PID file (keys taken: {took}, PID file left: " + f"{os.path.exists(pid_file)})", + text, + ) + + pooler = os.environ.get("POOLER_URL", "").strip().rstrip("/") + if pooler: + through_pooler(binary, env, work, pooler, stem, app_password, port, servers) + else: + print("skip the pass through a connection pooler: POOLER_URL is not set") + finally: + for server in servers: + stop(server) + if holder is not None: + holder.close() + cleanup(admin_url, stem) + shutil.rmtree(work, ignore_errors=True) + return finish() + + +def through_pooler( + binary: str, + env: dict, + work: str, + pooler: str, + stem: str, + app_password: str, + port: int, + servers: list, +) -> None: + """The commands with the catalog reached through a connection pooler, + which must refuse a login that sets parameters it does not track.""" + parts = urllib.parse.urlsplit(pooler) + url = ( + f"postgresql://{stem}:{urllib.parse.quote(app_password)}@{parts.hostname}:" + f"{parts.port or 5432}/{stem}_catalog" + ) + config = os.path.join(work, "pooler.toml") + with open(os.path.join(work, "extenddb.toml"), encoding="utf-8") as f: + text = f.read() + with open(config, "w", encoding="utf-8") as f: + for line in text.splitlines(): + if line.strip().startswith("connection_string"): + line = f'connection_string = "{url}"' + f.write(line + "\n") + os.chmod(config, 0o600) + migrate = [binary, "migrate", "--config", config] + expect_exit(0, "verify through the pooler", [binary, "verify", "--config", config], env, contains="HEALTHY") + expect_exit(0, "migrate preview through the pooler", migrate, env, contains="Up to date") + expect_exit(0, "migrate --yes through the pooler", migrate + ["--yes", "--lock-timeout", "5"], env) + output = expect_exit( + 0, + "migrate --yes --reconverge through the pooler", + migrate + ["--yes", "--reconverge", "--lock-timeout", "5"], + env, + contains="Ran again 1 baseline", + ) + report( + all(f"Ran again {n} " in output for n in (1, 3, 4)) + and "Skipped 2 resource_tag_condition_keys:" in output, + "the reconverge through the pooler ran migrations 1, 3 and 4 again and skipped 2", + output, + ) + expect_exit(0, "migrate --forget 3 through the pooler", migrate + ["--forget", "3"], env) + expect_exit( + 0, + "migrate --yes applies migration 3 again through the pooler", + migrate + ["--yes", "--lock-timeout", "5"], + env, + contains="Applied 3 sort_key_definitions", + ) + log = os.path.join(work, "serve-pooler.log") + server = start([binary, "serve", "--config", config, "--foreground", "--port", str(port)], env, log) + servers.append(server) + up = healthy(f"https://127.0.0.1:{port}/health", server) + stop(server) + with open(log, encoding="utf-8") as f: + report(up, "serve through the pooler", f.read()) + + +def cleanup(admin_url: str, stem: str) -> None: + conn = psycopg2.connect(f"{admin_url.rstrip('/')}/postgres") + conn.autocommit = True + with conn.cursor() as cur: + for database in (f"{stem}_catalog", f"{stem}_items"): + cur.execute(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)') + cur.execute(f'DROP ROLE IF EXISTS "{stem}"') + conn.close() + + +if __name__ == "__main__": + if len(sys.argv) not in (3, 4): + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) == 4 else 18452)) diff --git a/devtools/check-postgres-older-binary b/devtools/check-postgres-older-binary new file mode 100755 index 000000000..6fd2a446b --- /dev/null +++ b/devtools/check-postgres-older-binary @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""A PostgreSQL catalog this binary migrated, against the last 0.1.x image: + + devtools/check-postgres-older-binary \\ + [image] [port] + + is a superuser's URL without a database, reachable from this host +and from a container on the host network, for example +postgresql://postgres:devpass@127.0.0.1:5432. The fixture directory holds +catalog.sql and data.sql (crates/storage-postgres/tests/fixtures/released/). + +Two deployments: a fresh one from this binary's `init`, and the fixture +loaded into new databases and upgraded with this binary's `migrate --yes`. +On each, the image runs `verify`, then `migrate --yes`, then `serve`, and +each must end as the last 0.1.x release does on a v1.0 catalog: `verify` +exits 1 and writes nothing, `migrate --yes` exits 0 and writes the catalog, +and `serve` refuses and exits 1 by itself. For every run +the script prints what changed in the settings rows, the ledgers and the +schema, and checks what this binary then reports: after a run that wrote the +catalog, `migrate` and `verify` exit 5 naming WrittenByOlderRelease, `serve` +refuses with 5, and `migrate --yes --reconverge` restores Current with the +schema it had before; after a run that changed nothing, the catalog is still +Current. + +The image's containers are named "$CONTAINER" (default +extenddb-older-binary), use the host network, run as the caller's uid, and +are removed at the end, with the role, the databases and the temporary +directory. The servers bind 127.0.0.1 only. +""" +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.parse + +import psycopg2 + +from catalog_checks import finish, report, run + +HERE = os.path.dirname(os.path.realpath(__file__)) +class Deployment: + """A role and two databases on the admin URL's server.""" + + def __init__(self, admin_url: str, stem: str) -> None: + self.admin_url = admin_url.rstrip("/") + parts = urllib.parse.urlsplit(self.admin_url) + self.host = parts.hostname + self.port = parts.port or 5432 + self.role = stem + self.password = f"App{secrets.token_hex(8)}" + self.catalog = f"{stem}_catalog" + self.data = stem + + def url(self, database: str) -> str: + return f"postgresql://{self.role}:{self.password}@{self.host}:{self.port}/{database}" + + def admin(self, database: str = "postgres"): + conn = psycopg2.connect(f"{self.admin_url}/{database}") + conn.autocommit = True + return conn + + def create_from(self, fixture: str) -> None: + conn = self.admin() + with conn.cursor() as cur: + cur.execute(f"CREATE ROLE \"{self.role}\" LOGIN PASSWORD '{self.password}'") + for database in (self.catalog, self.data): + cur.execute(f'CREATE DATABASE "{database}" OWNER "{self.role}"') + conn.close() + for database, dump in ((self.catalog, "catalog.sql"), (self.data, "data.sql")): + load = psycopg2.connect(self.url(database)) + load.autocommit = True + with load.cursor() as cur, open(os.path.join(fixture, dump), encoding="utf-8") as f: + cur.execute(f.read()) + load.close() + conn = psycopg2.connect(self.url(self.catalog)) + conn.autocommit = True + with conn.cursor() as cur: + for key, value in ( + ("data_database_connection_string", self.url(self.data)), + ("data_database_name", self.data), + ): + cur.execute("UPDATE settings SET value = %s WHERE key = %s", (value, key)) + conn.close() + + def snapshot(self) -> list[str]: + """Settings rows, ledger rows and the schema of both databases.""" + lines = [] + for store, database in (("catalog", self.catalog), ("data", self.data)): + conn = psycopg2.connect(self.url(database)) + conn.set_session(readonly=True, autocommit=True) + with conn.cursor() as cur: + if store == "catalog": + cur.execute("SELECT key, value FROM settings ORDER BY key") + for key, value in cur.fetchall(): + if key == "encryption_key" or "connection_string" in key: + value = "" + lines.append(f"setting {key} = {value}") + for table, columns in ( + ("extenddb_migrations", "id, name, outcome"), + ("schema_history", "filename"), + ): + cur.execute("SELECT to_regclass(%s) IS NOT NULL", (f"public.{table}",)) + if cur.fetchone()[0]: + cur.execute(f"SELECT {columns} FROM {table} ORDER BY 1") + lines.extend(f"{store} {table}: {row}" for row in cur.fetchall()) + conn.close() + return lines + self.schema() + + def schema(self) -> list[str]: + base = f"postgresql://{self.role}:{self.password}@{self.host}:{self.port}" + out = subprocess.run( + [sys.executable, os.path.join(HERE, "schema-dump"), "postgres", base, self.catalog, self.data], + capture_output=True, text=True, check=True, + ).stdout + return out.splitlines() + + def drop(self) -> None: + conn = self.admin() + with conn.cursor() as cur: + for database in (self.catalog, self.data): + cur.execute(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)') + cur.execute(f'DROP ROLE IF EXISTS "{self.role}"') + conn.close() + + +def diff(before: list[str], after: list[str]) -> list[str]: + removed = [f"- {line}" for line in before if line not in after] + added = [f"+ {line}" for line in after if line not in before] + return removed + added + + +def write_config(path: str, deployment: Deployment, work: str, port: int) -> None: + tls = os.path.join(work, "home", ".extenddb", "tls") + with open(path, "w", encoding="utf-8") as f: + f.write( + f'[server]\nbind_addr = "127.0.0.1"\nport = {port}\n\n' + f'[server.tls]\ncert_path = "{tls}/cert.pem"\nkey_path = "{tls}/key.pem"\n\n' + f'[storage]\nbackend = "postgres"\n\n' + f'[storage.postgres]\nconnection_string = "{deployment.url(deployment.catalog)}"\n' + ) + os.chmod(path, 0o600) + + +def image_run( + image: str, name: str, work: str, args: list[str], password: str, wait: float = 60 +) -> tuple[str, str]: + """Run the image with `args`; returns how it ended and its output. A + command still running after `wait` seconds (a server that serves) is + stopped.""" + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + started = subprocess.run( + [ + "docker", "run", "-d", "--name", name, "--network", "host", + "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{work}:{work}", + "-e", f"HOME={work}/home", "-e", f"EXTENDDB_PG_PASSWORD={password}", image, *args, + ], + capture_output=True, text=True, check=False, + ) + if started.returncode != 0: + return "did not start", started.stderr + deadline = time.monotonic() + wait + ended = None + while time.monotonic() < deadline: + state = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}} {{.State.ExitCode}}", name], + capture_output=True, text=True, check=False, + ).stdout.split() + if state and state[0] == "false": + ended = f"exit {state[1]}" + break + time.sleep(0.5) + if ended is None: + subprocess.run(["docker", "stop", "-t", "20", name], capture_output=True, check=False) + ended = f"still running after {wait:.0f} s, stopped" + logs = subprocess.run(["docker", "logs", name], capture_output=True, text=True, check=False) + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + return ended, logs.stdout + logs.stderr + + +def check_case(label: str, d: Deployment, binary: str, image: str, name: str, work: str, port: int) -> None: + config = os.path.join(work, f"{d.role}.toml") + write_config(config, d, work, port) + env = dict(os.environ, HOME=os.path.join(work, "home")) + code, output = run([binary, "verify", "--config", config], env) + report(code == 0, f"exit {code}: {label}: this binary's verify before the image runs", output) + admin = urllib.parse.urlsplit(d.admin_url) + admin_user = urllib.parse.unquote(admin.username or "postgres") + admin_password = urllib.parse.unquote(admin.password or "") + # The release's migrate connects as an admin role, as its manual says. + for command in ( + ["verify"], + ["migrate", "--yes", "--pg-user", admin_user], + ["serve", "--foreground"], + ): + what = " ".join(command[:2]) + before = d.snapshot() + ended, output = image_run( + image, name, work, [command[0], "--config", config, *command[1:]], admin_password + ) + after = d.snapshot() + changes = diff(before, after) + print(f" {label}: the image's {what}: {ended}; {len(changes)} change(s)") + for line in changes: + print(f" | {line}") + for line in output.strip().splitlines()[-3:]: + print(f" | > {line}") + # What the image did, so an image that cannot reach the database or + # does not start fails here instead of passing with nothing checked. + if command[0] == "verify": + report( + ended == "exit 1" and not changes, + f"{ended}: {label}: the image's verify runs, fails its version check and writes nothing", + output, + ) + elif command[0] == "migrate": + report( + ended == "exit 0" and bool(changes), + f"{ended}: {label}: the image's migrate --yes runs and writes the catalog", + output, + ) + else: + report( + ended == "exit 1", + f"{ended}: {label}: the image's serve refuses the catalog and exits 1 by itself", + output, + ) + if not changes: + code, out = run([binary, "migrate", "--config", config], env) + report(code == 0, f"exit {code}: {label}: after the image's {what}, the catalog is Current", out) + continue + code, out = run([binary, "migrate", "--config", config], env) + report( + code == 5 and "a release before 1.0 wrote to this catalog" in out, + f"exit {code}: {label}: after the image's {what}, migrate reports WrittenByOlderRelease", + out, + ) + code, out = run([binary, "verify", "--config", config], env) + report(code == 5, f"exit {code}: {label}: after the image's {what}, verify exits 5", out) + code, out = run([binary, "serve", "--config", config, "--foreground", "--port", str(port)], env, timeout=60) + report(code == 5, f"exit {code}: {label}: after the image's {what}, serve refuses", out) + code, out = run([binary, "migrate", "--config", config, "--yes", "--reconverge"], env) + report(code == 0, f"exit {code}: {label}: migrate --yes --reconverge", out) + code, out = run([binary, "verify", "--config", config], env) + report(code == 0 and "OK: Current" in out, f"exit {code}: {label}: verify reports Current", out) + report(d.schema() == [l for l in before if not l.startswith(("setting ", "catalog ", "data "))], + f"{label}: the schema is the one it had before the image's {what}") + + +def main(binary: str, admin_url: str, fixture: str, image: str, port: int) -> int: + binary = os.path.realpath(binary) + name = os.environ.get("CONTAINER", "extenddb-older-binary") + work = tempfile.mkdtemp(prefix="eddb-older.") + os.chmod(work, 0o755) + env = dict(os.environ, HOME=os.path.join(work, "home")) + fresh = Deployment(admin_url, f"eddb_old_{secrets.token_hex(4)}") + upgraded = Deployment(admin_url, f"eddb_old_{secrets.token_hex(4)}") + try: + parts = urllib.parse.urlsplit(admin_url) + code, output = run( + [ + binary, "init", "--config", os.path.join(work, "init.toml"), + "--pg-host", parts.hostname, "--pg-port", str(parts.port or 5432), + "--pg-user", urllib.parse.unquote(parts.username or "postgres"), + "--catalog-db", fresh.catalog, "--data-db", fresh.data, + "--extenddb-user", fresh.role, "--bind-addr", "127.0.0.1", + ], + dict( + env, + EXTENDDB_PG_PASSWORD=urllib.parse.unquote(parts.password or ""), + EXTENDDB_APP_PASSWORD=fresh.password, + ), + ) + report(code == 0, f"exit {code}: init a fresh catalog", output) + + upgraded.create_from(fixture) + config = os.path.join(work, "upgrade.toml") + write_config(config, upgraded, work, port) + code, output = run([binary, "migrate", "--config", config, "--yes"], env) + report(code == 0, f"exit {code}: migrate --yes the fixture {os.path.basename(fixture.rstrip('/'))}", output) + + for label, d in (("fresh", fresh), ("upgraded", upgraded)): + check_case(label, d, binary, image, name, work, port) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + for d in (fresh, upgraded): + d.drop() + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (4, 5, 6): + sys.exit(__doc__) + sys.exit( + main( + sys.argv[1], + sys.argv[2], + sys.argv[3], + sys.argv[4] if len(sys.argv) > 4 else "extenddb/extenddb-postgres:0.1.14", + int(sys.argv[5]) if len(sys.argv) > 5 else 18453, + ) + ) diff --git a/devtools/check-released-through b/devtools/check-released-through new file mode 100755 index 000000000..3c5587de9 --- /dev/null +++ b/devtools/check-released-through @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Fail when a release would ship catalog migrations it does not freeze. + + devtools/check-released-through [repository root] + +A migration body is frozen once `RELEASED_THROUGH` in +crates/storage/src/migrations/registry.rs covers its id: each backend's +checksum test then pins the body's checksum, kind and convergent flag, and +`migrate --forget` refuses the id. The release PR raises the bound to the +last id and pastes the checksum lines the test prints (CONTRIBUTING.md, +"Adding a catalog migration", step 7). A tag cut without that step ships +bodies that a later change can edit with every test passing, and every +deployment that ran them would then refuse to serve. + +Exit 0 when the bound equals the last id of the `migration_ids!` list, or +when the commit has no migration registry (a release before the migration +framework). Exit 1, naming both numbers, otherwise. The release workflows' +gate jobs run this on the tagged commit. +""" + +import pathlib +import re +import sys + + +def main() -> int: + root = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else pathlib.Path(__file__).resolve().parent.parent + registry = root / "crates/storage/src/migrations/registry.rs" + if not registry.exists(): + print(f"{registry} does not exist; this commit has no catalog migrations to release") + return 0 + text = registry.read_text(encoding="utf-8") + bound = re.search(r"pub const RELEASED_THROUGH: u32 = (\d+);", text) + if bound is None: + print(f"FAIL: no `pub const RELEASED_THROUGH: u32 = ;` in {registry}") + return 1 + # Entries of `migration_ids!`: `Variant = , "",` + ids = [int(n) for n in re.findall(r"^\s*[A-Z][A-Za-z0-9]* = (\d+), \"", text, re.MULTILINE)] + if not ids: + print(f"FAIL: no migration ids found in {registry}") + return 1 + released, last = int(bound.group(1)), max(ids) + if released < last: + print( + f"FAIL: RELEASED_THROUGH is {released} and the last migration is {last}. " + f"Raise RELEASED_THROUGH to {last} and paste the checksum lines each backend's " + "checksum test prints into its RELEASED list before tagging " + "(CONTRIBUTING.md, \"Adding a catalog migration\", step 7)." + ) + return 1 + if released > last: + print(f"FAIL: RELEASED_THROUGH is {released}, above the last migration, {last}") + return 1 + print(f"ok: RELEASED_THROUGH is {released}, the last migration") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devtools/check-sqlite-catalog-lock b/devtools/check-sqlite-catalog-lock new file mode 100755 index 000000000..779a76def --- /dev/null +++ b/devtools/check-sqlite-catalog-lock @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Serve and migrate on one SQLite catalog, with a release binary: + + devtools/check-sqlite-catalog-lock [port] + +- the migrate preview and verify run beside a server and exit 0; +- migrate --yes waits for the server's lock and exits 6 when it outlasts + --lock-timeout; +- a server refuses with exit 6 while a migration holds the database (the + holder here is this script, with flock(2) on the lock file the migration + takes); +- once the database is free, migrate --yes and serve run; +- on a catalog behind this binary, verify and the preview exit 3, and + migrate --yes applies what is missing; +- on a config whose database does not exist, migrate --yes exits 1, names + extenddb init, and creates neither the database nor its directory. + +Everything runs in a new temporary directory, with init's HOME pointed into +it so its TLS certificate is not written to the caller's home. The server +binds 127.0.0.1 only. +""" +import fcntl +import os +import re +import shutil +import sqlite3 +import sys +import tempfile + +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop + + +def with_path(config: str, database: str, out: str) -> None: + """Write `config` to `out` with its `path = ` line naming `database`.""" + with open(config, encoding="utf-8") as f: + text = f.read() + with open(out, "w", encoding="utf-8") as f: + f.write(re.sub(r"(?m)^path = .*$", f'path = "{database}"', text)) + + +def main(binary: str, port: int) -> int: + binary = os.path.realpath(binary) + work = tempfile.mkdtemp() + servers = [] + holder = None + try: + os.chdir(work) + env = dict(os.environ, HOME=os.path.join(work, "home")) + database = os.path.join(work, "db.sqlite") + code, output = run( + [binary, "init", "--backend", "sqlite", "--config", "extenddb.toml", + "--sqlite-path", database], + env, + ) + if code != 0: + report(False, f"exit {code}: init", output) + return finish() + lock_file = os.path.realpath(database) + ".lock" + migrate = [binary, "migrate", "--config", "extenddb.toml"] + serve = [binary, "serve", "--config", "extenddb.toml", "--foreground", "--port", str(port)] + url = f"https://127.0.0.1:{port}/health" + + server = start(serve, None, "serve.log") + servers.append(server) + if not healthy(url, server, 30): + stop(server) + with open("serve.log", encoding="utf-8") as f: + report(False, "the server did not become healthy", f.read()) + return finish() + expect_exit(0, "migrate preview beside a server", migrate) + expect_exit(0, "verify beside a server", [binary, "verify", "--config", "extenddb.toml"]) + expect_exit(6, "migrate --yes while a server runs", migrate + ["--yes", "--lock-timeout", "1"]) + stop(server) + + holder = open(lock_file, "a+b") + fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB) + expect_exit(6, "serve while a migration holds the database", serve) + expect_exit( + 6, + "a second migrate --yes while a migration holds the database", + migrate + ["--yes", "--lock-timeout", "1"], + ) + expect_exit(0, "migrate preview while a migration holds the database", migrate) + fcntl.flock(holder, fcntl.LOCK_UN) + holder.close() + holder = None + + expect_exit(0, "migrate --yes once the database is free", migrate + ["--yes", "--lock-timeout", "5"]) + server = start(serve, None, "serve.log") + servers.append(server) + up = healthy(url, server, 30) + stop(server) + with open("serve.log", encoding="utf-8") as f: + report(up, "serve once the database is free", f.read()) + + # A catalog behind this binary: verify and the preview exit 3, and + # migrate --yes applies what is missing. + conn = sqlite3.connect(database) + conn.execute( + "DELETE FROM extenddb_migrations WHERE id = (SELECT MAX(id) FROM extenddb_migrations)" + ) + conn.commit() + conn.close() + expect_exit(3, "verify on a catalog behind this binary", [binary, "verify", "--config", "extenddb.toml"]) + expect_exit(3, "migrate preview on a catalog behind this binary", migrate) + expect_exit(0, "migrate --yes applies what is missing", migrate + ["--yes", "--lock-timeout", "5"]) + + # A config whose database does not exist: migrate --yes refuses with + # 1, names init, and creates neither the database nor its directory. + missing = os.path.join(work, "missing") + with_path("extenddb.toml", os.path.join(missing, "db.sqlite"), "missing.toml") + output = expect_exit( + 1, + "migrate --yes on a path with no database", + [binary, "migrate", "--config", "missing.toml", "--yes", "--lock-timeout", "1"], + ) + if "extenddb init" not in output: + report(False, "the refusal does not name extenddb init", output) + report(not os.path.exists(missing), "nothing was created for the missing database") + finally: + for server in servers: + stop(server) + if holder is not None: + holder.close() + os.chdir("/") + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3): + sys.exit(__doc__) + sys.exit(main(sys.argv[1], int(sys.argv[2]) if len(sys.argv) == 3 else 18450)) diff --git a/devtools/check-sqlite-older-binary b/devtools/check-sqlite-older-binary new file mode 100755 index 000000000..8b2bd0b5f --- /dev/null +++ b/devtools/check-sqlite-older-binary @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""A catalog this binary migrated, against the last 0.1.x dev image: + + devtools/check-sqlite-older-binary \\ + [image] [port] + +With DEV_BIN set to a dev-mode build of this binary (features +sqlite,dev-mode), each catalog the image wrote to is also given to that +build's serve with no config file, only EXTENDDB__STORAGE__SQLITE__PATH: it +must refuse with 5 and advise a migrate command without --config that sets +EXTENDDB__STORAGE__SQLITE__PATH to the database's path, and that command, +run as printed in a directory whose extenddb.toml names another database, +must bring the catalog back to Current. The same build's serve, started +beside an extenddb.toml naming a current catalog with the variable naming +the changed one, must advise a command with the variable and the config +file's absolute path, which, run as printed from another directory, brings +the changed catalog back to Current and leaves the current one unchanged. + +Two catalogs: a fresh one from `init`, and the fixture dump upgraded with +`migrate --yes`. On a copy of each, the image's `serve` (a dev build) runs +until it is healthy and is stopped; on another copy, the image runs +`migrate --yes`. For each run the script prints what changed, as a diff of +devtools/sqlite-dump before and after, and checks what this binary then +reports: after a run that wrote the catalog, `migrate` and `verify` exit 5 +naming WrittenByOlderRelease, `serve` refuses with 5 within 60 s, and +`migrate --yes --reconverge` restores Current with the schema it had before. +The image's `migrate --yes` must exit 0 and change the catalog, so an image +that cannot run fails the check; after an image `serve` that changed +nothing, the catalog must still be Current. + +The image's container is named "$CONTAINER", by default +extenddb-older-binary, publishes only on 127.0.0.1:, runs as the +caller's uid, and is removed at the end, with the temporary directory. +""" +import difflib +import glob +import os +import re +import shlex +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time + +from catalog_checks import expect_exit, finish, report, run + +HERE = os.path.dirname(os.path.realpath(__file__)) + + +def dump(database: str) -> list[str]: + """devtools/sqlite-dump of `database`, one line per object or row.""" + out = subprocess.run( + [sys.executable, os.path.join(HERE, "sqlite-dump"), database], + capture_output=True, text=True, check=True, + ).stdout + return out.splitlines() + + +def schema(database: str) -> list[str]: + return [line for line in dump(database) if not line.startswith("row ")] + + +def with_path(config: str, database: str, out: str) -> None: + """Write `config` to `out` with its `path = ` line naming `database`.""" + with open(config, encoding="utf-8") as f: + text = f.read() + with open(out, "w", encoding="utf-8") as f: + f.write(re.sub(r"(?m)^path = .*$", f'path = "{database}"', text)) + + +def changed_lines(diff: list[str]) -> list[str]: + """The removed and added lines of a unified diff, without its headers.""" + return [line for line in diff if len(line) > 1 and line[0] in "+-" and line[1] not in "+-"] + + +def docker(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=False) + + +class Image: + """The released dev image, run as the caller on one catalog directory.""" + + def __init__(self, image: str, name: str, port: int, init_config: str) -> None: + self.image = image + self.name = name + self.port = port + self.init_config = init_config + self.user = f"{os.getuid()}:{os.getgid()}" + + def serve(self, directory: str) -> bool: + """Run the image's server on `directory` until it is healthy, then + stop it; whether it became healthy. Its log goes to image-serve.log.""" + docker( + "run", "-d", "--name", self.name, "--user", self.user, + "-p", f"127.0.0.1:{self.port}:18080", "-v", f"{directory}:/var/lib/extenddb", self.image, + ) + up = False + for _ in range(120): + probe = subprocess.run( + ["curl", "-s", f"http://127.0.0.1:{self.port}/health"], + capture_output=True, text=True, check=False, + ) + if "healthy" in probe.stdout: + up = True + break + time.sleep(0.5) + docker("stop", "-t", "20", self.name) + logs = docker("logs", self.name) + with open(os.path.join(directory, "image-serve.log"), "w", encoding="utf-8") as f: + f.write(logs.stdout + logs.stderr) + docker("rm", "-f", self.name) + return up + + def migrate(self, directory: str) -> tuple[int, str]: + """The image's migrate --yes on `directory`, with init's config + pointing at the database's path in the container.""" + with_path(self.init_config, "/var/lib/extenddb/extenddb.sqlite", + os.path.join(directory, "extenddb.toml")) + done = docker( + "run", "--rm", "--name", self.name, "--user", self.user, + "-v", f"{directory}:/var/lib/extenddb", self.image, + "migrate", "--config", "/var/lib/extenddb/extenddb.toml", "--yes", + ) + output = done.stdout + done.stderr + with open(os.path.join(directory, "image-migrate.log"), "w", encoding="utf-8") as f: + f.write(output) + return done.returncode, output + + +def check(label: str, kind: str, source: str, binary: str, image: Image, work: str, port: int) -> None: + """The image's `kind` run (serve or migrate) on a copy of `source`, and + what this binary reports afterwards.""" + directory = os.path.join(work, f"{label}-{kind}") + os.makedirs(directory) + # Fold the WAL into the file, so the copy is the whole catalog. + conn = sqlite3.connect(source) + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + conn.close() + database = os.path.join(directory, "extenddb.sqlite") + shutil.copyfile(source, database) + os.chmod(database, 0o600) + config = os.path.join(directory, "v1.toml") + with_path(image.init_config, database, config) + before = dump(database) + + print(f"--- {label} catalog, the image's {kind}", flush=True) + if kind == "serve": + if image.serve(directory): + print("ok the image served the catalog", flush=True) + else: + with open(os.path.join(directory, "image-serve.log"), encoding="utf-8") as f: + report(False, "the image did not serve the catalog", f.read()) + else: + code, output = image.migrate(directory) + print(f" the image's migrate --yes exited {code}") + for line in output.splitlines()[-5:]: + print(f" | {line}") + if code != 0: + report(False, f"the image's migrate --yes exited {code}, expected 0") + after = dump(database) + changes = changed_lines(list(difflib.unified_diff(before, after, lineterm=""))) + if not changes: + print(" it changed nothing") + if kind == "migrate": + report(False, "the image's migrate --yes changed nothing in the catalog") + expect_exit(0, "still Current", [binary, "verify", "--config", config]) + return + print(" what it changed:") + for line in changes: + print(f" | {line}") + output = expect_exit( + 5, "migrate reports a catalog a release before 1.0 wrote to", + [binary, "migrate", "--config", config], + ) + if "a release before 1.0 wrote to this catalog" not in output: + report(False, "the preview does not name WrittenByOlderRelease", output) + expect_exit(5, "verify reports it", [binary, "verify", "--config", config]) + dev_bin = os.environ.get("DEV_BIN", "") + if dev_bin: + dev_server_repair(directory, source, dev_bin, port) + expect_exit( + 5, "serve refuses it", + [binary, "serve", "--config", config, "--foreground", "--port", str(port)], + timeout=60, + ) + expect_exit(0, "migrate --yes --reconverge", [binary, "migrate", "--config", config, "--yes", "--reconverge"]) + expect_exit(0, "Current after the reconverge", [binary, "verify", "--config", config]) + was, now = schema(source), schema(database) + if was == now: + print("ok the schema is the one it had before the image ran", flush=True) + else: + report( + False, + "the schema differs from the one before the image ran:", + "\n".join(difflib.unified_diff(was, now, lineterm="")), + ) + + +def advised_command(output: str) -> tuple[str, list[str], list[str]]: + """The migrate command a refusal advises: the line, its words, and the + assignments before `extenddb`.""" + advised = next( + (line for line in output.splitlines() if line.startswith(" ") and " extenddb migrate " in line), "" + ) + words = shlex.split(advised) + assignments = [w for w in words[: words.index("extenddb")] if "=" in w] if "extenddb" in words else [] + return advised, words, assignments + + +def run_as_printed(label: str, dev_bin: str, words: list[str], assignments: list[str], cwd: str) -> None: + """Run an advised command as it is printed, in `cwd`, with no variable + of the caller's: the command itself names the database it acts on.""" + command_env = {k: v for k, v in os.environ.items() if k != "EXTENDDB__STORAGE__SQLITE__PATH"} + command_env.update(a.split("=", 1) for a in assignments) + rest = words[words.index("extenddb") + 1 :] + expect_exit(0, label, [dev_bin, *rest], command_env, cwd=cwd) + + +def dev_server_repair(directory: str, source: str, dev_bin: str, port: int) -> None: + """A dev build's serve, started with no config file on a copy of the + catalog, refuses it; the command it advises, run as it is printed, + repairs the copy. Then a dev server started beside an extenddb.toml + naming a current catalog, with the variable naming another copy, refuses + that copy and advises a command that carries the variable and the config + file's absolute path: run as printed from another directory, it repairs + the copy and leaves the config file's catalog as it was.""" + dev = os.path.join(directory, "dev") + os.makedirs(dev) + database = os.path.join(dev, "extenddb.sqlite") + shutil.copyfile(os.path.join(directory, "extenddb.sqlite"), database) + os.chmod(database, 0o600) + env = dict(os.environ, EXTENDDB__STORAGE__SQLITE__PATH=database) + output = expect_exit( + 5, "a dev server with no config file refuses it", + [dev_bin, "serve", "--foreground", "--port", str(port)], env, timeout=60, cwd=dev, + ) + advised, words, assignments = advised_command(output) + if not advised: + report(False, "the dev server advised no migrate command", output) + elif "--config" in advised: + report(False, f"the dev server advised a config file it does not have: {advised}") + elif assignments != [f"EXTENDDB__STORAGE__SQLITE__PATH={database}"]: + report(False, f"the advised command does not name the database: {advised}") + else: + # In the case's directory, whose extenddb.toml names another database. + run_as_printed(f"the advised command, beside another extenddb.toml:{advised}", dev_bin, words, + assignments, directory) + expect_exit(0, "Current after the advised command", [dev_bin, "verify"], env, cwd=dev) + + configured = os.path.join(directory, "dev-config") + os.makedirs(configured) + current = os.path.join(configured, "current.sqlite") + shutil.copyfile(source, current) + os.chmod(current, 0o600) + with_path(os.path.join(directory, "v1.toml"), current, os.path.join(configured, "extenddb.toml")) + changed = os.path.join(configured, "changed.sqlite") + shutil.copyfile(os.path.join(directory, "extenddb.sqlite"), changed) + os.chmod(changed, 0o600) + with open(current, "rb") as f: + current_bytes = f.read() + env = dict(os.environ, EXTENDDB__STORAGE__SQLITE__PATH=changed) + output = expect_exit( + 5, "a dev server whose variable overrides ./extenddb.toml refuses the variable's database", + [dev_bin, "serve", "--foreground", "--port", str(port)], env, timeout=60, cwd=configured, + ) + advised, words, assignments = advised_command(output) + config = os.path.join(configured, "extenddb.toml") + given = words[words.index("--config") + 1] if "--config" in words[:-1] else "" + if not advised: + report(False, "the dev server advised no migrate command", output) + elif assignments != [f"EXTENDDB__STORAGE__SQLITE__PATH={changed}"] or given != config: + report(False, f"the advised command does not reproduce the variable and the config file: {advised}") + else: + run_as_printed(f"the advised command, from another directory:{advised}", dev_bin, words, assignments, + directory) + expect_exit(0, "the variable's database is Current after it", [dev_bin, "verify"], env, cwd=configured) + with open(current, "rb") as f: + same = f.read() == current_bytes + report(same, "the config file's database is unchanged") + + +def main(binary: str, fixture: str, image_name: str, port: int) -> int: + binary = os.path.realpath(binary) + fixture = os.path.realpath(fixture) + name = os.environ.get("CONTAINER", "extenddb-older-binary") + work = tempfile.mkdtemp() + try: + init_dir = os.path.join(work, "init") + released_dir = os.path.join(work, "released") + os.makedirs(init_dir) + os.makedirs(released_dir) + fresh = os.path.join(init_dir, "fresh.sqlite") + code, output = run( + [binary, "init", "--backend", "sqlite", "--config", "extenddb.toml", "--sqlite-path", fresh], + dict(os.environ, HOME=os.path.join(work, "home")), + cwd=init_dir, + ) + if code != 0: + report(False, f"exit {code}: init", output) + return finish() + image = Image(image_name, name, port, os.path.join(init_dir, "extenddb.toml")) + + released = os.path.join(released_dir, "released.sqlite") + os.close(os.open(released, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)) + conn = sqlite3.connect(released) + with open(fixture, encoding="utf-8") as f: + conn.executescript(f.read()) + conn.commit() + conn.close() + config = os.path.join(released_dir, "v1.toml") + with_path(image.init_config, released, config) + code, output = run([binary, "migrate", "--config", config, "--yes"]) + if code != 0: + report(False, f"exit {code}: migrate --yes the fixture", output) + return finish() + expect_exit(0, "the released catalog is Current after migrate --yes", [binary, "verify", "--config", config]) + # The copy migrate --yes took holds the released catalog; it is not + # part of what the image is given. + for copy in glob.glob(released + ".pre-migration-*"): + os.remove(copy) + # The fixture's dev access key is encrypted under the key the fixture + # replaced, so a dev build cannot read it; remove it, and the image + # seeds its own, as it does in a fresh catalog. + conn = sqlite3.connect(released) + conn.execute("DELETE FROM access_keys WHERE access_key_id = ?", ("AKIAIOSFODNN7EXAMPLE",)) + conn.commit() + conn.close() + + for kind in ("serve", "migrate"): + check("fresh", kind, fresh, binary, image, work, port) + check("upgraded", kind, released, binary, image, work, port) + finally: + docker("rm", "-f", name) + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (3, 4, 5): + sys.exit(__doc__) + sys.exit( + main( + sys.argv[1], + sys.argv[2], + sys.argv[3] if len(sys.argv) > 3 else "extenddb/extenddb-dev:0.1.14", + int(sys.argv[4]) if len(sys.argv) > 4 else 18451, + ) + ) diff --git a/devtools/schema-dump b/devtools/schema-dump new file mode 100755 index 000000000..2e027636c --- /dev/null +++ b/devtools/schema-dump @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Print the schema of a catalog as text, one object per line, in a fixed +order, so a fresh install can be compared with docs/schema/.txt. + + devtools/schema-dump sqlite + devtools/schema-dump postgres + +On PostgreSQL the dump lists tables, columns, indexes, constraints, +sequences, functions, triggers, views and types; the per-table `_ddb_*` +tables are left out, because no migration creates them, and so are the +objects an extension owns (pgvector's functions and types), so a server with +pgvector and one without print the same file. A migration that creates any +other kind of object adds a query for it here. On SQLite every schema object +is listed; a fresh install has no per-table objects. +""" +import sqlite3 +import sys + + +def sqlite_schema(path: str) -> list[str]: + # Read-write: a read-only connection cannot open a WAL database whose + # -shm file does not exist. Nothing here writes. + conn = sqlite3.connect(path) + rows = conn.execute( + "SELECT type, name, tbl_name, sql FROM sqlite_master " + "WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name" + ).fetchall() + conn.close() + return [ + f"{kind} {name} on {table}: {' '.join((sql or '').split())}" + for kind, name, table, sql in rows + ] + + +POSTGRES_QUERIES = [ + ( + "table", + """SELECT c.relname, '' FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') + AND c.relname NOT LIKE '\\_ddb\\_%' + ORDER BY 1""", + ), + ( + "column", + """SELECT c.relname || '.' || a.attname, + format_type(a.atttypid, a.atttypmod) + || CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END + || COALESCE(' DEFAULT ' || pg_get_expr(d.adbin, d.adrelid), '') + || CASE WHEN a.attcollation <> t.typcollation + THEN ' COLLATE ' || quote_ident(co.collname) ELSE '' END + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_type t ON t.oid = a.atttypid + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + LEFT JOIN pg_collation co ON co.oid = a.attcollation + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') + AND c.relname NOT LIKE '\\_ddb\\_%' + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY c.relname, a.attnum""", + ), + ( + "index", + """SELECT indexname, indexdef FROM pg_indexes + WHERE schemaname = 'public' AND tablename NOT LIKE '\\_ddb\\_%' + ORDER BY 1""", + ), + ( + "constraint", + """SELECT c.relname || '.' || k.conname, pg_get_constraintdef(k.oid) + FROM pg_constraint k + JOIN pg_class c ON c.oid = k.conrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname NOT LIKE '\\_ddb\\_%' + ORDER BY 1""", + ), + ( + "sequence", + """SELECT sequencename, data_type || ' START ' || start_value + || ' INCREMENT ' || increment_by + FROM pg_sequences WHERE schemaname = 'public' ORDER BY 1""", + ), + ( + "function", + """SELECT p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')', '' + FROM pg_proc p + WHERE p.pronamespace = 'public'::regnamespace + AND NOT EXISTS (SELECT 1 FROM pg_depend d + WHERE d.objid = p.oid AND d.deptype = 'e') + ORDER BY 1""", + ), + ( + "view", + """SELECT c.relname, pg_get_viewdef(c.oid) FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('v', 'm') + AND NOT EXISTS (SELECT 1 FROM pg_depend d + WHERE d.objid = c.oid AND d.deptype = 'e') + ORDER BY 1""", + ), + ( + "type", + # Enums, domains, ranges and standalone composite types; a table's + # own row type is not listed. + """SELECT t.typname, t.typtype::text || COALESCE(' ' || ( + SELECT string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) + FROM pg_enum e WHERE e.enumtypid = t.oid), '') + FROM pg_type t + WHERE t.typnamespace = 'public'::regnamespace + AND t.typtype IN ('e', 'd', 'r', 'm', 'c') + AND (t.typtype <> 'c' OR EXISTS (SELECT 1 FROM pg_class c + WHERE c.oid = t.typrelid AND c.relkind = 'c')) + AND NOT EXISTS (SELECT 1 FROM pg_depend d + WHERE d.objid = t.oid AND d.deptype = 'e') + ORDER BY 1""", + ), + ( + "trigger", + """SELECT c.relname || '.' || t.tgname, pg_get_triggerdef(t.oid) + FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND NOT t.tgisinternal + AND c.relname NOT LIKE '\\_ddb\\_%' + ORDER BY 1""", + ), +] + + +def postgres_schema(base: str, database: str) -> list[str]: + import psycopg2 # in requirements.txt + + conn = psycopg2.connect(f"{base.rstrip('/')}/{database}") + conn.set_session(readonly=True, autocommit=True) + lines = [] + with conn.cursor() as cur: + for kind, query in POSTGRES_QUERIES: + cur.execute(query) + for name, text in cur.fetchall(): + lines.append(f"{kind} {name}: {' '.join(text.split())}".rstrip(": ")) + conn.close() + return lines + + +def main(argv: list[str]) -> None: + if len(argv) == 2 and argv[0] == "sqlite": + print("\n".join(sqlite_schema(argv[1]))) + elif len(argv) == 4 and argv[0] == "postgres": + base, catalog, data = argv[1:] + for store, database in (("catalog", catalog), ("data", data)): + print(f"# {store} database") + print("\n".join(postgres_schema(base, database))) + else: + sys.exit(__doc__) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/devtools/sqlite-dump b/devtools/sqlite-dump new file mode 100755 index 000000000..b0e69804f --- /dev/null +++ b/devtools/sqlite-dump @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Print a SQLite catalog as text two runs can be compared with: every +schema object with its SQL, then every table's rows in rowid order, one per +line. The encryption key, stored secrets and password hashes are replaced, +so the output can go to a log. + + devtools/sqlite-dump +""" +import importlib.machinery +import importlib.util +import json +import pathlib +import sqlite3 +import sys + + +def schema_dump(): + """devtools/schema-dump as a module, for its SQLite schema lines.""" + path = pathlib.Path(__file__).with_name("schema-dump") + loader = importlib.machinery.SourceFileLoader("schema_dump", str(path)) + spec = importlib.util.spec_from_loader("schema_dump", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def main(path: str) -> None: + for line in schema_dump().sqlite_schema(path): + print(line) + # Read-write: a read-only connection cannot open a WAL database whose + # -shm file does not exist. Nothing here writes. + conn = sqlite3.connect(path) + for (table,) in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' " + "ORDER BY name" + ).fetchall(): + quoted = '"' + table.replace('"', '""') + '"' + columns = [row[1] for row in conn.execute(f"PRAGMA table_info({quoted})")] + for row in conn.execute(f"SELECT * FROM {quoted} ORDER BY rowid"): + values = dict(zip(columns, row)) + if table == "settings" and values.get("key") == "encryption_key": + values["value"] = "" + for secret in ("secret_key_encrypted", "password_hash"): + if values.get(secret) is not None: + values[secret] = "" + print(f"row {table}: {json.dumps(values, sort_keys=True, default=repr)}") + conn.close() + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/docker/README.md b/docker/README.md index c13982acd..902d4a7ca 100644 --- a/docker/README.md +++ b/docker/README.md @@ -142,20 +142,23 @@ are written mode 0600. ### Upgrade -Back up PostgreSQL, stop or scale down incompatible serving versions, then run -one migration Job with `EXTENDDB_PG_PASSWORD` injected from the orchestrator's -secret store: +Back up PostgreSQL, stop or scale down the serving containers, then run one +migration Job with the serving image and its config: ```bash extenddb migrate \ --config /var/lib/extenddb/extenddb.toml \ - --yes \ - --pg-user + --yes ``` -The generated config stores the application connection but not the PostgreSQL -bootstrap password, so the migration Job must receive that elevated credential -separately. Do not give it to the serving container. +The Job connects as the application role in the generated config, which does +all the schema work, so it needs no credential the serving container lacks. +To have it create the pgvector extension as well, add +`--pg-user ` and inject `EXTENDDB_PG_PASSWORD` from the +orchestrator's secret store into the Job only; without them the Job reports +whether pgvector is available and goes on. A backup taken by a 0.1.x release +can restore without its secondary indexes, so take a new backup of each table +after the upgrade. ### Steady state diff --git a/docs/adr/0003-catalog-migration-mechanism.md b/docs/adr/0003-catalog-migration-mechanism.md index 8136635b5..91cf126b6 100644 --- a/docs/adr/0003-catalog-migration-mechanism.md +++ b/docs/adr/0003-catalog-migration-mechanism.md @@ -1,6 +1,6 @@ # ADR-0003: Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations -- Status: Accepted +- Status: Superseded by [ADR-0023](0023-catalog-migrations.md) - Date: 2026-06-17 - Deciders: ExtendDB CODEOWNERS diff --git a/docs/adr/0023-catalog-migrations.md b/docs/adr/0023-catalog-migrations.md new file mode 100644 index 000000000..7febf6246 --- /dev/null +++ b/docs/adr/0023-catalog-migrations.md @@ -0,0 +1,422 @@ +# ADR-0023: One list of catalog migrations for every backend, with a ledger in every store + +- Status: Accepted +- Date: 2026-10-29 +- Deciders: @LeeroyHannigan + +## Context + +A deployment's catalog holds its table and index definitions, accounts, IAM +principals and policies, backups and settings. Each backend keeps it in its +own stores: PostgreSQL in a catalog database and a data database, SQLite in +one database file, Cassandra in a catalog keyspace and one keyspace per +account, and MongoDB in a catalog database and a data database. Deployments +created by 0.1.x releases hold data, and each new release upgrades their +catalogs in place. The code references in this section are to main at +`cf321f05`, before this decision was implemented. + +Each backend migrated its stores its own way: + +- PostgreSQL applied numbered SQL files and recorded each one by file name, + with no checksum (`crates/storage-postgres/src/migrations.rs:480-506`). + `migrate` and `serve` also ran shape steps that no file held, and a code + migration added an index to every index table. +- SQLite applied one schema file, `SCHEMA_SQL`, again on every upgrade, then + ran converge steps for the changes `CREATE TABLE IF NOT EXISTS` cannot make. +- Cassandra applied numbered CQL files after a `USE` on the shared session, so + a statement could run in another keyspace (#632). +- MongoDB created its collections and indexes in Rust code during `init`. It + wrote a ledger document that nothing read, and no step changed the data + database after `init`. + +These mechanisms failed in the following ways: + +- The same change had a different number on each backend. PostgreSQL's + catalog file 003 and Cassandra's V005 make the same change. The version + strings moved at different releases: SQLite reached schema version 0.0.3 at + v0.1.7, PostgreSQL reached it at v0.1.11, and MongoDB stayed at 0.0.2. +- Released files were edited in place, and no check compared a file with the + one that had been applied. PostgreSQL's `001_schema.sql` changed for v0.1.2, + v0.1.7 and v0.1.14, and Cassandra's V001 changed once. The runner skips a + file it has recorded, so the v0.1.2 edit, which added `table_class`, + `sse_specification` and `on_demand_throughput` to `tables`, never reached + catalogs that v0.1.0 or v0.1.1 created. Measured on catalogs built from the + SQL files of those two tags: `migrate --yes` built from `cf321f05` exits 0, + `verify` reports the catalog healthy, and every CreateTable then fails with + `column "table_class" of relation "tables" does not exist`. +- ADR-0003 recorded that failure and chose `sqlx::migrate` for PostgreSQL, + with `destroy` and `init` as the upgrade path. It was not implemented. Main + still ran the runner above, the workspace `sqlx` dependency has no `migrate` + feature (`Cargo.toml:69`), `.gitattributes` has no rule for `*.sql`, and + upgrades ran in place through `extenddb migrate`. +- The ledgers and the `catalog_version` settings row could disagree, and + `migrate` decided from the row. The files wrote the version themselves, so + running Cassandra's V001 again reset the row and overwrote settings an + operator had changed. +- SQLite's re-applied file could not change a table that existed. Every + column change needed a hand-written converge step, and SQLite index tables + created by v0.1.3 to v0.1.7 never got the base key index that CreateTable + adds to new ones. Re-applying the file also inserted the resource-tag marker + row before the policy rewrite ran (`crates/storage-sqlite/src/schema.rs:460`, + `crates/app/src/cmd_migrate.rs:305-307` and `331`), so a rewrite that + stopped at a collision left a catalog that `serve` accepted. +- MongoDB and Cassandra took no migration lock + (`crates/storage/src/bootstrapper.rs:107-127`), and nothing stopped + `migrate` while a PostgreSQL, MongoDB or Cassandra server was running. +- On a Cassandra cluster of more than one node, a statement after a + `CREATE TABLE` can reach a node that does not have the table yet. With two + `cassandra:4.1` nodes and statements sent round robin, ten runs of the + migrations at `cf321f05` failed nine times with one connection per node and + ten times with ten connections per node (#637). +- The startup check differed by backend. PostgreSQL and SQLite refused any + mismatch and told the operator to run `migrate`, which is the wrong advice + for a catalog newer than the binary. MongoDB and Cassandra never read the + version. +- No test started from a catalog that a released binary created, and no test + compared an upgraded catalog with a fresh one. + +v1.0 supports one serving instance per catalog (`README.md:141`). Upgrades +have to run in place: `destroy` drops the databases with their items, and +ADR-0003 accepted that only while no deployment depended on a catalog. + +## Options Considered + +1. One numbered list of migrations for every backend, a ledger in every + store, and one runner that each backend implements (the Decision below). +2. Numbering each backend's migrations separately, as Temporal and + Cassandra Reaper do. Each backend would keep its own directory and + sequence, and a change would be written once per backend under that + backend's next number. +3. A desired-state schema file per backend, applied by a tool that diffs it + against the live store. +4. sqlx's migrator, which ADR-0003 chose. sqlx records a checksum for each + migration and commits the ledger row in the migration's transaction, and + this design keeps both properties; sqlx ran migrations again while the two + committed separately + ([sqlx#1966](https://github.com/launchbadge/sqlx/issues/1966), fixed by + #1991). +5. Destroy and init, the upgrade path ADR-0003 chose. `destroy` drops the + databases, items included, and `init` creates new ones. + +## Decision + +Number every catalog migration once for all backends, record each one a +store applies in that store's ledger, and apply the list with one runner +that each backend implements, as follows. + +1. Every migration has one number, the same on every backend. The list is + `MigrationId` in `crates/storage/src/migrations/registry.rs`, written as + one `migration_ids!` invocation that generates the enum, `MigrationId::ALL` + and each id's name and description; `MigrationId::LATEST` is the last + entry of `ALL`. Each backend has a table in its `src/migrations/mod.rs`, + an exhaustive `match` from `MigrationId` to that backend's entry with no + wildcard arm, so a backend without an entry for an id does not compile. + An entry gives a body for each kind of store the migration changes, its + kind (`Additive` or `Breaking`) and whether it is convergent, meaning safe + to run again on a store at any later id, over data that clients wrote + through the API since it was applied. A body that rewrites such data is + not convergent: migration 2 would rewrite a policy written after it and + change that policy's decision. An entry can instead be + `NotApplicable` with a reason that says why the change has no effect on + that backend. A body is a SQL or CQL file embedded with `include_str!`, or + a Rust code body with a read-only `detect` step and an idempotent `run` + step. Kong defines its migrations the same way, one module per migration + with a required body for each of its stores. + +2. Every store has a ledger, `extenddb_migrations`, with a checksum for each + migration it applied. Its columns are `id`, `name`, `kind`, `checksum`, + `outcome` (`applied` or `not_applicable`), `notes`, `applied_at` and + `applied_by`. The runner applies ids in order and, for each id, the body of + every store the id changes, catalog store first. On PostgreSQL and SQLite a + body and its ledger row commit in one transaction, except a PostgreSQL + body that must run outside one (migration 4's `CREATE INDEX + CONCURRENTLY`), whose row is written after the body completes. On Cassandra and MongoDB + every statement of a body is idempotent and the row is written after the + body completes, so after a crash the next run executes the body again. The + catalog ledger has a row for every id, `NotApplicable` ones included, and + is the deployment's record. + + A file body's checksum is the SHA-256 of its text with CRLF normalised to + LF. A code body records its fingerprint, a constant of its module that + changes only with a new migration. A recorded checksum that differs from + the binary's stops `migrate` and `serve` with exit 5. A released body does + not change. Each backend's `RELEASED` table pins the checksum, kind and + convergent flag of every body at or below `RELEASED_THROUGH`, and a unit + test refuses a changed one unless its entry carries a `rebless` reason, + which a reviewer sees in the diff. The release PR that raises + `RELEASED_THROUGH` pastes the lines that test prints, so a migration under + review carries no checksum literal. + +3. The catalog version is read from the ledgers. It is the highest id such + that it and every lower id have a row in the catalog ledger and in every + store their bodies touch, so it is one integer, the same on every backend. + No file or constant carries a version string. The runner also writes the + integer to the `catalog_version` settings row with each catalog ledger row, + because releases before 1.0 read that row: the v0.1.14 PostgreSQL binary's + `serve` refuses the integer with `invalid catalog version '4': expected 3 + dot-separated components`, its `verify` reports a failed check, and its + `migrate --yes` accepts it and prints `4 -> 4`. Two decisions in 1.0 read + that row: the check for a catalog that an older release wrote to + (decision 6), and whether a store with no ledger rows is a catalog at all + (one with neither is `Uninitialized`). + +4. Migration 1 is one convergent body per store. It creates the v1.0 schema + in an empty store and brings every released catalog to the same schema. + Fresh installs and upgrades run the same body, and it is recorded as + `applied` in both cases. Nothing identifies the legacy state first, so a + store in a state nobody listed converges the same way. Changes after + migration 1 are later numbered migrations, because a file that is edited + for every change cannot upgrade a store that already exists. + + - PostgreSQL: `crates/storage-postgres/migrations/0001_baseline.sql` and + `crates/storage-postgres/data_migrations/0001_baseline.sql` hold the + legacy files' statements in order. Every `CREATE INDEX` is guarded by + `IF NOT EXISTS` and keeps its legacy name, the three `tables` columns that + catalogs from v0.1.0 and v0.1.1 lack are added with + `ADD COLUMN IF NOT EXISTS`, the shape steps that `migrate` and `serve` + ran are in the file, and the sequence reset never lowers a live sequence. + - SQLite: a code body runs `crates/storage-sqlite/migrations/0001_baseline.sql` + and then the two steps that SQLite cannot make conditional in SQL. It + rebuilds `backup_items` only when the table has no `id` column, and adds + `table_restores.source_table_arn` only when the column is missing. + - Cassandra: one CQL file for the catalog keyspace and one for the account + keyspaces, with every statement `IF NOT EXISTS`. + - MongoDB: code bodies for the catalog and data databases, which accept an + existing collection or index only by its error code. + + The first time migration 1 is applied to a store, the backend's apply step + compares the store's definitions with `0001_baseline.expected.json`, which is generated + by running migration 1 on an empty store and committed beside the body. + The comparison runs inside migration 1's transaction, so on a difference + the runner commits nothing and names the object. It ignores objects the + file does not name, because the same stores hold per-table objects that + requests create. + + Migration 1 seeds no settings rows, because every seeded setting has an + equal default in code; on PostgreSQL `init`, its catalog transaction + writes the data database's connection rows. It deletes the `resource_tag_migration` marker row + that every released baseline seeded. It records every legacy file in the + legacy ledgers (the `schema_history` tables on PostgreSQL and Cassandra, + the `001_initial` document on MongoDB), which stay and are never read + again, so a released 0.1.x `migrate --yes` finds every legacy file recorded + and runs none of them. Without those rows it replays them. Measured with + the v0.1.14 image against a PostgreSQL catalog whose data database had no + legacy ledger: its `migrate --yes` replayed data file 003, which drops and + recreates `idempotency_tokens`, and a token row planted beforehand was + gone. + +5. Every backend has a migration lock, and a serving registration keeps + `migrate --yes` and `serve` apart. Each side takes its own before it looks + for the other, so two that start together cannot both miss each other. + + | Backend | Migration lock | Serving registration | + |---|---|---| + | PostgreSQL | A transaction-level advisory lock on the catalog database, on the same key that every release since v0.1.3 takes, held on a dedicated connection that runs `SELECT 1` every 15 s | `serve` holds a second advisory key in shared mode, on a connection whose TCP keepalive settings end the session within 30 s of the serving host dying; `migrate --yes` takes that key in exclusive mode | + | SQLite | `flock` on `.lock`, retried every 250 ms up to the lock timeout | The same `flock`, which `serve` holds while it runs | + | Cassandra | A row in the catalog's `settings` table, written only by lightweight transactions at `SERIAL`, with a 60 s TTL renewed every 15 s | A row per server in `extenddb_servers`, renewed every 15 s with a 45 s TTL | + | MongoDB | A document in the `settings` collection, claimed by a conditional update with majority write concern and renewed by its owner | A document per server in `extenddb_servers` that expires 45 s after its last renewal | + + On SQLite `.lock` is never removed, also not after a refused `serve` + or `migrate --yes`. `flock` locks the open file: if one process removed + the file while another held it, a third could create and lock a new file + of the same name, and the two holders would not exclude each other. + + The Cassandra and MongoDB rows are the design those backends implement + when they move onto the runner; their intervals and TTLs may change then. + + On PostgreSQL there are two layouts in which the serving key does not + match the servers that run, and the upgrade manual's step that stops + every server before an upgrade covers both. Behind PgBouncer, the serving + session's keepalives (`tcp_keepalives_idle` 10, `tcp_keepalives_interval` + 5, `tcp_keepalives_count` 4, so 30 s) run on the database's connection to + PgBouncer, which stays up when a serving host dies: the dead host's key + stays until PgBouncer closes its connection, which by PgBouncer's + defaults (the operating system's, two hours idle on Linux) takes hours, + and PgBouncer's `tcp_keepalive = 1`, `tcp_keepidle = 10`, + `tcp_keepintvl = 5` and `tcp_keepcnt = 4` give the same 30 s. After a + failover of the catalog's server, the new primary holds no serving key, + while a server that served through the old primary goes on serving until + its registration counts as lost, at most 30 s after its last renewal, so + a migration can run beside it for up to 30 s. + + `migrate --yes` waits up to `--lock-timeout` (60 s by default) for the lock + and for running servers to stop, then exits 6. A server that finds a + migration holding the lock after it registers exits 6, so a restart loop + retries. The `migrate` preview and `verify` take no lock and write + nothing, so they run beside a server. + + Cassandra uses `SERIAL` rather than the `LOCAL_SERIAL` the crate uses + elsewhere. With two datacenters of one node each and the link between them + cut, `INSERT ... IF NOT EXISTS USING TTL 60` at `LOCAL_SERIAL` succeeded in + both datacenters for two different owners in a measured run, and at + `SERIAL` it was refused. The Cassandra and MongoDB locks live in the + existing `settings` table and collection because a lock table that the + migrator creates would be created by two migrators at once, and AtlasDB + found that two Cassandra nodes creating the same table concurrently can + lose data after a restart + ([AtlasDB ADR-0002](https://palantir.github.io/atlasdb/html/miscellaneous/doc/adr/0002-prevent-tables-from-being-creating-simultaneously-in-cassandra-via-a-locks-table.html)). + +6. One catalog status decides what each command does. `Runner::status` reads + the ledgers and writes nothing, and `serve`, `migrate` and `verify` act on + its result: + + | Status | Meaning | Exit | + |---|---|---| + | `Current` | Every migration the binary carries is recorded in every store it touches | 0 | + | `AheadCompatible` | The catalog ledger has ids above the binary's, all `additive` | 0, with a warning | + | `Behind` | Migrations the binary carries are missing; a catalog with no ledger is behind at migration 1 | 3 | + | `AheadIncompatible` | The catalog ledger has an id above the binary's that is `breaking` | 4 | + | `NeedsAttention` | A pending code body's `detect` returned items that a person must fix | 5 | + | `Changed` | A recorded checksum differs from the binary's body | 5 | + | `WrittenByOlderRelease` | The catalog ledger has a row for migration 1, and a release before 1.0 wrote to the catalog afterwards | 5 | + | `Uninitialized` | The store holds no catalog | 1 in a production build | + + Exit 1 also means an error, 2 a usage error, and 6 a lock or a running + server that outlasted the timeout. `serve` reads the status before the + backend creates its engine, because creating the engine writes to the + catalog on several backends. It serves `Current` and `AheadCompatible`. + It refuses every other status with that status's exit code and a message + that lists the pending migrations with their kinds, says what to back up + on that backend, and gives the `migrate` command. A dev build applies an + `Uninitialized` or `Behind` catalog at startup through the same runner and + refuses the other statuses as a production build does. + + A binary serves a catalog ahead of it only when every id it does not know + is additive, so a binary can be rolled back across additive migrations + without a restore. The reader of `kind` ships in 1.0, because the check + runs in the older binary. + +7. No switch serves a catalog whose status forbids it. A binary on a catalog + behind it sends statements to tables and columns that do not exist, which + is the `table_class` failure above. A binary on a catalog with a breaking + migration it does not know writes rows in the shape that migration + replaced. Each case such a switch would cover has another path. After + additive migrations, run the older binary. After a breaking migration, + restore the backup taken before it. For development, use a dev build, + which migrates at startup. For a catalog an older release wrote to, run + `extenddb migrate --yes --reconverge`, which runs every convergent + migration again, skips a recorded one that is not convergent and prints a + line that names it, and rewrites the version row. + +## Rationale + +- Option 2 lost because one change then has a different number on each + backend, and no number says what a deployment has. ExtendDB's backends + already differed this way: PostgreSQL's catalog file 003 is Cassandra's + V005. In Temporal v1.32.1 the manifest "Adds tasks_v2 table for fairness + tasks" is v1.13 on Cassandra, v1.18 on MySQL and PostgreSQL and v0.10 on + SQLite. In Cassandra Reaper 2.3.1 the "percent repaired schedule" change is + 028 on Cassandra, 004 on Astra and V21_0_0 on PostgreSQL and H2. With + separate sequences the compiler cannot reject a backend that lacks a change, + and `migrate --list`, the status and the upgrade manual cannot name a + migration the same way on every backend. Reaper removed its PostgreSQL and + H2 backends in 3.0.0, and the issue that proposed it gave the reason: "It + often resulted in features that worked fine with a specific backend and were + buggy with others" + ([cassandra-reaper#1120](https://github.com/thelastpickle/cassandra-reaper/issues/1120)). +- Option 3 lost because the diff tools surveyed (Atlas, Skeema, sqldef) + support SQL databases only, so Cassandra and MongoDB would still need + ordered migrations; every tool surveyed that migrates MongoDB or Cassandra + uses ordered migrations with a ledger in the store. A computed diff also + cannot tell a rename from a drop followed by an add: Skeema refuses renames, + and Atlas and sqldef need an annotation in the schema to plan one. A diff + has no place for a data change, and migrations 2 and 3 are data changes. + Atlas's documentation recommends versioned files "if you're shipping + software that customers install and manage themselves" ([Atlas, declarative + vs versioned](https://atlasgo.io/concepts/declarative-vs-versioned)). + SQLite's re-applied file showed the same limit inside ExtendDB: a file that + creates only what is missing never changes a table that exists. +- Option 4 lost because it migrates SQL databases only, so of ExtendDB's + backends it covers PostgreSQL and SQLite, and ExtendDB would still have + three migration mechanisms: sqlx's, and one runner each for Cassandra and + MongoDB. It also numbers migrations per migrator and checksums SQL text + only, and it has no code bodies with a read-only `detect` step, no + per-migration kind for an older binary to read, and no way to see a running + server. +- Option 5 lost because deployments created by every 0.1.x release hold data + that an upgrade has to keep. ADR-0003 accepted the loss only while no + deployment depended on a catalog, and that condition no longer holds. + +## Consequences + +- A contributor who changes a catalog adds an id to `migration_ids!` and an + arm for it in the table of every backend on the runner: a file, a code body + with a golden test, or `NotApplicable` with a reason. The contributor + declares the kind by the rule in CONTRIBUTING.md and commits the + current-schema file that the backend's CI job regenerates. A change to + per-table objects also changes the DDL that creates new ones, in the same + PR, and is breaking. Released bodies are not edited, and the release PR + pastes the checksum literals. The procedure is in the + [catalog migration section of CONTRIBUTING.md](../../CONTRIBUTING.md#adding-a-catalog-migration). +- An operator upgrades by stopping the server, backing up the stores the + backend uses, running `extenddb migrate` to see what is pending (exit 3 + when something is), running `extenddb migrate --yes`, and starting `serve`. + On SQLite `migrate --yes` copies the database beside it before its first + change. A run that commits nothing removes its copy, and once its copy is + complete a run removes an earlier copy that has the same bytes as its + own, which a run killed before it committed anything leaves; every other + copy stays until the operator removes it. `extenddb migrate --list` prints every + migration the binary carries with its kind on that backend, `verify` prints + the status, and `serve` refuses with the status's exit code and the command + to run. +- Rollback after additive migrations runs the previous 1.x binary. Rollback + after a breaking migration restores the backup taken before the upgrade. + All four v1.0 migrations are breaking, so rolling back from 1.0 is a + restore. +- Releases before 1.0 refuse a 1.0 catalog or leave a trace in it. PostgreSQL + `serve` refuses the integer version row, and v0.1.14 `serve` on every + backend refuses a catalog without the `resource_tag_migration` row, which + migration 1 deletes. A released `migrate --yes` runs no legacy file and + inserts the marker row again, and SQLite dev builds from v0.1.7 to v0.1.14 + write a dotted version back at startup. 1.0 reports either trace as + `WrittenByOlderRelease` (exit 5), and `extenddb migrate --yes --reconverge` + repairs it. Measured on SQLite with the v0.1.14 dev image by + `devtools/check-sqlite-older-binary`: after the image's `migrate --yes` or + its `serve` ran on a migrated catalog, 1.0 reported + `WrittenByOlderRelease`, and `--reconverge` restored `Current` with the + schema the catalog had before. The repair does not run migration 2 again, + so a policy that a SQLite dev build from v0.1.7 to v0.1.13 wrote after the + upgrade keeps its `dynamodb:ResourceTag` keys: that build evaluated them + against the table's tags, and 1.0 leaves them absent. +- A 0.1.x server that is still running while migration 1 adopts its catalog + is not detected, because released servers do not register. On SQLite, + servers from v0.1.14 on hold the `flock` that `migrate` takes, so only older + servers are not detected. The `migrate` preview, the refusal message and + the upgrade manual tell the operator to stop every server before the 1.0 + upgrade. +- `migrate --yes` refuses while a server serves, so every 1.x upgrade stops + the server first and no server runs while its catalog changes. The kind of + each migration still matters for rolling a binary back. +- The tests of each backend on the runner load committed fixtures of every + released state, upgrade them, and compare each with a fresh install seeded + the same way, per-table objects included, against a list of accepted + differences per state. GitLab's CI does the same with a seeded PostgreSQL + dump of its latest required upgrade stop. +- Every backend carries an entry for every id, including a reason for a + change that does not apply to it. Code bodies may call application code, + such as `ManagementStore` in migration 2. Each keeps its golden test (SQLite's + migration 1 is checked against its expected definitions file instead) for + as long as the body ships, so a change in that code that alters a body's output + fails the test. +- On Cassandra a migration needs every replica of the catalog keyspace + reachable, because the lock is taken at `SERIAL` and running servers are + read at `ALL`. When Cassandra moves onto the runner, + the runner is to wait for schema agreement after every DDL statement, reading `schema_version` from `system.local` on each node. +- SQLite and PostgreSQL run on the runner first. Cassandra and MongoDB keep + their bootstrapper migrations and the marker check in `serve.rs` until they + move onto it, and each backend's previous mechanism is deleted when it + moves. +- The shared numbering and the ledger in every store are read by every later + release, so reversing this decision means migrating every deployment's + ledgers. A released body changes only through a `rebless` entry. + +--- + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is +a trademark of Amazon.com, Inc. diff --git a/docs/adr/README.md b/docs/adr/README.md index 2deec1259..06108fe20 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -32,7 +32,7 @@ decision, write a new ADR. |---|-------|--------| | [0001](0001-documentation-format.md) | Documentation format — Markdown over LaTeX | Accepted | | [0002](0002-sql-injection-defense.md) | SQL injection defense | Accepted | -| [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Accepted | +| [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Superseded by [ADR-0023](0023-catalog-migrations.md) | | [0004](0004-vector-search-exact-scan.md) | Vector search is an exact scan over one row per vector | Accepted | | [0005](0005-index-build-lifecycle-ownership.md) | Index-build lifecycle stays in the backend until a second backend needs it | Accepted | | [0006](0006-pgvector-storage-and-scoring.md) | Vector storage on PostgreSQL uses pgvector's type, and the engine decides what it cannot compute | Accepted | @@ -48,4 +48,5 @@ decision, write a new ADR. | [0019](0019-cassandra-logical-backup-restore.md) | Cassandra: logical backup and restore | Accepted | | [0020](0020-cassandra-occ-update-item.md) | Cassandra: optimistic concurrency control for UpdateItem and PutItem | Accepted | | [0021](0021-cassandra-lwt-delete-atomicity-gap.md) | Cassandra: LWT delete atomicity gap and required worker/saga pattern | Accepted | -| [0022](0022-trust-policy-deny-over-allow.md) | Role trust-policy evaluation follows IAM Deny-over-Allow order | Accepted | \ No newline at end of file +| [0022](0022-trust-policy-deny-over-allow.md) | Role trust-policy evaluation follows IAM Deny-over-Allow order | Accepted | +| [0023](0023-catalog-migrations.md) | One list of catalog migrations for every backend, with a ledger in every store | Accepted | \ No newline at end of file diff --git a/docs/build-docs.py b/docs/build-docs.py index 6c57d02b5..4b76c6b99 100755 --- a/docs/build-docs.py +++ b/docs/build-docs.py @@ -131,12 +131,14 @@ def get_extenddb_version() -> str: def get_catalog_version() -> str: - """Extract CATALOG_VERSION from storage-postgres lib.rs.""" - lib_rs = ROOT / "crates" / "storage-postgres" / "src" / "lib.rs" - text = lib_rs.read_text() - m = re.search(r"CatalogVersion::new\((\d+),\s*(\d+),\s*(\d+)\)", text) - if m: - return f"{m.group(1)}.{m.group(2)}.{m.group(3)}" + """The catalog the manuals describe: `migration `, the last id of the + `migration_ids!` list in crates/storage/src/migrations/registry.rs, as + `extenddb version` prints it on PostgreSQL and SQLite.""" + registry = ROOT / "crates" / "storage" / "src" / "migrations" / "registry.rs" + text = registry.read_text() + ids = [int(n) for n in re.findall(r"^\s*[A-Z][A-Za-z0-9]* = (\d+), \"", text, re.MULTILINE)] + if ids: + return f"migration {max(ids)}" return "unknown" diff --git a/docs/getting-started.md b/docs/getting-started.md index 8632ad33d..91ec7a27d 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -265,12 +265,15 @@ You should see all checks pass: ``` === extenddb verify === +Config: extenddb.toml +Catalog database: extenddb_catalog + +--- Checking catalog migrations... + OK: Current: the catalog is at migration 4 --- Checking catalog connection... OK: Connected to catalog. ---- Checking catalog version... - OK: Catalog version 0.0.4 --- Checking data database... - OK: Connected to data database 'extenddb_catalog_data'. + OK: Connected to data database 'extenddb'. --- Enumerating tables... Tables: 0 Indexes: 0 @@ -284,7 +287,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.1.14 (catalog 0.0.4) starting on 127.0.0.1:18443 +# extenddb 0.1.14 (catalog migration 4) starting on 127.0.0.1:18443 # storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` @@ -1363,7 +1366,7 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog 0.0.4 (postgres) +# catalog migration 4 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index bd05b6552..4978c3668 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -205,7 +205,7 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -The catalog version is 0.0.4 on PostgreSQL, SQLite, and Cassandra. PostgreSQL and SQLite store it in the `settings` table under the key `catalog_version` and check it at startup; Cassandra has no startup check (see the Upgrade Manual). +On PostgreSQL and SQLite the catalog is versioned by its migration ledger, the `extenddb_migrations` table: the catalog version is the highest migration recorded in every store, migration 4 for this release, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. The Cassandra catalog version is 0.0.4, stored in the `settings` table under the key `catalog_version`; Cassandra has no startup check (see the Upgrade Manual). The MongoDB backend tracks its own catalog version, 0.0.2. A mismatch between the compiled-in version and the stored one prevents a PostgreSQL or SQLite server from starting; run `extenddb migrate` to upgrade. diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index 0f57420d8..5cd1bd682 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -130,7 +130,7 @@ Check the version: ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog 0.0.4 (postgres) +# catalog migration 4 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -168,12 +168,15 @@ Expected output: ``` === extenddb verify === +Config: extenddb.toml +Catalog database: extenddb_catalog + +--- Checking catalog migrations... + OK: Current: the catalog is at migration 4 --- Checking catalog connection... OK: Connected to catalog. ---- Checking catalog version... - OK: Catalog version 0.0.4 --- Checking data database... - OK: Connected to data database 'extenddb_catalog_data'. + OK: Connected to data database 'extenddb'. --- Enumerating tables... Tables: 0 Indexes: 0 @@ -403,7 +406,7 @@ This drops both databases after confirmation. | Symptom | Fix | |---------|-----| | `connection refused` on port 18443 | Server not running. Start with `extenddb serve --config extenddb.toml` | -| `CATALOG_VERSION_MISMATCH` | Run `extenddb migrate --config extenddb.toml` | +| `serve` exits 3, naming pending catalog migrations | Run `extenddb migrate --config extenddb.toml --yes`, then start the server | | `ResourceNotFoundException` on CreateTable | Table is still in CREATING status. Poll DescribeTable until `TableStatus` is `ACTIVE` | | `UnrecognizedClientException` | Invalid access key. Check credentials — verify the access key ID and secret match what was returned by `create-access-key` | | `AccessDeniedException` | IAM policy does not allow the operation. Attach a policy with the required permissions | diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index a0cf56e7e..1c2589181 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -923,14 +923,9 @@ Error: error communicating with database Check that PostgreSQL is running and the connection string in `extenddb.toml` is correct. -**Catalog version mismatch:** +**The catalog has pending migrations, or is ahead of the binary:** - -``` -Error: catalog version mismatch: found 1.0.0, expected 0.0.4 -``` - -Run `extenddb migrate --config extenddb.toml` to upgrade the catalog schema. The check is exact equality in both directions, so this also appears when a binary meets a catalog a newer build already migrated; in that case upgrade the binary rather than the catalog. `extenddb migrate` refuses a catalog newer than the binary (`catalog version X is newer than this binary's Y`) instead of stamping the older version onto it. See the Upgrade Manual for the version history and the stop / migrate / start sequence. +On PostgreSQL and SQLite, `serve` reads the catalog's migration status before it opens the engine and refuses a catalog it cannot serve, printing the status and the command that resolves it. A catalog with pending migrations exits 3, for example `the catalog is at migration 3, and ExtendDB needs migration 4`, followed by the pending migrations and `extenddb migrate --config extenddb.toml --yes`. A catalog that a newer release migrated with a breaking migration exits 4 and names that release's migrations; upgrade the binary rather than the catalog. The [Upgrade Manual](07-upgrade-manual.md#the-catalog-status) lists every status with its exit code and the stop / migrate / start sequence. On Cassandra and MongoDB the catalog checks are those of the 0.1.x releases; their sections of the Upgrade Manual describe them. ### Authentication Errors diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index f4451a0f5..17ddf280c 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,124 +4,46 @@ ## Current Status -Catalog 0.0.4 is current. **Every existing PostgreSQL, SQLite, and Cassandra deployment must run `extenddb migrate`** to reach it. PostgreSQL and SQLite servers refuse to start against a catalog version they were not built for; a Cassandra server starts anyway and serves tables and items, but CreateBackup, RestoreTableFromBackup, and DeleteBackup are refused with a `ValidationException` naming `extenddb migrate --yes` until the catalog is migrated (see [Cassandra](#cassandra)). MongoDB's catalog version does not change in this release. +This release versions the PostgreSQL and SQLite catalogs by a migration ledger and carries catalog migrations 1 to 4, which `extenddb migrate --list` prints. **Every PostgreSQL and SQLite deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. Cassandra stays at catalog 0.0.4 and MongoDB at 0.0.2, and their sections below apply unchanged. -See [Catalog 0.0.4](#catalog-004-current) below for what changes and the exact sequence. +A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. + +The [general upgrade procedure](#general-upgrade-procedure) gives the sequence. The [version history](#version-history) records the 0.1.x catalog versions and what the 0.1.x binaries print. ## How Catalog Upgrades Work ### The Migration System -Migrations are SQL files in `crates/storage-postgres/migrations/`, applied in filename order: - -``` -001_schema.sql ← the complete initial schema -002_vector_indexes.sql ← vector index metadata, catalog 0.0.3 -003_backup_definitions.sql ← backup table definitions, catalog 0.0.4 -``` - -The `schema_history` table tracks which files have been applied. When `extenddb migrate` runs, it: - -1. Reads all migration files embedded in the binary (via `include_str!`) -2. Checks `schema_history` for each filename -3. Applies any unapplied migrations in order -4. Records each applied filename in `schema_history` +Every backend shares one numbered list of catalog migrations, in `crates/storage/src/migrations/registry.rs`. For each id a backend declares the body it runs in each of its stores, a SQL file under `crates/storage-/migrations/` or code, or why the id does not apply to it. `extenddb migrate --yes` applies the pending bodies in id order, catalog store first, and commits each body with a row in that store's ledger, the `extenddb_migrations` table: the id, name, kind, checksum, notes, time, and the release that applied it. On PostgreSQL the catalog database and the data database each have a ledger; on SQLite the database file has one. -Running `extenddb migrate` on an up-to-date catalog is a no-op. +The catalog's version is the highest id such that it and every lower id are recorded in every store its bodies touch. The `catalog_version` row in `settings` holds the same integer, written with each catalog row, for tools that read it. The `schema_history` tables of 0.1.x PostgreSQL catalogs stay in place and are not read. -### The Catalog Version +Running `extenddb migrate --yes` on an up-to-date catalog changes nothing. -A single row in the `settings` table stores the catalog version: +### The Catalog Status -```sql -SELECT value FROM settings WHERE key = 'catalog_version'; --- '0.0.2' -``` +`serve`, `migrate` and `verify` read the catalog's status from the ledgers, take no lock to do it, and act on it with the same exit codes: -The binary embeds an expected catalog version (`CATALOG_VERSION` constant in `crates/storage-postgres/src/lib.rs`). At startup, the server compares the database value against the binary's expectation. If they don't match, the server refuses to start and directs the operator to run `extenddb migrate`. +| Status | Meaning | `serve` | `migrate` | `migrate --yes` | `verify` | +|--------|---------|---------|-----------|-----------------|----------| +| Current | Every migration this binary carries is recorded | serves | 0 | 0 | 0 | +| Behind | Migrations are pending | 3 | 3 | applies them, 0 | 3 | +| AheadCompatible | The catalog has migrations this binary does not know, all additive | serves, with a warning | 0 | 0 | 0 | +| AheadIncompatible | The catalog has a migration this binary does not know that is breaking | 4 | 4 | 4 | 4 | +| NeedsAttention | A pending migration found data that a person must fix; each item is named | 5 | 5 | 5 | 5 | +| Changed | A recorded checksum differs from this binary's body | 5 | 5 | 5 | 5 | +| WrittenByOlderRelease | A 0.1.x release wrote to the catalog after it was migrated | 5 | 5 | 5; `--yes --reconverge` repairs it | 5 | +| Uninitialized | Not a catalog; `extenddb init` creates one | 1 | 1 | 1 | 1 | -### Version Semantics +Exit status 6 means the migration lock, or a running server, was still held when `--lock-timeout` ran out; run the command again once it is free. Every refusal prints the command that resolves it. A dev build's `serve` applies a Behind or Uninitialized catalog at startup instead of refusing it. The `serve` codes are those of `serve --foreground`; in daemon mode `serve` exits 1 and writes the refusal to syslog. -The catalog version follows semantic versioning: +`verify` also checks the connection to the data database. When that check fails, `verify` exits 1 on a Behind catalog instead of 3, because `migrate --yes` cannot apply the pending migrations while the data database cannot be reached. It still exits 4 or 5 for the statuses that need a person. -- **MAJOR**: Breaking schema changes that may require data migration or downtime -- **MINOR**: New tables or columns (backward-compatible, additive) -- **PATCH**: Index changes, constraint fixes, seed data updates +A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 4 are breaking: do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. `migrate --yes --reconverge` repairs it by running the convergent migrations again. Migration 2, which rewrites stored policies, is not convergent: the repair skips it and prints a line that says so (see the v0.1.14 notes below). ## Writing a New Migration -When you need to change the catalog schema, here's the process: - -### 1. Create the migration file - -Add a new SQL file with the next sequence number: - -``` -crates/storage-postgres/migrations/002_your_feature.sql -``` - -The file should be a single transaction: - -```sql --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Migration 002: Brief description of what this adds/changes. - -BEGIN; - --- Your DDL here. -ALTER TABLE tables ADD COLUMN IF NOT EXISTS new_column TEXT; - --- Bump the catalog version. -UPDATE settings SET value = '0.1.0' WHERE key = 'catalog_version'; - -COMMIT; -``` - -### 2. Register it in the migration runner - -Add the file to `CATALOG_MIGRATIONS` in `crates/storage-postgres/src/migrations.rs`: - -```rust -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ - ( - "001_schema.sql", - include_str!("../../storage-postgres/migrations/001_schema.sql"), - ), - ( - "002_your_feature.sql", - include_str!("../../storage-postgres/migrations/002_your_feature.sql"), - ), -]; -``` - -### 3. Bump the catalog version constant - -In `crates/storage-postgres/src/lib.rs`: - -```rust -pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 1, 0); -``` - -This must match the version written by your migration's `UPDATE settings` statement. - -### 4. Update 001_schema.sql - -The consolidated schema file is what fresh installs get. Add your new column/table/index to `001_schema.sql` as well, and update its `INSERT INTO settings` to seed the new version. This way fresh installs get the final schema in one pass, while existing deployments get there via the incremental migration. - -### Design Considerations - -**Idempotency.** Use `IF NOT EXISTS`, `IF EXISTS`, and `ADD COLUMN IF NOT EXISTS` so migrations can be safely re-run. - -**Backward compatibility.** Prefer additive changes (new columns with defaults, new tables) over destructive ones (dropping columns, renaming tables). A running server on the old binary should survive the schema change until it's restarted with the new binary. - -**Transaction boundaries.** Wrap each migration in `BEGIN`/`COMMIT`. If any statement fails, the entire migration rolls back and the catalog stays at the previous version. - -**No data migrations in DDL files.** If a schema change requires backfilling data, do it in Rust code triggered by `extenddb migrate`, not in raw SQL. This gives you error handling, progress reporting, and the ability to batch large updates. - -**Test both paths.** Every migration must be tested two ways: -1. Fresh install (`extenddb init`) — verifies `001_schema.sql` is correct -2. Upgrade (`extenddb migrate` on a catalog at the previous version) — verifies the incremental migration works +Writing a catalog migration is contributor work. The procedure, with a worked example and the tests to run on each backend, is in the [catalog migration section](../../CONTRIBUTING.md#adding-a-catalog-migration) of `CONTRIBUTING.md` in the repository root. `extenddb migrate --list` prints the migrations a PostgreSQL or SQLite binary carries, with each one's kind on its backend; a Cassandra or MongoDB binary refuses `--list`, because those backends still migrate through their bootstrappers. ## General Upgrade Procedure @@ -152,24 +74,35 @@ cargo build --release 4. **See what is pending, then apply it** ```bash -export EXTENDDB_PG_PASSWORD='' -extenddb migrate --config extenddb.toml --pg-user -extenddb migrate --config extenddb.toml --pg-user --yes +extenddb migrate --config extenddb.toml +extenddb migrate --config extenddb.toml --yes ``` -`migrate` connects as a PostgreSQL admin role, not as the application role in the config file. Without `--pg-user` it tries the operating-system user name, and without `EXTENDDB_PG_PASSWORD` it sends no password; the error then names the role it tried, for example `Error: Cannot connect to PostgreSQL at 127.0.0.1:5432 as admin user "alice": error returned from database: password authentication failed for user "alice". Pass the admin role with --pg-user and its password in the EXTENDDB_PG_PASSWORD environment variable.` +`migrate` connects as the application role in the config file, which does all the schema work. On PostgreSQL that role must own every table, view and sequence in the public schema of both databases, and may create tables there, as it does after `init`. A catalog restored by another role (for example from a `pg_dump --no-owner` dump loaded by an admin) is not owned by it, and such a dump carries no grants, so the role cannot read the restored tables either: the preview, `verify` and `migrate --yes` exit 1 with the database's `permission denied for table settings` followed by the catalog database's note. The data database's name is a row of the catalog's `settings`, which the role cannot read, so the commands do not guess it and print a line saying so after the note. Run the catalog database's statements first, and the next run prints the data database's note, or pass `migrate --data-db ` to see both notes at once. When the role can read the tables but does not own them, the preview and `verify` print a note per database, and `migrate --yes` refuses with exit status 1 before it takes the lock, printing one note per line. Each note names the objects the role does not own, with their owners, and the statements to run in that database, numbered in the order they run, each with the role that runs it. For each owner a `DO` block, run by that owner or by a superuser, changes the owner of that owner's objects with `ALTER ... OWNER TO` and of nothing else. Where no superuser exists, as on a managed service, PostgreSQL lets an owner give an object only to a role it is a member of, and only when that role may create objects in the object's schema. Call the role that owns the restored objects ``. When it is not a superuser, the note first prints `GRANT "" TO ""`, run by an administrator that may grant the application role (on PostgreSQL 16, a role with `ADMIN OPTION` on it, which the role that created it holds; the application role cannot grant itself to ``). Then it prints `GRANT CREATE ON SCHEMA public TO ""` when the application role may not create tables there, run by the public schema's owner, then the `DO` blocks, and last `REVOKE "" FROM ""`, which takes back the membership the `DO` blocks needed. Skip that last statement if `` was already a member of ``. A superuser can run every statement as printed. The note does not use `REASSIGN OWNED BY`, which would also give the role the old owner's other databases and its objects in other schemas. A migration that still fails on a privilege (`must be owner of table ...`) stops, commits nothing in that database, and prints the same note. `--pg-user `, with the password in `EXTENDDB_PG_PASSWORD`, names an admin role for one step only, creating the pgvector extension in the data database, on that database's server; without it `migrate --yes` reports whether pgvector is available and goes on. -`migrate` finds the data database through the name `init` recorded in the catalog's `settings` (`data_database_name`), so a deployment initialised with `init --data-db ` migrates without further flags; when that recorded name differs from the one derived from the catalog database name, `migrate` prints it (`--- Data database '' (recorded in the catalog's settings; ...)`), and a deployment with the usual name sees no such line. A catalog that has no recorded name (one initialised by a build that did not record it) falls back to the catalog database name without its `_catalog` suffix; if the data database is named otherwise, pass `--data-db ` to `migrate`, which also overrides a recorded name. A wrong name fails before anything is written: `Error: Cannot connect to : error returned from database: database "" does not exist`. +`migrate` finds the data database through the name `init` recorded in the catalog's `settings` (`data_database_name`), and connects to it with the connection string `init` recorded beside it (`data_database_connection_string`), which the server connects with. A catalog that has no recorded name falls back to the catalog database name without its `_catalog` suffix. `--data-db ` overrides both and uses that database on the recorded connection string's server. When the two rows name different databases and `--data-db` is not given, `migrate`, `verify` and `serve` refuse with exit status 1 and name both; correct the row that is wrong. -Without `--yes`, `migrate` prints what is pending and changes nothing. Its exit status says what it found: +The PostgreSQL commands and `serve` work through PgBouncer in transaction mode and in session mode: no session sets a parameter at login other than `application_name`, which PgBouncer passes on to the server, and every other setting is sent inside the transaction that needs it. -| Exit status | Meaning | -|-------------|---------| -| 0 | Nothing is pending (or, with `--yes`, everything pending was applied) | -| 3 | Work is pending; run again with `--yes` to apply it | -| 1 | An error: the connection failed, the catalog is newer than this binary or unreadable, or a migration failed | +`migrate --yes` finds running servers through the key each one holds on the catalog database while it serves. In two layouts the key does not match the servers that run, and stopping every server before the upgrade (step 1) covers both: -A script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. +- Behind PgBouncer, the TCP keepalives the serving session sets (30 s) run on the database's connection to PgBouncer, which stays up when a serving host dies. The database keeps that host's key until PgBouncer closes the dead host's connection, which PgBouncer's own keepalive settings decide: by default the operating system's, two hours idle on Linux. Until then `migrate --yes` exits 6 naming the dead server. Set `tcp_keepalive = 1`, `tcp_keepidle = 10`, `tcp_keepintvl = 5` and `tcp_keepcnt = 4` in PgBouncer's `[pgbouncer]` section for the same 30 s, or end the session with `SELECT pg_terminate_backend()` on the pid `migrate` names. +- After a failover of the catalog's server, the new primary holds no server's key, while a server that served through the old primary goes on serving through connections that reconnect to the new primary, until it counts its key as lost, at most 30 s after its last renewal, and registers again on the new primary. A `migrate --yes` in those 30 s does not see that server. Wait 30 s after a failover before you run `migrate --yes`, or stop the servers first. + +A server notices at its next renewal, within 15 s, that the database ended the session holding its key: a restart or failover of the catalog's server, `SELECT pg_terminate_backend()`, or a pooler that closed the connection. From then on it answers each new request, `/health` included, with `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and lets the requests already running finish. It tries to register again on a new session at once and then every second, and serves again once it holds the key and the catalog's status is the one it started with. When a migration holds the migration lock or waits for the servers to stop, when the status changed, or when 30 s pass without a registration, the server stops its background workers at once and then stops the way SIGTERM stops it, finishing the requests in flight and removing its PID file, and exits 1. Until the renewal that finds the session gone, the server serves as before, so a `migrate --yes` that was waiting for the servers can start beside it in those seconds: stop every server before you migrate, as the steps above say. + +The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. + +Without `--yes`, `migrate` reads the [catalog status](#the-catalog-status), prints each pending migration, and changes nothing; it takes no lock, so it runs beside a server. While migration 2 is pending, the preview also prints the number of stored policies migration 2 rewrites and names the first 20 of them, each with its account, principal, policy name and number of keys, followed by the number of the others. To list them, it reads the catalog's policies a second time. With `--yes` it takes the migration lock, waiting up to `--lock-timeout` seconds (60 by default) for running servers and other migrations, reads the status again under the lock, and applies what is pending. It refuses a store that is not a catalog with exit status 1 and the advice to run `extenddb init`. The exit statuses are those in the status table; a script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. + +On PostgreSQL `migrate --yes` can also print: + +- `Waiting up to s for the statements of an earlier migrator that stopped`: a migrator that was killed left a statement running in a database. The run waits until it ends, and after `--lock-timeout` exits 6 naming each session with `SELECT pg_cancel_backend()` to end it; a statement that waits for a lock does not end by itself. The run looks for them in each database through its own connection to it, so a data database on another server is covered. +- A line naming an index statement and the sessions it waits for: migration 4 builds its indexes with `CREATE INDEX CONCURRENTLY`, which waits for every transaction on the table. After 5 s it cancels the build and tries again, three times, and then stops naming the sessions; end them and run again. +- `could not lock a table ... within 5 s, 4 times`: other sessions held locks on the tables a migration changes, which it names. Nothing of that migration was committed; stop those sessions and run again. +- `after migration 1 the database differs from the v1.0 schema in place(s)`: the first run of migration 1 on that database found objects that differ from the v1.0 schema, such as a column added by hand or a changed default, and committed nothing there. Each following line names an object and its table, what the database has, what v1.0 has, and the change to make. Make each change, then run `migrate --yes` again. + +On PostgreSQL one `--lock-timeout` bounds every wait of a run together: the wait for another migrator's lock, the wait for running servers, and the wait for an earlier migrator's statements. Each wait gets the time the earlier ones left, and its line prints the seconds left. With `--lock-timeout 0` a run makes one attempt at each: it exits 6 at once when another migrator or a running server holds its key, or when an earlier migrator's statement is still running. 5. **Verify** @@ -177,6 +110,8 @@ A script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq extenddb verify --config extenddb.toml ``` +`verify` prints the catalog status, and on a Behind catalog the policies migration 2 rewrites, as the preview does. For each migration that recorded notes in the ledger, such as the policies migration 2 rewrote, it prints the number of notes and the first 5; `verify --all-notes` prints every note. + 6. **Start the server** ```bash @@ -202,7 +137,9 @@ To discard data written after the upgrade as well, restore `` from its ## Version History -### Catalog 0.0.4 (Current) +The catalog versions below are those of the 0.1.x releases, and the commands, output and procedures in them are those of the 0.1.x binaries. This release upgrades the catalog of any 0.1.x release with the [general upgrade procedure](#general-upgrade-procedure): its migration 1 applies the changes each version below made. + +### Catalog 0.0.4 (the last 0.1.x catalog) Adds the table definition a backup records and the registration of a running restore: @@ -235,7 +172,7 @@ On its first control-plane pass after the upgrade (at startup), the server remov #### SQLite -The database is one file plus its `-wal` and `-shm` files beside it. Stop the server before copying them: `migrate` and `serve` take the same exclusive lock on `.lock`, so `migrate` refuses to run while a server is up (`this command needs exclusive use of the SQLite database and cannot have it: another extenddb process is already using ...`). A stopped server can leave a non-empty `-wal` file, so copy all three. +The database is one file plus its `-wal` and `-shm` files beside it. Stop the server before copying them: `migrate --yes` and `serve` take the same exclusive lock on `.lock`, so while a server is up `migrate --yes` waits for up to `--lock-timeout` seconds (60 by default) and then exits 6 (`another extenddb process is already using ...`). `migrate` without `--yes` takes no lock, writes nothing, and runs beside a server. Before its first change `migrate --yes` copies the database to `.pre-migration--to--`, readable by its owner only (mode 0600) whatever the database's mode, and prints the copy's path and the database's size before it copies and the time the copy took after. It takes the copy with SQLite's online backup API, which reads the database file and the committed frames of its WAL in one read transaction, so the copy holds every committed write, and its time grows with the database's size. The copy is in rollback-journal mode, so it can be read without a `-shm` file beside it, also from a directory the reader may not write to. A run that stops before it commits anything removes its copy, since the database is unchanged; a run that stops after a step committed keeps its copy and names it in its error. A run that is killed after its copy is complete and before it commits anything leaves the copy; the next run whose own copy has the same bytes removes it once that copy is complete, and names it, also in its error when that run then stops before its first change, unless a `-wal` or `-journal` file beside it shows that a process opened it or the file is a link to the database. Every other copy stays until you remove it. `.lock` stays beside the database after every run and server, also after a refused `serve` or `migrate --yes`. Do not remove it while an ExtendDB process may have it open: a process that then creates and locks a new file of that name does not exclude the process that holds the old one. A stopped server can leave a non-empty `-wal` file, so copy all three. ```bash extenddb stop --config extenddb.toml @@ -245,16 +182,20 @@ extenddb migrate --config extenddb.toml --yes extenddb serve --config extenddb.toml ``` -The upgrade rebuilds `backup_items` with a stable restore cursor, copying every row into a new table inside one write transaction; the database is locked for writes until it finishes. The preview names it with its row count, for example `Pending: catalog 0.0.3 -> 0.0.4; data migrations [rebuild backup_items (80 rows)].`, and `--yes` logs its start and end at info: +The upgrade rebuilds `backup_items` with a stable restore cursor, copying every row into a new table inside one write transaction; the database is locked for writes until it finishes. From v1.0 migration 1 makes this rebuild on a database whose `backup_items` has no `id` column. The preview lists migration 1 and does not name the rebuild. `--yes` prints the rebuild as a note under migration 1, and `verify` prints the same note from the ledger: ``` -catalog upgrade: rebuilding backup_items with a stable restore cursor (80 rows); the database is locked for writes until this finishes -catalog upgrade: rebuilt backup_items (80 rows) in 0.005s + Applied 1 baseline + rebuilt backup_items with an id column, copying 80 row(s); the old table's pages stay free inside the database file until a VACUUM ``` +The 0.1.x binaries named the rebuild in the preview, as `Pending: catalog 0.0.3 -> 0.0.4; data migrations [rebuild backup_items (80 rows)].`, and logged its start and end at info. + To roll back, stop the server, put the copied files back in place of the current ones (remove the current `-wal` and `-shm` first), and start the previous binary. -On SQLite, `extenddb serve` refuses to start a file database whose tables are in a pre-release 0.0.4 shape and names the pending step; stop the server and run `extenddb migrate --yes`. +On SQLite, `extenddb serve` reads the catalog's migration ledger before it opens the database and refuses a catalog with pending migrations, exiting 3 under `--foreground` with each pending migration and the `extenddb migrate --config --yes` command to run (in daemon mode it exits 1 and writes the refusal to syslog); a dev build applies them at startup instead, taking the same copy. + +The `extenddb migrate` command that `serve`, `migrate` and `verify` advise names the config file by its absolute path, quoted for a POSIX shell, so it can be pasted as printed. A relative `path` in `[storage.sqlite]`, such as the `extenddb.sqlite` that `init` writes, resolves against the directory a command runs in. So when the config file names the database by a relative `path`, or names none, the advised command starts with `EXTENDDB__STORAGE__SQLITE__PATH=`, and it reaches the same database from any directory. On startup the SQLite server removes any restore target left in `CREATING` by a server that stopped mid-copy, which is only safe because one server at a time can hold the file. A backup keeps a WAL read snapshot open while it copies; checkpoints cannot reclaim pages needed by that snapshot, so allow disk headroom for WAL growth proportional to the backup duration and concurrent write volume. Set `pool_size` to 3 or more (see the Admin Guide); the server runs at most `pool_size / 2 - 1` backups at once (4 with the default of 10) and answers `LimitExceededException` to a `CreateBackup` that waited 30 seconds for a slot. Running backups hold one connection each for their read snapshot plus one shared writer (`n` backups hold `n + 1` connections), so from `pool_size` 4 up a flood of backups leaves at least half the pool to other requests; with a `pool_size` of 2 or 3 the one admitted backup holds two of them. A GSI added by `UpdateTable` with `ProvisionedThroughput` on a `PROVISIONED` table made every `DescribeTable` of that table fail on earlier builds (`Internal("index pt: missing field NumberOfDecreasesToday")`); this build writes the index's throughput in the shape `DescribeTable` reads and reads the shape earlier builds wrote (as the same units with `NumberOfDecreasesToday` 0), so such a table is describable again after the upgrade without any change to the database file. @@ -610,9 +551,9 @@ for any of these changes. **Who is affected.** Any deployment with an IAM policy that uses a `dynamodb:ResourceTag/` condition key or a `${dynamodb:ResourceTag/...}` policy variable: inline user, group and role policies, user and role permissions boundaries, and session policies. A policy that uses `aws:ResourceTag/` is not affected. Role trust policies are not affected: the AssumeRole path never resolved `dynamodb:ResourceTag/`, so a trust policy that names it behaved then as it does now. -**Migration is required.** `extenddb serve` refuses to start on a catalog that has not been migrated, with an error that names `extenddb migrate --yes`. Run `extenddb migrate --yes` once per deployment before starting the new binary. On PostgreSQL, `migrate` connects as the admin user, so pass `--pg-user ` and set `EXTENDDB_PG_PASSWORD`, as for `init`. The migration rewrites every stored `dynamodb:ResourceTag/` condition key and `${dynamodb:ResourceTag/...}` policy variable to the `aws:ResourceTag/` form, keeping the operator, the tag key and the value unchanged. The rewrite preserves each policy's decision, because `aws:ResourceTag/` resolves from the same table tags the old `dynamodb:ResourceTag/` key read. The one difference is a `${dynamodb:ResourceTag/}` policy variable on a table without that tag: neither form expands, and the literal text in the comparison changes from `${dynamodb:ResourceTag/}` to `${aws:ResourceTag/}`. The decision changes only for a request value equal to one of those literals. A freshly initialised deployment needs no migrate step; its catalog is already recorded as migrated. +**Migration is required.** `extenddb serve` refuses to start on a catalog that has not been migrated, with an error that names `extenddb migrate --yes`. Run `extenddb migrate --yes` once per deployment before starting the new binary. On PostgreSQL, the 0.1.14 `migrate` connects as the admin user, so pass `--pg-user ` and set `EXTENDDB_PG_PASSWORD`, as for `init`; from v1.0 the rewrite is catalog migration 2, which `migrate --yes` applies as the application role. The migration rewrites every stored `dynamodb:ResourceTag/` condition key and `${dynamodb:ResourceTag/...}` policy variable to the `aws:ResourceTag/` form, keeping the operator, the tag key and the value unchanged. The rewrite preserves the decision of each policy that a release before 0.1.14 stored, because `aws:ResourceTag/` resolves from the same table tags the old `dynamodb:ResourceTag/` key read. A policy that names `dynamodb:ResourceTag/` and that 0.1.14 stored after its last `migrate --yes` had no effect on 0.1.14, which leaves that key absent; migration 2 rewrites it with the others, so it takes effect after the upgrade to v1.0. The one difference is a `${dynamodb:ResourceTag/}` policy variable on a table without that tag: neither form expands, and the literal text in the comparison changes from `${dynamodb:ResourceTag/}` to `${aws:ResourceTag/}`. The decision changes only for a request value equal to one of those literals. A freshly initialised deployment needs no migrate step: 0.1.14's `init` records the catalog as migrated, and from v1.0 `init` applies migration 2 with the others. -**What `migrate` prints.** Without `--yes`, `migrate` lists each policy it would rewrite, one line per policy with the account, the principal, the policy name and the number of condition keys, and writes nothing. With `--yes`, it rewrites and records that the migration is complete. It runs on every `migrate`, so an interrupted run is finished by the next one. If a policy lookup fails, or a policy needs attention (below), `migrate` reports it, exits non-zero, and does not record completion, so a later run retries. Policy bodies are never printed. +**What `migrate` prints.** Without `--yes`, `migrate` lists each policy it would rewrite, one line per policy with the account, the principal, the policy name and the number of condition keys, and writes nothing. With `--yes`, it rewrites and records that the migration is complete. It runs on every `migrate`, so an interrupted run is finished by the next one. If a policy lookup fails, or a policy needs attention (below), `migrate` reports it, exits non-zero, and does not record completion, so a later run retries. Policy bodies are never printed. From v1.0 the preview of a catalog with migration 2 pending prints the number of policies migration 2 rewrites and names the first 20 of them in that form, followed by the number of the others, and `migrate --yes` prints a line for each policy it rewrote, which the ledger keeps as notes of migration 2. A failed lookup is named with the account and principal it was reading and the error's text, and the run exits 1 without recording migration 2; a policy that needs attention makes the run exit 5. **Both forms on one key.** If one operator block in a policy holds both `aws:ResourceTag/` and `dynamodb:ResourceTag/` for the same tag key with the same value, the two were one condition before the upgrade, and `migrate` drops the `dynamodb:` key. With different values they cannot share one JSON key. `migrate` then leaves the policy unchanged, lists it as `NEEDS ATTENTION` with the account, the principal and the policy name, exits non-zero, and does not record the migration as complete, so `serve` keeps refusing the catalog. Fix the policy with the previous version still serving the catalog (keep one of the two forms), then run `migrate --yes` with the new binary. A session policy that does not parse is reported in the log and skipped; it cannot be evaluated under either form. @@ -631,6 +572,8 @@ for any of these changes. To get the intended behaviour, write the condition on `aws:ResourceTag/`. +**`--reconverge` and policies written after the upgrade.** From v1.0 the rewrite is catalog migration 2, which runs once. `migrate --yes --reconverge` skips it, so a policy written after the upgrade keeps its `dynamodb:ResourceTag/` keys and its decision. This includes a policy that a SQLite dev build from v0.1.7 to v0.1.13 wrote to the catalog after the upgrade: that build evaluated the key against the table's tags, and v1.0 leaves the key absent, as the table above describes. Rewrite such a policy to the `aws:ResourceTag/` form by hand. A v0.1.14 `migrate --yes` runs its rewrite whenever it finds the `dynamodb:` form, including in policies written through the API after the upgrade, so do not run it against a migrated catalog. + **Sessions issued before the migration.** Session policies are stored in the catalog, so `migrate` rewrites them along with the identity policies. A session issued before the upgrade is evaluated with its rewritten policy once `migrate --yes` has run, and it does not need to be issued again. The new binary does not serve an unmigrated catalog, so no request is evaluated against the old policy text under the new rules. ### IAM: `ArnEquals` and `ArnNotEquals` honour wildcards @@ -663,7 +606,7 @@ A policy whose Arn values are literal ARNs, applied to a context value that is a - A bare negated operator on `aws:TagKeys` (`StringNotEquals`, `StringNotLike`) matched on the absent key before. It no longer matches a request that carries tag keys. It still matches every operation that leaves the key absent, as IAM does (measured with `iam simulate-custom-policy` on 2026-10-24), so an `Allow` gated on it stops granting the tag requests and keeps granting the other operations. - A `Deny` gated on a bare negated operator on `aws:RequestTag/` (`StringNotEquals`) matched on the absent key before, so it blocked every request its `Action` covered. It now stops blocking a `TagResource` or `CreateTable` request whose tag carries an allowed value, and still blocks a request that omits the tag. -No catalog migration is required. Before upgrading a deployment that restricts tagging, check the policies of each account for the statements above. +This change needs no catalog migration of its own. Before upgrading a deployment that restricts tagging, check the policies of each account for the statements above. --- diff --git a/docs/manuals/08-install-linux.md b/docs/manuals/08-install-linux.md index 7414ab006..b24887b90 100755 --- a/docs/manuals/08-install-linux.md +++ b/docs/manuals/08-install-linux.md @@ -118,7 +118,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.4 + OK: Current: the catalog is at migration 4 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/09-install-macos.md b/docs/manuals/09-install-macos.md index feec0a409..beea83b9c 100755 --- a/docs/manuals/09-install-macos.md +++ b/docs/manuals/09-install-macos.md @@ -96,7 +96,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.4 + OK: Current: the catalog is at migration 4 ... === HEALTHY: All checks passed === ``` diff --git a/docs/schema/postgres.txt b/docs/schema/postgres.txt new file mode 100644 index 000000000..f3164d0a5 --- /dev/null +++ b/docs/schema/postgres.txt @@ -0,0 +1,373 @@ +# catalog database +table access_keys +table accounts +table admin_users +table backup_definitions +table backup_items +table backups +table continuous_backups +table extenddb_migrations +table iam_group_members +table iam_groups +table iam_permissions_boundaries +table iam_policies +table iam_role_tags +table iam_roles +table iam_sessions +table iam_user_tags +table iam_users +table idempotency_tokens +table indexes +table login_attempts +table metrics +table schema_history +table settings +table stream_records +table stream_shards +table table_restores +table tables +table tags +table vector_indexes +column access_keys.access_key_id: text NOT NULL +column access_keys.secret_key_encrypted: bytea NOT NULL +column access_keys.account_id: text NOT NULL +column access_keys.user_name: text NOT NULL +column access_keys.is_active: boolean NOT NULL DEFAULT true +column access_keys.created_at: timestamp with time zone NOT NULL DEFAULT now() +column accounts.account_id: text NOT NULL +column accounts.account_name: text NOT NULL +column accounts.created_at: timestamp with time zone NOT NULL DEFAULT now() +column admin_users.admin_name: text NOT NULL +column admin_users.password_hash: text NOT NULL +column admin_users.created_at: timestamp with time zone NOT NULL DEFAULT now() +column backup_definitions.backup_arn: text NOT NULL +column backup_definitions.definition: jsonb NOT NULL +column backup_items.backup_arn: text NOT NULL +column backup_items.pk: text NOT NULL +column backup_items.sk: text +column backup_items.item_data: jsonb NOT NULL +column backups.backup_arn: text NOT NULL +column backups.backup_name: text NOT NULL +column backups.table_id: text NOT NULL +column backups.table_name: text NOT NULL +column backups.account_id: text NOT NULL +column backups.backup_status: text NOT NULL DEFAULT 'AVAILABLE'::text +column backups.backup_type: text NOT NULL DEFAULT 'USER'::text +column backups.backup_size_bytes: bigint NOT NULL DEFAULT 0 +column backups.item_count: bigint NOT NULL DEFAULT 0 +column backups.key_schema: jsonb NOT NULL +column backups.attribute_definitions: jsonb NOT NULL +column backups.billing_mode: text NOT NULL DEFAULT 'PAY_PER_REQUEST'::text +column backups.provisioned_throughput: jsonb +column backups.stream_specification: jsonb +column backups.created_at: timestamp with time zone NOT NULL DEFAULT now() +column backups.vector_indexes: jsonb +column continuous_backups.account_id: text NOT NULL +column continuous_backups.table_name: text NOT NULL +column continuous_backups.pitr_enabled: boolean NOT NULL DEFAULT false +column continuous_backups.earliest_restorable: timestamp with time zone +column continuous_backups.latest_restorable: timestamp with time zone +column extenddb_migrations.id: integer NOT NULL +column extenddb_migrations.name: text NOT NULL +column extenddb_migrations.kind: text NOT NULL +column extenddb_migrations.checksum: text NOT NULL +column extenddb_migrations.outcome: text NOT NULL +column extenddb_migrations.notes: text NOT NULL DEFAULT ''::text +column extenddb_migrations.applied_at: timestamp with time zone NOT NULL DEFAULT now() +column extenddb_migrations.applied_by: text NOT NULL +column iam_group_members.account_id: text NOT NULL +column iam_group_members.group_name: text NOT NULL +column iam_group_members.user_name: text NOT NULL +column iam_groups.account_id: text NOT NULL +column iam_groups.group_name: text NOT NULL +column iam_groups.group_arn: text NOT NULL +column iam_groups.created_at: timestamp with time zone NOT NULL DEFAULT now() +column iam_permissions_boundaries.account_id: text NOT NULL +column iam_permissions_boundaries.principal_type: text NOT NULL +column iam_permissions_boundaries.principal_name: text NOT NULL +column iam_permissions_boundaries.policy_document: jsonb NOT NULL +column iam_policies.account_id: text NOT NULL +column iam_policies.principal_type: text NOT NULL +column iam_policies.principal_name: text NOT NULL +column iam_policies.policy_name: text NOT NULL +column iam_policies.policy_document: jsonb NOT NULL +column iam_policies.created_at: timestamp with time zone NOT NULL DEFAULT now() +column iam_role_tags.account_id: text NOT NULL +column iam_role_tags.role_name: text NOT NULL +column iam_role_tags.tag_key: text NOT NULL +column iam_role_tags.tag_value: text NOT NULL +column iam_roles.account_id: text NOT NULL +column iam_roles.role_name: text NOT NULL +column iam_roles.role_arn: text NOT NULL +column iam_roles.trust_policy: jsonb NOT NULL +column iam_roles.permissions_boundary_arn: text +column iam_roles.created_at: timestamp with time zone NOT NULL DEFAULT now() +column iam_sessions.session_token: text NOT NULL +column iam_sessions.access_key_id: text NOT NULL +column iam_sessions.secret_key_encrypted: bytea NOT NULL +column iam_sessions.account_id: text NOT NULL +column iam_sessions.role_name: text NOT NULL +column iam_sessions.session_name: text NOT NULL +column iam_sessions.session_tags: jsonb +column iam_sessions.session_policy: jsonb +column iam_sessions.expires_at: timestamp with time zone NOT NULL +column iam_sessions.created_at: timestamp with time zone NOT NULL DEFAULT now() +column iam_user_tags.account_id: text NOT NULL +column iam_user_tags.user_name: text NOT NULL +column iam_user_tags.tag_key: text NOT NULL +column iam_user_tags.tag_value: text NOT NULL +column iam_users.account_id: text NOT NULL +column iam_users.user_name: text NOT NULL +column iam_users.user_arn: text NOT NULL +column iam_users.password_hash: text +column iam_users.created_at: timestamp with time zone NOT NULL DEFAULT now() +column idempotency_tokens.token: text NOT NULL +column idempotency_tokens.fingerprint: text NOT NULL +column idempotency_tokens.created_at: timestamp with time zone NOT NULL DEFAULT now() +column indexes.table_id: text NOT NULL +column indexes.index_id: text NOT NULL DEFAULT (gen_random_uuid())::text +column indexes.index_name: text NOT NULL +column indexes.index_type: text NOT NULL +column indexes.key_schema: jsonb NOT NULL +column indexes.projection: jsonb NOT NULL +column indexes.index_status: text NOT NULL DEFAULT 'ACTIVE'::text +column indexes.provisioned_throughput: jsonb +column indexes.propagation_delay_ms: integer +column login_attempts.principal: text NOT NULL +column login_attempts.attempted_at: timestamp with time zone NOT NULL DEFAULT now() +column login_attempts.success: boolean NOT NULL +column login_attempts.source_ip: text +column metrics.bucket: timestamp with time zone NOT NULL +column metrics.metric: text NOT NULL +column metrics.table_name: text NOT NULL DEFAULT ''::text +column metrics.index_name: text NOT NULL DEFAULT ''::text +column metrics.operation: text NOT NULL DEFAULT ''::text +column metrics.sum: double precision NOT NULL DEFAULT 0 +column metrics.count: bigint NOT NULL DEFAULT 0 +column metrics.min: double precision NOT NULL DEFAULT 'Infinity'::double precision +column metrics.max: double precision NOT NULL DEFAULT '-Infinity'::double precision +column schema_history.filename: text NOT NULL +column schema_history.applied_at: timestamp with time zone NOT NULL DEFAULT now() +column settings.key: text NOT NULL +column settings.value: text NOT NULL +column stream_records.shard_id: text NOT NULL +column stream_records.sequence_number: text NOT NULL +column stream_records.table_id: text NOT NULL +column stream_records.event_name: text NOT NULL +column stream_records.record_data: jsonb NOT NULL +column stream_records.created_at: timestamp with time zone NOT NULL DEFAULT now() +column stream_shards.shard_id: text NOT NULL +column stream_shards.table_id: text NOT NULL +column stream_shards.parent_shard_id: text +column stream_shards.starting_sequence_number: text NOT NULL +column stream_shards.ending_sequence_number: text +column stream_shards.created_at: timestamp with time zone NOT NULL DEFAULT now() +column table_restores.table_id: text NOT NULL +column table_restores.source_backup_arn: text NOT NULL +column table_restores.source_table_arn: text NOT NULL +column table_restores.restore_date_time: timestamp with time zone NOT NULL +column tables.account_id: text NOT NULL +column tables.table_name: text NOT NULL +column tables.key_schema: jsonb NOT NULL +column tables.attribute_definitions: jsonb NOT NULL +column tables.billing_mode: text NOT NULL DEFAULT 'PAY_PER_REQUEST'::text +column tables.provisioned_throughput: jsonb +column tables.stream_specification: jsonb +column tables.table_status: text NOT NULL DEFAULT 'CREATING'::text +column tables.creation_date_time: timestamp with time zone NOT NULL DEFAULT now() +column tables.table_size_bytes: bigint NOT NULL DEFAULT 0 +column tables.item_count: bigint NOT NULL DEFAULT 0 +column tables.table_arn: text NOT NULL +column tables.table_id: text NOT NULL +column tables.ttl_attribute: text +column tables.deletion_protection_enabled: boolean NOT NULL DEFAULT false +column tables.status_transition_at: timestamp with time zone +column tables.stream_label: text +column tables.ttl_index_ready: boolean NOT NULL DEFAULT false +column tables.table_class: text +column tables.sse_specification: jsonb +column tables.on_demand_throughput: jsonb +column tags.resource_arn: text NOT NULL +column tags.tag_key: text NOT NULL +column tags.tag_value: text NOT NULL +column vector_indexes.table_id: text NOT NULL +column vector_indexes.index_id: text NOT NULL +column vector_indexes.index_name: text NOT NULL +column vector_indexes.dimensions: integer NOT NULL +column vector_indexes.distance_function: text NOT NULL +column vector_indexes.vector_attribute: jsonb NOT NULL +column vector_indexes.search_schema: jsonb +column vector_indexes.projection: jsonb NOT NULL +column vector_indexes.index_status: text NOT NULL DEFAULT 'CREATING'::text +column vector_indexes.backfilling: boolean +column vector_indexes.skipped_item_count: bigint +column vector_indexes.build_owner: text +column vector_indexes.build_heartbeat_at: timestamp with time zone +index access_keys_pkey: CREATE UNIQUE INDEX access_keys_pkey ON public.access_keys USING btree (access_key_id) +index accounts_account_name_key: CREATE UNIQUE INDEX accounts_account_name_key ON public.accounts USING btree (account_name) +index accounts_pkey: CREATE UNIQUE INDEX accounts_pkey ON public.accounts USING btree (account_id) +index admin_users_pkey: CREATE UNIQUE INDEX admin_users_pkey ON public.admin_users USING btree (admin_name) +index backup_definitions_pkey: CREATE UNIQUE INDEX backup_definitions_pkey ON public.backup_definitions USING btree (backup_arn) +index backups_pkey: CREATE UNIQUE INDEX backups_pkey ON public.backups USING btree (backup_arn) +index continuous_backups_pkey: CREATE UNIQUE INDEX continuous_backups_pkey ON public.continuous_backups USING btree (account_id, table_name) +index extenddb_migrations_pkey: CREATE UNIQUE INDEX extenddb_migrations_pkey ON public.extenddb_migrations USING btree (id) +index iam_group_members_pkey: CREATE UNIQUE INDEX iam_group_members_pkey ON public.iam_group_members USING btree (account_id, group_name, user_name) +index iam_groups_group_arn_key: CREATE UNIQUE INDEX iam_groups_group_arn_key ON public.iam_groups USING btree (group_arn) +index iam_groups_pkey: CREATE UNIQUE INDEX iam_groups_pkey ON public.iam_groups USING btree (account_id, group_name) +index iam_permissions_boundaries_pkey: CREATE UNIQUE INDEX iam_permissions_boundaries_pkey ON public.iam_permissions_boundaries USING btree (account_id, principal_type, principal_name) +index iam_policies_pkey: CREATE UNIQUE INDEX iam_policies_pkey ON public.iam_policies USING btree (account_id, principal_type, principal_name, policy_name) +index iam_role_tags_pkey: CREATE UNIQUE INDEX iam_role_tags_pkey ON public.iam_role_tags USING btree (account_id, role_name, tag_key) +index iam_roles_pkey: CREATE UNIQUE INDEX iam_roles_pkey ON public.iam_roles USING btree (account_id, role_name) +index iam_roles_role_arn_key: CREATE UNIQUE INDEX iam_roles_role_arn_key ON public.iam_roles USING btree (role_arn) +index iam_sessions_access_key_id_key: CREATE UNIQUE INDEX iam_sessions_access_key_id_key ON public.iam_sessions USING btree (access_key_id) +index iam_sessions_pkey: CREATE UNIQUE INDEX iam_sessions_pkey ON public.iam_sessions USING btree (session_token) +index iam_user_tags_pkey: CREATE UNIQUE INDEX iam_user_tags_pkey ON public.iam_user_tags USING btree (account_id, user_name, tag_key) +index iam_users_pkey: CREATE UNIQUE INDEX iam_users_pkey ON public.iam_users USING btree (account_id, user_name) +index iam_users_user_arn_key: CREATE UNIQUE INDEX iam_users_user_arn_key ON public.iam_users USING btree (user_arn) +index idempotency_tokens_pkey: CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (token) +index idx_backup_items_arn: CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn) +index idx_backups_table: CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name) +index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at) +index idx_login_attempts_principal_time: CREATE INDEX idx_login_attempts_principal_time ON public.login_attempts USING btree (principal, attempted_at DESC) +index idx_login_attempts_source_ip_time: CREATE INDEX idx_login_attempts_source_ip_time ON public.login_attempts USING btree (source_ip, attempted_at DESC) WHERE (source_ip IS NOT NULL) +index idx_metrics_bucket: CREATE INDEX idx_metrics_bucket ON public.metrics USING btree (bucket) +index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) +index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) +index idx_table_restores_backup: CREATE INDEX idx_table_restores_backup ON public.table_restores USING btree (source_backup_arn) +index idx_tables_pending_transition: CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL) +index idx_vector_indexes_index_id: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id) +index indexes_pkey: CREATE UNIQUE INDEX indexes_pkey ON public.indexes USING btree (table_id, index_name) +index metrics_pkey: CREATE UNIQUE INDEX metrics_pkey ON public.metrics USING btree (bucket, metric, table_name, index_name, operation) +index schema_history_pkey: CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename) +index settings_pkey: CREATE UNIQUE INDEX settings_pkey ON public.settings USING btree (key) +index stream_records_pkey: CREATE UNIQUE INDEX stream_records_pkey ON public.stream_records USING btree (shard_id, sequence_number) +index stream_shards_pkey: CREATE UNIQUE INDEX stream_shards_pkey ON public.stream_shards USING btree (shard_id) +index table_restores_pkey: CREATE UNIQUE INDEX table_restores_pkey ON public.table_restores USING btree (table_id) +index tables_pkey: CREATE UNIQUE INDEX tables_pkey ON public.tables USING btree (account_id, table_name) +index tables_table_id_unique: CREATE UNIQUE INDEX tables_table_id_unique ON public.tables USING btree (table_id) +index tags_pkey: CREATE UNIQUE INDEX tags_pkey ON public.tags USING btree (resource_arn, tag_key) +index vector_indexes_pkey: CREATE UNIQUE INDEX vector_indexes_pkey ON public.vector_indexes USING btree (table_id, index_name) +constraint access_keys.access_keys_account_id_user_name_fkey: FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +constraint access_keys.access_keys_pkey: PRIMARY KEY (access_key_id) +constraint accounts.accounts_account_name_key: UNIQUE (account_name) +constraint accounts.accounts_pkey: PRIMARY KEY (account_id) +constraint admin_users.admin_users_pkey: PRIMARY KEY (admin_name) +constraint backup_definitions.backup_definitions_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE +constraint backup_definitions.backup_definitions_pkey: PRIMARY KEY (backup_arn) +constraint backup_items.backup_items_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE +constraint backups.backups_pkey: PRIMARY KEY (backup_arn) +constraint continuous_backups.continuous_backups_pkey: PRIMARY KEY (account_id, table_name) +constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) +constraint iam_group_members.iam_group_members_account_id_group_name_fkey: FOREIGN KEY (account_id, group_name) REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE +constraint iam_group_members.iam_group_members_account_id_user_name_fkey: FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +constraint iam_group_members.iam_group_members_pkey: PRIMARY KEY (account_id, group_name, user_name) +constraint iam_groups.iam_groups_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint iam_groups.iam_groups_group_arn_key: UNIQUE (group_arn) +constraint iam_groups.iam_groups_pkey: PRIMARY KEY (account_id, group_name) +constraint iam_permissions_boundaries.iam_permissions_boundaries_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint iam_permissions_boundaries.iam_permissions_boundaries_pkey: PRIMARY KEY (account_id, principal_type, principal_name) +constraint iam_permissions_boundaries.iam_permissions_boundaries_principal_type_check: CHECK ((principal_type = ANY (ARRAY['user'::text, 'role'::text]))) +constraint iam_policies.iam_policies_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint iam_policies.iam_policies_pkey: PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +constraint iam_policies.iam_policies_principal_type_check: CHECK ((principal_type = ANY (ARRAY['user'::text, 'group'::text, 'role'::text]))) +constraint iam_role_tags.iam_role_tags_account_id_role_name_fkey: FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +constraint iam_role_tags.iam_role_tags_pkey: PRIMARY KEY (account_id, role_name, tag_key) +constraint iam_roles.iam_roles_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint iam_roles.iam_roles_pkey: PRIMARY KEY (account_id, role_name) +constraint iam_roles.iam_roles_role_arn_key: UNIQUE (role_arn) +constraint iam_sessions.iam_sessions_access_key_id_key: UNIQUE (access_key_id) +constraint iam_sessions.iam_sessions_account_id_role_name_fkey: FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +constraint iam_sessions.iam_sessions_pkey: PRIMARY KEY (session_token) +constraint iam_user_tags.iam_user_tags_account_id_user_name_fkey: FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +constraint iam_user_tags.iam_user_tags_pkey: PRIMARY KEY (account_id, user_name, tag_key) +constraint iam_users.iam_users_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint iam_users.iam_users_pkey: PRIMARY KEY (account_id, user_name) +constraint iam_users.iam_users_user_arn_key: UNIQUE (user_arn) +constraint idempotency_tokens.idempotency_tokens_pkey: PRIMARY KEY (token) +constraint indexes.chk_propagation_delay_ms_non_negative: CHECK (((propagation_delay_ms IS NULL) OR (propagation_delay_ms >= 0))) +constraint indexes.indexes_pkey: PRIMARY KEY (table_id, index_name) +constraint indexes.indexes_table_id_fkey: FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE +constraint metrics.metrics_pkey: PRIMARY KEY (bucket, metric, table_name, index_name, operation) +constraint schema_history.schema_history_pkey: PRIMARY KEY (filename) +constraint settings.settings_pkey: PRIMARY KEY (key) +constraint stream_records.stream_records_pkey: PRIMARY KEY (shard_id, sequence_number) +constraint stream_records.stream_records_shard_id_fkey: FOREIGN KEY (shard_id) REFERENCES stream_shards(shard_id) ON DELETE CASCADE +constraint stream_shards.stream_shards_pkey: PRIMARY KEY (shard_id) +constraint stream_shards.stream_shards_table_id_fkey: FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE +constraint table_restores.table_restores_pkey: PRIMARY KEY (table_id) +constraint table_restores.table_restores_table_id_fkey: FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE +constraint tables.tables_account_id_fkey: FOREIGN KEY (account_id) REFERENCES accounts(account_id) ON DELETE CASCADE +constraint tables.tables_pkey: PRIMARY KEY (account_id, table_name) +constraint tables.tables_table_id_unique: UNIQUE (table_id) +constraint tags.tags_pkey: PRIMARY KEY (resource_arn, tag_key) +constraint vector_indexes.chk_vector_active_has_no_backfilling: CHECK (((index_status <> 'ACTIVE'::text) OR (backfilling IS NULL))) +constraint vector_indexes.chk_vector_dimensions_positive: CHECK ((dimensions > 0)) +constraint vector_indexes.vector_indexes_pkey: PRIMARY KEY (table_id, index_name) +constraint vector_indexes.vector_indexes_table_id_fkey: FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE +sequence stream_seq: bigint START 1 INCREMENT 1 +# data database +table extenddb_migrations +table gsi_pending +table idempotency_tokens +table schema_history +table stream_records +table stream_shards +table vector_index_holds +column extenddb_migrations.id: integer NOT NULL +column extenddb_migrations.name: text NOT NULL +column extenddb_migrations.kind: text NOT NULL +column extenddb_migrations.checksum: text NOT NULL +column extenddb_migrations.outcome: text NOT NULL +column extenddb_migrations.notes: text NOT NULL DEFAULT ''::text +column extenddb_migrations.applied_at: timestamp with time zone NOT NULL DEFAULT now() +column extenddb_migrations.applied_by: text NOT NULL +column gsi_pending.id: bigint NOT NULL DEFAULT nextval('gsi_pending_id_seq'::regclass) +column gsi_pending.table_id: text NOT NULL +column gsi_pending.worker_partition: integer NOT NULL +column gsi_pending.old_item: jsonb +column gsi_pending.new_item: jsonb +column gsi_pending.index_context: jsonb NOT NULL +column gsi_pending.ready_at: timestamp with time zone NOT NULL DEFAULT now() +column idempotency_tokens.account_id: text NOT NULL +column idempotency_tokens.token: text NOT NULL +column idempotency_tokens.fingerprint: text NOT NULL +column idempotency_tokens.created_at: timestamp with time zone NOT NULL DEFAULT now() +column schema_history.filename: text NOT NULL +column schema_history.applied_at: timestamp with time zone NOT NULL DEFAULT now() +column stream_records.shard_id: text NOT NULL +column stream_records.sequence_number: text NOT NULL +column stream_records.table_id: text NOT NULL +column stream_records.event_name: text NOT NULL +column stream_records.record_data: jsonb NOT NULL +column stream_records.created_at: timestamp with time zone NOT NULL DEFAULT now() +column stream_shards.shard_id: text NOT NULL +column stream_shards.table_id: text NOT NULL +column stream_shards.parent_shard_id: text +column stream_shards.starting_sequence_number: text NOT NULL +column stream_shards.ending_sequence_number: text +column stream_shards.created_at: timestamp with time zone NOT NULL DEFAULT now() +column vector_index_holds.table_id: text NOT NULL +column vector_index_holds.index_id: text NOT NULL +column vector_index_holds.created_at: timestamp with time zone NOT NULL DEFAULT now() +index extenddb_migrations_pkey: CREATE UNIQUE INDEX extenddb_migrations_pkey ON public.extenddb_migrations USING btree (id) +index gsi_pending_pkey: CREATE UNIQUE INDEX gsi_pending_pkey ON public.gsi_pending USING btree (id) +index idempotency_tokens_pkey: CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (account_id, token) +index idx_gsi_pending_claim: CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id) +index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at) +index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) +index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) +index idx_vector_index_holds_table: CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id) +index schema_history_pkey: CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename) +index stream_records_pkey: CREATE UNIQUE INDEX stream_records_pkey ON public.stream_records USING btree (shard_id, sequence_number) +index stream_shards_pkey: CREATE UNIQUE INDEX stream_shards_pkey ON public.stream_shards USING btree (shard_id) +index vector_index_holds_pkey: CREATE UNIQUE INDEX vector_index_holds_pkey ON public.vector_index_holds USING btree (table_id, index_id) +constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) +constraint gsi_pending.gsi_pending_pkey: PRIMARY KEY (id) +constraint idempotency_tokens.idempotency_tokens_pkey: PRIMARY KEY (account_id, token) +constraint schema_history.schema_history_pkey: PRIMARY KEY (filename) +constraint stream_records.stream_records_pkey: PRIMARY KEY (shard_id, sequence_number) +constraint stream_records.stream_records_shard_id_fkey: FOREIGN KEY (shard_id) REFERENCES stream_shards(shard_id) ON DELETE CASCADE +constraint stream_shards.stream_shards_pkey: PRIMARY KEY (shard_id) +constraint vector_index_holds.vector_index_holds_pkey: PRIMARY KEY (table_id, index_id) +sequence gsi_pending_id_seq: bigint START 1 INCREMENT 1 +sequence stream_seq: bigint START 1 INCREMENT 1 diff --git a/docs/schema/sqlite.txt b/docs/schema/sqlite.txt new file mode 100644 index 000000000..ab0b7ce86 --- /dev/null +++ b/docs/schema/sqlite.txt @@ -0,0 +1,43 @@ +index idx_backup_items_arn on backup_items: CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn) +index idx_backups_table on backups: CREATE INDEX idx_backups_table ON backups (account_id, table_name) +index idx_gsi_pending_claim on gsi_pending: CREATE INDEX idx_gsi_pending_claim ON gsi_pending (worker_partition, ready_at, id) +index idx_idempotency_tokens_created on idempotency_tokens: CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at) +index idx_login_attempts_principal_time on login_attempts: CREATE INDEX idx_login_attempts_principal_time ON login_attempts (principal, attempted_at DESC) +index idx_login_attempts_source_ip_time on login_attempts: CREATE INDEX idx_login_attempts_source_ip_time ON login_attempts (source_ip, attempted_at DESC) WHERE source_ip IS NOT NULL +index idx_metrics_bucket on metrics: CREATE INDEX idx_metrics_bucket ON metrics (bucket) +index idx_stream_records_created on stream_records: CREATE INDEX idx_stream_records_created ON stream_records (created_at) +index idx_stream_shards_table on stream_shards: CREATE INDEX idx_stream_shards_table ON stream_shards (table_id) +index idx_table_restores_backup on table_restores: CREATE INDEX idx_table_restores_backup ON table_restores (source_backup_arn) +index idx_tables_pending_transition on tables: CREATE INDEX idx_tables_pending_transition ON tables (status_transition_at) WHERE status_transition_at IS NOT NULL +index idx_vector_indexes_index_id on vector_indexes: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id) +table access_keys on access_keys: CREATE TABLE access_keys ( access_key_id TEXT PRIMARY KEY, secret_key_encrypted BLOB NOT NULL, account_id TEXT NOT NULL, user_name TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE ) +table accounts on accounts: CREATE TABLE accounts ( account_id TEXT PRIMARY KEY, account_name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table admin_users on admin_users: CREATE TABLE admin_users ( admin_name TEXT PRIMARY KEY, password_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table backup_definitions on backup_definitions: CREATE TABLE backup_definitions ( backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, definition TEXT NOT NULL ) +table backup_items on backup_items: CREATE TABLE backup_items ( id INTEGER PRIMARY KEY AUTOINCREMENT, backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, pk TEXT NOT NULL, sk TEXT, item_data TEXT NOT NULL ) +table backups on backups: CREATE TABLE backups ( backup_arn TEXT PRIMARY KEY, backup_name TEXT NOT NULL, table_id TEXT NOT NULL, table_name TEXT NOT NULL, account_id TEXT NOT NULL, backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', backup_type TEXT NOT NULL DEFAULT 'USER', backup_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table continuous_backups on continuous_backups: CREATE TABLE continuous_backups ( account_id TEXT NOT NULL, table_name TEXT NOT NULL, pitr_enabled INTEGER NOT NULL DEFAULT 0, earliest_restorable TEXT, latest_restorable TEXT, PRIMARY KEY (account_id, table_name) ) +table extenddb_migrations on extenddb_migrations: CREATE TABLE extenddb_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL, checksum TEXT NOT NULL, outcome TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), applied_by TEXT NOT NULL) +table gsi_pending on gsi_pending: CREATE TABLE gsi_pending ( id INTEGER PRIMARY KEY AUTOINCREMENT, table_id TEXT NOT NULL, worker_partition INTEGER NOT NULL, old_item TEXT, new_item TEXT, index_context TEXT NOT NULL, ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table iam_group_members on iam_group_members: CREATE TABLE iam_group_members ( account_id TEXT NOT NULL, group_name TEXT NOT NULL, user_name TEXT NOT NULL, PRIMARY KEY (account_id, group_name, user_name), FOREIGN KEY (account_id, group_name) REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE ) +table iam_groups on iam_groups: CREATE TABLE iam_groups ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, group_name TEXT NOT NULL, group_arn TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, group_name) ) +table iam_permissions_boundaries on iam_permissions_boundaries: CREATE TABLE iam_permissions_boundaries ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'role')), principal_name TEXT NOT NULL, policy_document TEXT NOT NULL, PRIMARY KEY (account_id, principal_type, principal_name) ) +table iam_policies on iam_policies: CREATE TABLE iam_policies ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, principal_type TEXT NOT NULL CHECK (principal_type IN ('user', 'group', 'role')), principal_name TEXT NOT NULL, policy_name TEXT NOT NULL, policy_document TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, principal_type, principal_name, policy_name) ) +table iam_role_tags on iam_role_tags: CREATE TABLE iam_role_tags ( account_id TEXT NOT NULL, role_name TEXT NOT NULL, tag_key TEXT NOT NULL, tag_value TEXT NOT NULL, PRIMARY KEY (account_id, role_name, tag_key), FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE ) +table iam_roles on iam_roles: CREATE TABLE iam_roles ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, role_name TEXT NOT NULL, role_arn TEXT NOT NULL UNIQUE, trust_policy TEXT NOT NULL, permissions_boundary_arn TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, role_name) ) +table iam_sessions on iam_sessions: CREATE TABLE iam_sessions ( session_token TEXT PRIMARY KEY, access_key_id TEXT NOT NULL UNIQUE, secret_key_encrypted BLOB NOT NULL, account_id TEXT NOT NULL, role_name TEXT NOT NULL, session_name TEXT NOT NULL, session_tags TEXT, session_policy TEXT, expires_at TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE ) +table iam_user_tags on iam_user_tags: CREATE TABLE iam_user_tags ( account_id TEXT NOT NULL, user_name TEXT NOT NULL, tag_key TEXT NOT NULL, tag_value TEXT NOT NULL, PRIMARY KEY (account_id, user_name, tag_key), FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE ) +table iam_users on iam_users: CREATE TABLE iam_users ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, user_name TEXT NOT NULL, user_arn TEXT NOT NULL UNIQUE, password_hash TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, user_name) ) +table idempotency_tokens on idempotency_tokens: CREATE TABLE idempotency_tokens ( account_id TEXT NOT NULL, token TEXT NOT NULL, fingerprint TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, token) ) +table indexes on indexes: CREATE TABLE indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, index_type TEXT NOT NULL, key_schema TEXT NOT NULL, projection TEXT NOT NULL, index_status TEXT NOT NULL DEFAULT 'ACTIVE', provisioned_throughput TEXT, propagation_delay_ms INTEGER, PRIMARY KEY (table_id, index_name), CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) ) +table login_attempts on login_attempts: CREATE TABLE login_attempts ( principal TEXT NOT NULL, attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), success INTEGER NOT NULL, source_ip TEXT ) +table metrics on metrics: CREATE TABLE metrics ( bucket TEXT NOT NULL, metric TEXT NOT NULL, table_name TEXT NOT NULL DEFAULT '', index_name TEXT NOT NULL DEFAULT '', operation TEXT NOT NULL DEFAULT '', sum REAL NOT NULL DEFAULT 0, count INTEGER NOT NULL DEFAULT 0, min REAL NOT NULL DEFAULT 9e999, max REAL NOT NULL DEFAULT -9e999, PRIMARY KEY (bucket, metric, table_name, index_name, operation) ) +table schema_history on schema_history: CREATE TABLE schema_history ( filename TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table seq_counters on seq_counters: CREATE TABLE seq_counters ( name TEXT PRIMARY KEY, value INTEGER NOT NULL DEFAULT 0 ) +table settings on settings: CREATE TABLE settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ) +table stream_records on stream_records: CREATE TABLE stream_records ( shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, sequence_number TEXT NOT NULL, table_id TEXT NOT NULL, event_name TEXT NOT NULL, record_data TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (shard_id, sequence_number) ) +table stream_shards on stream_shards: CREATE TABLE stream_shards ( shard_id TEXT PRIMARY KEY, table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, parent_shard_id TEXT, starting_sequence_number TEXT NOT NULL, ending_sequence_number TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) +table table_restores on table_restores: CREATE TABLE table_restores ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, source_backup_arn TEXT NOT NULL, source_table_arn TEXT, restore_date_time TEXT NOT NULL ) +table tables on tables: CREATE TABLE tables ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, table_name TEXT NOT NULL, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, table_status TEXT NOT NULL DEFAULT 'CREATING', creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), table_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, table_arn TEXT NOT NULL, table_id TEXT NOT NULL, ttl_attribute TEXT, deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, status_transition_at TEXT, stream_label TEXT, ttl_index_ready INTEGER NOT NULL DEFAULT 0, table_class TEXT, sse_specification TEXT, on_demand_throughput TEXT, PRIMARY KEY (account_id, table_name), CONSTRAINT tables_table_id_unique UNIQUE (table_id) ) +table tags on tags: CREATE TABLE tags ( resource_arn TEXT NOT NULL, tag_key TEXT NOT NULL, tag_value TEXT NOT NULL, PRIMARY KEY (resource_arn, tag_key) ) +table vector_indexes on vector_indexes: CREATE TABLE vector_indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, dimensions INTEGER NOT NULL, distance_function TEXT NOT NULL, vector_attribute TEXT NOT NULL, search_schema TEXT, projection TEXT NOT NULL, index_status TEXT NOT NULL DEFAULT 'CREATING', backfilling INTEGER, -- Items the backfill skipped because their stored bytes cannot enter the -- index (unparseable row, malformed or wrong-dimension vector). NULL until -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so -- an operator can see that an ACTIVE index deliberately omits rows, rather -- than the build looping forever on them or dying part-way. skipped_item_count INTEGER, PRIMARY KEY (table_id, index_name), CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), CONSTRAINT chk_vector_backfilling_bool CHECK (backfilling IS NULL OR backfilling IN (0, 1)), -- An ACTIVE index must not carry the member at all, which is what the -- service does. Enforced here as well as in core, so a bug in the backend -- cannot persist a state the wire contract forbids. CONSTRAINT chk_vector_active_has_no_backfilling CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) ) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 11a35c1b5..f22a2c4d1 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -65,17 +65,17 @@ See `docs/local-postgres-setup.md` for full setup instructions. **Fix:** Check the PostgreSQL logs (`~/pgdata/server.log`). Ensure the `extenddb` user has CREATE TABLE permissions on the `extenddb` database. -### `Catalog version mismatch: expected X, found Y. Run 'extenddb migrate' to update.` +### `serve`, `migrate` or `verify` exits 3, 4, 5 or 6 naming the catalog -**Cause:** The catalog database was initialized with a different version of extenddb. The binary expects catalog version X but the database has version Y. +**Cause:** The catalog's status, read from its migration ledgers, is not one this binary serves: exit 3 means migrations are pending, 4 that the catalog has a breaking migration this binary does not know, 5 that a person must act, and 6 that the migration lock or a running server outlasted `--lock-timeout`. The [status table](manuals/07-upgrade-manual.md#the-catalog-status) of the upgrade manual lists each status with its exit codes. -**Fix:** Run `extenddb migrate` to apply schema migrations and update the catalog version. Back up the database first. +**Fix:** Run the command the message prints. For pending migrations that is `extenddb migrate --config extenddb.toml --yes`, after the backup the message names. -### `Catalog not initialized. Run 'extenddb init' to set up the catalog.` +### `the catalog at is not initialized; create it with 'extenddb init'` -**Cause:** The server connected to the database but the catalog tables don't exist. The database hasn't been initialized with `extenddb init`. +**Cause:** The database holds no catalog: no migration ledger and no `catalog_version` row. `serve`, `migrate` and `verify` exit 1 and change nothing. -**Fix:** Run `extenddb init` to create the catalog schema and data database. See `docs/getting-started.md`. +**Fix:** Check that the config names the right database. For a new deployment, run `extenddb init` to create the catalog and data databases. See `docs/getting-started.md`. ### `Database '' already exists. Run 'extenddb destroy --config ' first, then re-run 'extenddb init'.` @@ -95,7 +95,7 @@ extenddb serve --port 19443 --config extenddb.toml # use a different port ### `another extenddb process is already using (lock held on .lock)` -**Cause:** A SQLite database file has one owner at a time. `extenddb serve` takes an exclusive lock on `.lock` (next to the file SQLite opens, after resolving any `sqlite:` or `file:` form, relative path, or symlink) before it opens the database and holds it until it exits. `init` (through to the admin user being written), `migrate`, and `destroy` take the same lock for as long as they run, because they rewrite or remove the file. The backend serializes writers inside one process and recovers interrupted work at startup, so a second server, or a migration or destroy under a running server, would corrupt the first one's work. Read-only commands (`settings`, `manage`, `verify`, `status`) take no lock and run alongside a server. +**Cause:** A SQLite database file has one owner at a time. `extenddb serve` takes an exclusive lock on `.lock` (next to the file SQLite opens, after resolving any `sqlite:` or `file:` form, relative path, or symlink) before it opens the database and holds it until it exits. `init` (through to the admin user being written), `migrate --yes`, and `destroy` take the same lock for as long as they run, because they rewrite or remove the file; `migrate --yes` waits for it for up to `--lock-timeout` seconds (60 by default) and then exits 6. The backend serializes writers inside one process and recovers interrupted work at startup, so a second server, or a migration or destroy under a running server, would corrupt the first one's work. Read-only commands (`settings`, `manage`, `verify`, `status`, and `migrate` without `--yes`) take no lock and run alongside a server. **Fix:** Stop the other process (`extenddb stop --config ` for a server), or point this command at a different database (`--sqlite-path` at `init` time). The lock is released by the operating system when the holder exits, including after a crash, so a leftover `.lock` file is harmless and needs no cleanup. diff --git a/packaging/README.md b/packaging/README.md index 75d5c1871..85da98149 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -98,7 +98,12 @@ deliberate second step): 1. Land the release state on `main` with the workspace version bumped and `npm/package.json` bumped to match, then tag that commit `vX.Y.Z` and push the tag. This is the same tag the container release consumes; the two - publish workflows are dispatched independently against it. + publish workflows are dispatched independently against it. When the + release ships a catalog migration, the same state raises + `RELEASED_THROUGH` to the last migration and pastes the checksum lines the + checksum test prints (CONTRIBUTING.md, "Adding a catalog migration"); the + gate jobs of the two container release workflows run + `devtools/check-released-through` on the tag and refuse it otherwise. 2. Dispatch the `npm-publish` workflow from `main` with the tag as input. The gate validates shape, existence, ancestry against `main`, and the version match. Five native runners build, smoke test, and run the full diff --git a/tests/test_cli_lifecycle.py b/tests/test_cli_lifecycle.py index 9164abab2..b4fbbcdd8 100644 --- a/tests/test_cli_lifecycle.py +++ b/tests/test_cli_lifecycle.py @@ -54,11 +54,11 @@ def test_init_creates_schema(self, cli_env): assert os.path.isfile(os.path.join(cli_env["tls_dir"], "key.pem")) def test_data_migrations_tracked(self, cli_env): - """init records every data migration in the data DB's schema_history. + """init writes the legacy data ledger rows in the data DB's schema_history. - Validates the data-migration registry: migrations are tracked (so - `extenddb migrate` knows what is pending and never re-runs an applied - one), and the GSI-pending migration is among them. + Migration 1 records every legacy data migration file and the legacy + code migration there, so a release before 1.0 started against this + catalog finds nothing to replay. """ import psycopg2 @@ -81,24 +81,19 @@ def test_data_migrations_tracked(self, cli_env): assert "001_data_schema.sql" in tracked, tracked assert "002_gsi_pending.sql" in tracked, tracked - # Programmatic (code) migration for the GSI base-key index is tracked in - # the same ledger, so `migrate` treats it exactly like a SQL migration. + # The legacy code migration's row: v0.1.14 runs it whenever the name is + # missing. assert "003_gsi_base_key_index" in tracked, tracked - def test_migrate_applies_pending_data_migration(self, cli_env): - """`extenddb migrate` applies a pending *data* migration on an existing - deployment — not just catalog migrations. + def test_migrate_applies_a_migration_pending_in_the_data_database(self, cli_env): + """`extenddb migrate` applies a migration whose catalog body is recorded + and whose data body is not. - Regression: data migrations live in the data database's own - `schema_history`, independent of the catalog version. `migrate` - previously only ran catalog migrations and returned "up to date" when - the catalog version matched, so a release that only added a data - migration (e.g. `002_gsi_pending.sql`) was never applied on upgrade — - the `gsi_pending` table was missing and every async-GSI write failed. - - Simulates a pre-002 deployment by dropping `gsi_pending` and its - ledger row, then asserts `migrate` re-applies it (and refuses without - `--yes`). + Each database has its own ledger, and a migration counts as applied only + when every database it touches records it. Simulates a deployment whose + data database lost migration 1 (its `gsi_pending` table and its ledger + row), then asserts the preview reports it, changes nothing, and that + `migrate --yes` applies it. """ import psycopg2 @@ -114,47 +109,43 @@ def test_migrate_applies_pending_data_migration(self, cli_env): def _data_conn(): return psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - # Simulate a deployment created before 002 existed. conn = _data_conn() try: conn.autocommit = True with conn.cursor() as cur: cur.execute("DROP TABLE IF EXISTS gsi_pending") - cur.execute( - "DELETE FROM schema_history WHERE filename = '002_gsi_pending.sql'" - ) + cur.execute("DELETE FROM extenddb_migrations WHERE id = 1") cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") assert cur.fetchone()[0] is False finally: conn.close() - # Without --yes, migrate must refuse while a data migration is pending. - refused = _run_extenddb( + # Without --yes, migrate reports the pending body and exits 3. + pending = _run_extenddb( "migrate", *_pg_args(), config=cli_env["config_path"], check=False, ) - assert refused.returncode != 0, refused.stdout + refused.stderr + assert pending.returncode == 3, pending.stdout + pending.stderr + assert "1 baseline (data, breaking)" in pending.stdout, pending.stdout - # With --yes, migrate applies the pending data migration. + # With --yes, migrate applies it in the data database. applied = _run_extenddb( "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False, ) assert applied.returncode == 0, applied.stdout + applied.stderr + assert "Applied 1 baseline" in applied.stdout, applied.stdout conn = _data_conn() try: with conn.cursor() as cur: cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") assert cur.fetchone()[0] is True - cur.execute( - "SELECT EXISTS(SELECT 1 FROM schema_history " - "WHERE filename = '002_gsi_pending.sql')" - ) - assert cur.fetchone()[0] is True - # The 002 schema must carry the per-index queue columns. + cur.execute("SELECT count(*) FROM extenddb_migrations WHERE id = 1") + assert cur.fetchone()[0] == 1 + # The per-index queue columns. cur.execute( "SELECT column_name FROM information_schema.columns " "WHERE table_name = 'gsi_pending'" @@ -164,20 +155,18 @@ def _data_conn(): finally: conn.close() - def test_migrate_applies_base_key_index_code_migration(self, cli_env): - """`extenddb migrate` applies the programmatic GSI base-key-index - migration (`003_gsi_base_key_index`) and never re-runs it. + def test_migrate_applies_the_base_key_index_migration(self, cli_env): + """`extenddb migrate` applies migration 4 (`index_table_base_key`), the + code body that gives every index table its base key index, and does not + run it again once recorded. - The base-key index makes GSI-propagation deletes (`WHERE base_pk = $1 + The base key index makes GSI propagation deletes (`WHERE base_pk = $1 AND base_sk_* = $2`) index scans instead of sequential scans. New tables - get it at creation time; this migration adds it to tables created before - the index existed. It is a *code* migration (it enumerates the - dynamically-named `_ddb_*` tables and uses `CREATE INDEX CONCURRENTLY`), - so this verifies it is wired into the same pending/apply/tracking flow as - the SQL migrations: pending is detected, `migrate` refuses without - `--yes`, applies with `--yes`, records the ledger row, and is idempotent - on a second run. (Index derivation itself is the same logic exercised at - table-creation time across the GSI integration suite.) + get it at creation; migration 4 adds it to tables created before the + index existed, building it with `CREATE INDEX CONCURRENTLY` outside a + transaction. This checks it goes through the same pending, apply and + ledger flow as the other migrations: reported as pending, applied with + `--yes`, recorded, and up to date on a second run. """ import psycopg2 @@ -193,53 +182,40 @@ def test_migrate_applies_base_key_index_code_migration(self, cli_env): def _data_conn(): return psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - # Simulate a deployment created before 003 existed: drop only its ledger - # row so `migrate` sees the code migration as pending again. + # Remove only the data database's ledger row for migration 4. conn = _data_conn() try: conn.autocommit = True with conn.cursor() as cur: - cur.execute( - "DELETE FROM schema_history " - "WHERE filename = '003_gsi_base_key_index'" - ) - cur.execute( - "SELECT EXISTS(SELECT 1 FROM schema_history " - "WHERE filename = '003_gsi_base_key_index')" - ) - assert cur.fetchone()[0] is False + cur.execute("DELETE FROM extenddb_migrations WHERE id = 4") finally: conn.close() - # Without --yes, migrate must refuse while the code migration is pending. - refused = _run_extenddb( + pending = _run_extenddb( "migrate", *_pg_args(), config=cli_env["config_path"], check=False, ) - assert refused.returncode != 0, refused.stdout + refused.stderr - assert "003_gsi_base_key_index" in (refused.stdout + refused.stderr) + assert pending.returncode == 3, pending.stdout + pending.stderr + assert "4 index_table_base_key (data, breaking)" in pending.stdout, pending.stdout - # With --yes, migrate applies and records the code migration. applied = _run_extenddb( "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False, ) assert applied.returncode == 0, applied.stdout + applied.stderr + assert "Applied 4 index_table_base_key" in applied.stdout, applied.stdout conn = _data_conn() try: with conn.cursor() as cur: - cur.execute( - "SELECT EXISTS(SELECT 1 FROM schema_history " - "WHERE filename = '003_gsi_base_key_index')" - ) - assert cur.fetchone()[0] is True + cur.execute("SELECT count(*) FROM extenddb_migrations WHERE id = 4") + assert cur.fetchone()[0] == 1 finally: conn.close() - # Idempotent: a second migrate finds nothing pending. + # A second migrate finds nothing pending. again = _run_extenddb( "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], diff --git a/tests/test_cli_migrate_concurrency.py b/tests/test_cli_migrate_concurrency.py index 04a7f31cc..25c3bb7b0 100644 --- a/tests/test_cli_migrate_concurrency.py +++ b/tests/test_cli_migrate_concurrency.py @@ -3,8 +3,15 @@ """Migration concurrency CLI tests. -Cover the PostgreSQL advisory lock that serializes `extenddb migrate`, so two -replicas starting at once cannot race each other applying schema changes. +Cover the PostgreSQL migration lock that serializes `extenddb migrate`, so two +migrators that start together cannot both apply a migration. + +The barrier: the runner commits a migration's body and its ledger row in one +transaction, so a session that holds an uncommitted `extenddb_migrations` row +for the same id makes the migrator's ledger insert wait, inside its +transaction and under the migration lock, at a known point. Each statement of +a body waits at most 5 s for a lock and is retried three times, so the tests +release the barrier within that window. Like the other CLI lifecycle tests these require a PostgreSQL instance (EXTENDDB_TEST_PG_CONNECTION_STRING) and a built binary, and are excluded from @@ -27,11 +34,13 @@ _run_extenddb, ) -# Must match ADVISORY_LOCK_NAMESPACE and MIGRATION_LOCK_OBJID in -# crates/storage-postgres/src/bootstrapper.rs. +# The advisory key of the migration lock, `NAMESPACE` and `MIGRATION_KEY` in +# crates/storage-postgres/src/migrations/locks.rs. LOCK_NAMESPACE = 0x0045_4442 MIGRATION_LOCK_OBJID = 1 +WAITING = "Another migrator holds the migration lock" + def _init(cli_env): result = _run_extenddb( @@ -79,8 +88,38 @@ def _read_until(proc, expected, timeout=15.0): ) -def _wait_for_blocked_ledger_insert(data_db, timeout=15.0): - """Wait until a migrator is blocked recording 002 in schema_history.""" +def _lose_migration_1_in_the_data_database(data_db): + """The data database without migration 1: its `gsi_pending` table and its + ledger row gone, so `migrate --yes` has one body to run there.""" + import psycopg2 + + conn = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS gsi_pending") + cur.execute("DELETE FROM extenddb_migrations WHERE id = 1") + finally: + conn.close() + + +def _barrier(data_db): + """A session holding an uncommitted ledger row for migration 1 in the data + database. Roll it back to let the migrator's own insert through.""" + import psycopg2 + + blocker = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) + with blocker.cursor() as cur: + cur.execute( + "INSERT INTO extenddb_migrations (id, name, kind, checksum, outcome, applied_by) " + "VALUES (1, 'baseline', 'breaking', 'barrier', 'applied', 'the test')" + ) + return blocker + + +def _wait_for_blocked_ledger_insert(data_db, exclude=(), timeout=15.0): + """Wait until a migrator waits on the barrier, inside its transaction, in + a session other than those in `exclude`; returns that session's pid.""" import psycopg2 observer = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) @@ -91,41 +130,47 @@ def _wait_for_blocked_ledger_insert(data_db, timeout=15.0): with observer.cursor() as cur: cur.execute( """ - SELECT EXISTS( - SELECT 1 FROM pg_stat_activity - WHERE datname = %s - AND state = 'active' - AND wait_event_type = 'Lock' - AND query LIKE 'INSERT INTO schema_history%%' - ) + SELECT pid FROM pg_stat_activity + WHERE datname = %s + AND state = 'active' + AND wait_event_type = 'Lock' + AND query LIKE 'INSERT INTO extenddb_migrations%%' + AND NOT (pid = ANY(%s)) """, - (data_db,), + (data_db, list(exclude)), ) - if cur.fetchone()[0]: - return + row = cur.fetchone() + if row: + return row[0] time.sleep(0.05) + with observer.cursor() as cur: + cur.execute( + "SELECT datname, pid, application_name, state, wait_event_type, left(query, 60) " + "FROM pg_stat_activity WHERE datname IN (%s, %s)", + (data_db, data_db + "_catalog"), + ) + sessions = cur.fetchall() finally: observer.close() - raise AssertionError("migrator did not block on the schema_history insert") + raise AssertionError( + f"migrator did not block on the extenddb_migrations insert; sessions: {sessions}" + ) -def _assert_002_applied_but_unrecorded(data_db): - """Prove the barrier is after migration commit and before ledger commit.""" +def _assert_applied_once(data_db): import psycopg2 - observer = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - observer.autocommit = True + conn = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) try: - with observer.cursor() as cur: + with conn.cursor() as cur: cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") - assert cur.fetchone()[0] is True, "002 schema should already be committed" - cur.execute( - "SELECT count(*) FROM schema_history " - "WHERE filename = '002_gsi_pending.sql'" - ) - assert cur.fetchone()[0] == 0, "002 ledger row should still be uncommitted" + assert cur.fetchone()[0] is True + cur.execute("SELECT count(*), max(checksum) FROM extenddb_migrations WHERE id = 1") + count, checksum = cur.fetchone() + assert count == 1 + assert checksum != "barrier", "the barrier row was committed" finally: - observer.close() + conn.close() def _terminate(proc): @@ -144,29 +189,10 @@ class TestMigrateConcurrency: """Concurrent `migrate` runs are serialized by an advisory lock.""" def test_concurrent_migrate_no_race(self, cli_env): - import psycopg2 - _init(cli_env) data_db = cli_env["db_name"][: -len("_catalog")] - - # Simulate a pre-002 deployment so migrate has real work to do. - blocker = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - blocker.autocommit = True - with blocker.cursor() as cur: - cur.execute("DROP TABLE IF EXISTS gsi_pending") - cur.execute( - "DELETE FROM schema_history WHERE filename = '002_gsi_pending.sql'" - ) - - # Insert the ledger row without committing it. A migrator's ordinary - # pending check cannot see this row, so it applies 002, then its own - # INSERT blocks on the uncommitted unique-key conflict. This gives the - # test a deterministic point after migration SQL and before recording. - blocker.autocommit = False - with blocker.cursor() as cur: - cur.execute( - "INSERT INTO schema_history (filename) VALUES ('002_gsi_pending.sql')" - ) + _lose_migration_1_in_the_data_database(data_db) + blocker = _barrier(data_db) cmd = _migrate_cmd(cli_env) p1 = None @@ -178,17 +204,14 @@ def test_concurrent_migrate_no_race(self, cli_env): stdin=subprocess.DEVNULL, bufsize=0, ) _wait_for_blocked_ledger_insert(data_db) - _assert_002_applied_but_unrecorded(data_db) - # The first migrator now holds the migration lock at a known point. - # The second must stop at that lock rather than reach its pending - # check or ledger insert. Reading this line before releasing the - # blocker also proves that wait visibility is flushed immediately. + # The first migrator holds the migration lock at a known point. + # The second stops at the lock, and says so at once. p2 = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, bufsize=0, ) - p2_prefix = _read_until(p2, "waiting for it to finish") + p2_prefix = _read_until(p2, WAITING) assert p2.poll() is None, "second migrator exited instead of waiting" blocker.rollback() @@ -197,10 +220,7 @@ def test_concurrent_migrate_no_race(self, cli_env): out1_raw = p1.communicate(timeout=60) out2_tail_raw = p2.communicate(timeout=60) out1 = (_decode(out1_raw[0]), _decode(out1_raw[1])) - out2 = ( - p2_prefix + _decode(out2_tail_raw[0]), - _decode(out2_tail_raw[1]), - ) + out2 = (p2_prefix + _decode(out2_tail_raw[0]), _decode(out2_tail_raw[1])) finally: try: if not barrier_released: @@ -217,52 +237,30 @@ def test_concurrent_migrate_no_race(self, cli_env): assert p1.returncode == 0, out1 assert p2.returncode == 0, out2 - # Exactly one process applied 002. The second took its pending snapshot - # only after the first committed the ledger row and released the lock. - outputs = [out1[0], out2[0]] - applied = [o for o in outputs if "Applying 002_gsi_pending.sql" in o] - assert len(applied) == 1, f"expected exactly one migrator to apply 002: {outputs}" - others = [o for o in outputs if "Applying 002_gsi_pending.sql" not in o] - assert len(others) == 1, outputs - assert ( - "Everything is up to date" in others[0] - or "002_gsi_pending.sql — already applied" in others[0] - ), f"the second migrator should have observed 002 as done: {others[0]}" - assert "Migration lock acquired" in out2[0], out2[0] - - # The migration and its ledger entry each landed exactly once. - conn = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - try: - with conn.cursor() as cur: - cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") - assert cur.fetchone()[0] is True - cur.execute( - "SELECT count(*) FROM schema_history " - "WHERE filename = '002_gsi_pending.sql'" - ) - assert cur.fetchone()[0] == 1 - finally: - conn.close() + # The first applied migration 1 in the data database; the second read + # the ledger only after the first committed and released the lock. + assert "Applied 1 baseline" in out1[0], out1 + assert "Applied" not in out2[0], out2 + assert "Up to date" in out2[0], out2 + _assert_applied_once(data_db) def test_sigkill_releases_lock_and_waiting_peer_recovers(self, cli_env): - """A killed lock holder releases the lock and exposes the ledger gap.""" + """A killed lock holder releases the lock, commits nothing, and the + waiting peer applies the migration. + + The killed holder's body session can outlive the process: it is blocked + on the barrier, and the database notices its client is gone only through + `client_connection_check_interval` or when the lock wait ends. The peer + takes the lock at once, then waits until no session of an earlier + migrator runs a statement, so it never runs migration 1 beside the + holder's uncommitted tables. + """ import psycopg2 _init(cli_env) data_db = cli_env["db_name"][: -len("_catalog")] - - blocker = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - blocker.autocommit = True - with blocker.cursor() as cur: - cur.execute("DROP TABLE IF EXISTS gsi_pending") - cur.execute( - "DELETE FROM schema_history WHERE filename = '002_gsi_pending.sql'" - ) - blocker.autocommit = False - with blocker.cursor() as cur: - cur.execute( - "INSERT INTO schema_history (filename) VALUES ('002_gsi_pending.sql')" - ) + _lose_migration_1_in_the_data_database(data_db) + blocker = _barrier(data_db) cmd = _migrate_cmd(cli_env) holder = None @@ -273,35 +271,46 @@ def test_sigkill_releases_lock_and_waiting_peer_recovers(self, cli_env): cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, bufsize=0, ) - _wait_for_blocked_ledger_insert(data_db) - _assert_002_applied_but_unrecorded(data_db) + holder_session = _wait_for_blocked_ledger_insert(data_db) peer = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, bufsize=0, ) - peer_waiting = _read_until(peer, "waiting for it to finish") + peer_waiting = _read_until(peer, WAITING) assert peer.poll() is None, "peer exited instead of waiting for holder" - # SIGKILL gives the holder no chance to call release. PostgreSQL must - # roll back the dedicated transaction when the connection dies, - # which releases pg_advisory_xact_lock for the waiting peer. + # SIGKILL gives the holder no chance to release. PostgreSQL ends the + # lock's session at once, which frees the lock for the waiting peer, + # and the body's session when its lock wait ends. holder.kill() holder_out_raw = holder.communicate(timeout=10) holder_out = (_decode(holder_out_raw[0]), _decode(holder_out_raw[1])) assert holder.returncode == -signal.SIGKILL, holder_out - peer_acquired = _read_until(peer, "Migration lock acquired") - assert peer.poll() is None, "peer should next block on the ledger barrier" + observer = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) + observer.autocommit = True + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + with observer.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_stat_activity WHERE pid = %s", + (holder_session,), + ) + if cur.fetchone()[0] == 0: + break + time.sleep(0.1) + else: + raise AssertionError("the killed holder's body session did not end") + finally: + observer.close() blocker.rollback() barrier_released = True - peer_tail_raw = peer.communicate(timeout=60) - peer_out = ( - peer_waiting + peer_acquired + _decode(peer_tail_raw[0]), - _decode(peer_tail_raw[1]), - ) + peer_tail_raw = peer.communicate(timeout=90) + peer_out = (peer_waiting + _decode(peer_tail_raw[0]), _decode(peer_tail_raw[1])) finally: try: if not barrier_released: @@ -315,22 +324,10 @@ def test_sigkill_releases_lock_and_waiting_peer_recovers(self, cli_env): finally: _terminate(peer) + assert WAITING in peer_out[0], peer_out[0] assert peer.returncode == 0, peer_out - assert "waiting for it to finish" in peer_out[0], peer_out[0] - assert "Migration lock acquired" in peer_out[0], peer_out[0] - - conn = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - try: - with conn.cursor() as cur: - cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") - assert cur.fetchone()[0] is True - cur.execute( - "SELECT count(*) FROM schema_history " - "WHERE filename = '002_gsi_pending.sql'" - ) - assert cur.fetchone()[0] == 1 - finally: - conn.close() + assert "Applied 1 baseline" in peer_out[0], peer_out[0] + _assert_applied_once(data_db) def test_migrate_waits_for_a_held_lock(self, cli_env): """migrate blocks on a lock held elsewhere, and says so immediately. @@ -360,9 +357,9 @@ def test_migrate_waits_for_a_held_lock(self, cli_env): _migrate_cmd(cli_env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, bufsize=0, ) - # Observe the explanation while the external session still holds - # the lock. This also guards the explicit stdout flush in Rust. - prefix = _read_until(proc, "waiting for it to finish") + # The explanation appears while the external session still holds + # the lock, so stdout is flushed before the wait. + prefix = _read_until(proc, WAITING) assert proc.poll() is None, "migrate should block while the lock is held" with conn.cursor() as cur: @@ -376,8 +373,8 @@ def test_migrate_waits_for_a_held_lock(self, cli_env): out = prefix + _decode(out_tail) err = _decode(err) assert proc.returncode == 0, (out, err) - assert "waiting for it to finish" in out, out - assert "Migration lock acquired" in out, out + assert WAITING in out, out + assert "Up to date" in out, out finally: try: if lock_held: diff --git a/tests/test_cli_settings_redaction.py b/tests/test_cli_settings_redaction.py index bc8122edf..c41b6accb 100644 --- a/tests/test_cli_settings_redaction.py +++ b/tests/test_cli_settings_redaction.py @@ -60,13 +60,18 @@ def _parse_list(stdout: str) -> dict[str, str]: def _assert_masked_list_and_get(config_path: str, secret_keys: dict[str, str]) -> None: """`secret_keys` maps each secret setting key to its stored value.""" + # No catalog seeds the delay settings, so store a non-secret value to + # check that `list` and `get` print it unmasked. + res = _settings(config_path, "set", "control_plane_delay_seconds", "0.25") + assert res.returncode == 0, res.stderr res = _settings(config_path, "list") assert res.returncode == 0, res.stderr rows = _parse_list(res.stdout) for key, stored in secret_keys.items(): assert rows.get(key) == MASK, f"{key} in list: {rows.get(key)!r}" assert stored not in res.stdout - # Non-secret keys seeded by `init` still print their values. + # Non-secret keys print their values: the one `init` writes, and the + # delay stored above. assert rows.get("data_database_name"), rows assert rows.get("control_plane_delay_seconds") == "0.25", rows diff --git a/tests/test_cli_vector_catalog_migration.py b/tests/test_cli_vector_catalog_migration.py index e50f9f175..4f8273f76 100644 --- a/tests/test_cli_vector_catalog_migration.py +++ b/tests/test_cli_vector_catalog_migration.py @@ -3,13 +3,15 @@ """Catalog migration tests against a live PostgreSQL deployment. -The vector index schema (catalog 0.0.3) and the backup table definitions -(catalog 0.0.4) are exercised through an upgrade from the pre-vector shape. - -These exercise the migration against a live deployment rather than asserting the -SQL text: init one, roll its catalog back to the pre-vector shape, and check that -the binary refuses to serve it, that `extenddb migrate` upgrades it, and that the -upgraded deployment then serves. +The vector index schema and the backup table definitions are part of +migration 1, which creates the v1.0 catalog on an empty database and brings a +released catalog to the same schema. These tests load a deployment of a +release from before the vector index schema, from its committed fixture, and +check that the binary refuses to serve it, that `extenddb migrate` upgrades +it, and that the upgraded deployment then serves. They also check that +`--reconverge` runs every convergent migration again on a current catalog, +and that a catalog ledger row for a migration this binary does not carry +makes `serve` refuse. Shared lifecycle helpers and the `cli_env` fixture live in `lifecycle_helpers.py`. Like the other CLI lifecycle tests these require a PostgreSQL instance @@ -19,11 +21,16 @@ from __future__ import annotations +import functools +import os +import re import subprocess import time from lifecycle_helpers import ( PG_ADMIN_CONN, + PG_CONN, + PG_USER, _init_args, _patch_config_port, _pg_args, @@ -34,54 +41,123 @@ VECTOR_MIGRATION = "002_vector_indexes.sql" BACKUP_DEFINITIONS_MIGRATION = "003_backup_definitions.sql" -# The version the binary expects, written by the last catalog migration. Every -# current-version assertion reads it from here so the next bump is one edit. -CURRENT_CATALOG_VERSION = "0.0.4" +# The last release before the vector index schema (catalog 0.0.2), and the +# directory of the committed fixtures: both databases of each released state +# as that release's own `init`, `serve` and seed steps left them. +PRE_VECTOR_RELEASE = "0.1.10" +RELEASED_FIXTURES = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", "crates", "storage-postgres", "tests", "fixtures", "released", +) + +# One line of `extenddb migrate --list` per migration: the id, the name, the +# kind, whether it is convergent, and the stores it records rows in. +LISTED = re.compile( + r"^\s*(\d+)\s+\S+\s+(?:breaking|additive)\s+(?:convergent|not convergent)\s+(.+)$" +) + + +# The same lines, for a migration that is not convergent. +NOT_CONVERGENT = re.compile(r"^\s*(\d+)\s+\S+\s+(?:breaking|additive)\s+not convergent\s") + + +@functools.cache +def _not_convergent(): + """The ids `migrate --list` marks as not convergent, which `--reconverge` + does not run again once they are recorded.""" + listed = _run_extenddb("migrate", "--list") + return { + int(match.group(1)) + for match in map(NOT_CONVERGENT.match, listed.stdout.splitlines()) + if match + } + + +@functools.cache +def _listed(): + """The stores of every migration the binary carries, by id, from + `migrate --list`, which opens no database. Adding a migration then needs + no edit here; a literal id in a test means that migration itself.""" + listed = _run_extenddb("migrate", "--list") + stores = {} + for line in listed.stdout.splitlines(): + match = LISTED.match(line) + if match: + stores[int(match.group(1))] = [s.strip() for s in match.group(2).split(",")] + assert stores, listed.stdout + return stores -# A version no release has reached, for the symmetric version-gate test. -FUTURE_CATALOG_VERSION = "0.0.5" +def _latest(): + """The highest migration the binary carries, which the runner writes to + `catalog_version`.""" + return max(_listed()) -def _catalog_conn(cli_env): + +def _ids_in(store): + """The ids a complete run records in `store` ("catalog" or "data").""" + return [i for i, stores in sorted(_listed().items()) if store in stores] + + +def _conn(database): import psycopg2 - return psycopg2.connect(PG_ADMIN_CONN + "/" + cli_env["db_name"]) + conn = psycopg2.connect(PG_ADMIN_CONN + "/" + database) + conn.autocommit = True + return conn + + +def _data_db(cli_env): + return cli_env["db_name"][: -len("_catalog")] -def _catalog_query(cli_env, sql): - conn = _catalog_conn(cli_env) +def _query(database, sql, params=()): + conn = _conn(database) try: with conn.cursor() as cur: - cur.execute(sql) + cur.execute(sql, params) return cur.fetchone()[0] finally: conn.close() def _catalog_version(cli_env): - return _catalog_query( - cli_env, "SELECT value FROM settings WHERE key = 'catalog_version'" + return _query( + cli_env["db_name"], "SELECT value FROM settings WHERE key = 'catalog_version'" ) def _vector_table_exists(cli_env): - return _catalog_query( - cli_env, "SELECT to_regclass('public.vector_indexes') IS NOT NULL" - ) + return _query(cli_env["db_name"], "SELECT to_regclass('public.vector_indexes') IS NOT NULL") def _backups_has_vector_column(cli_env): - return _catalog_query( - cli_env, + return _query( + cli_env["db_name"], "SELECT EXISTS(SELECT 1 FROM information_schema.columns " "WHERE table_name = 'backups' AND column_name = 'vector_indexes')", ) -def _backup_definitions_table_exists(cli_env): - return _catalog_query( - cli_env, "SELECT to_regclass('public.backup_definitions') IS NOT NULL" - ) +def _ledger_rows(database): + """`(id, checksum, applied_at)` of every row of a store's ledger.""" + conn = _conn(database) + try: + with conn.cursor() as cur: + cur.execute("SELECT id, checksum, applied_at FROM extenddb_migrations ORDER BY id") + return cur.fetchall() + finally: + conn.close() + + +def _ledger_ids(database): + conn = _conn(database) + try: + with conn.cursor() as cur: + cur.execute("SELECT id FROM extenddb_migrations ORDER BY id") + return [row[0] for row in cur.fetchall()] + finally: + conn.close() def _init(cli_env): @@ -93,91 +169,136 @@ def _init(cli_env): assert result.returncode == 0, result.stdout + result.stderr -def _roll_catalog_back_to_pre_vector(cli_env): - """Turn a freshly initialised catalog into the shape 0.0.2 deployments have. +def _serve_refused(cli_env): + """Run `serve --foreground`; it must exit on its own, and the result is + returned. A server that starts serving is stopped and fails the test.""" + try: + return _run_extenddb( + "serve", "--foreground", + config=cli_env["config_path"], + check=False, + timeout=20, + ) + except subprocess.TimeoutExpired: + _run_extenddb("stop", config=cli_env["config_path"], check=False) + raise AssertionError("serve started against a catalog it must refuse") from None + + +def _load_released_state(cli_env, state): + """Replace the deployment's two databases with a released state's fixture. - Reproduces an upgrade rather than a fresh install, which is the case that - matters: a fresh install applies the migrations in order and reaches the same - end state trivially. + `extenddb destroy` removes the databases `init` created, keeping the + config file. They are created again, owned by the application role, and + each fixture is loaded as that role, as the release's own `init` left it. + The catalog's connection rows then point at the test's data database. """ - conn = _catalog_conn(cli_env) + import psycopg2 + + destroyed = _run_extenddb( + "destroy", "--yes", *_pg_args(), config=cli_env["config_path"], check=False + ) + assert destroyed.returncode == 0, destroyed.stdout + destroyed.stderr + databases = (cli_env["db_name"], _data_db(cli_env)) + admin = psycopg2.connect(PG_ADMIN_CONN + "/postgres") + admin.autocommit = True try: + with admin.cursor() as cur: + for database in databases: + cur.execute(f'CREATE DATABASE "{database}" OWNER "{PG_USER}"') + finally: + admin.close() + for database, dump in zip(databases, ("catalog.sql", "data.sql")): + with open(os.path.join(RELEASED_FIXTURES, state, dump), encoding="utf-8") as f: + sql = f.read() + conn = psycopg2.connect(PG_CONN + "/" + database) conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute(sql) + finally: + conn.close() + conn = _conn(cli_env["db_name"]) + try: with conn.cursor() as cur: - cur.execute("DROP TABLE IF EXISTS backup_definitions") - cur.execute("DROP TABLE IF EXISTS vector_indexes") - cur.execute("ALTER TABLE backups DROP COLUMN IF EXISTS vector_indexes") - cur.execute( - "DELETE FROM schema_history WHERE filename IN (%s, %s)", - (VECTOR_MIGRATION, BACKUP_DEFINITIONS_MIGRATION), - ) - cur.execute("UPDATE settings SET value = '0.0.2' WHERE key = 'catalog_version'") + for key, value in ( + ("data_database_connection_string", PG_CONN + "/" + _data_db(cli_env)), + ("data_database_name", _data_db(cli_env)), + ): + cur.execute("UPDATE settings SET value = %s WHERE key = %s", (value, key)) finally: conn.close() class TestVectorCatalogMigration: - """Migrations from the pre-vector catalog shape to the current version.""" + """Migrations from a catalog a release before 1.0 created, on the ledger.""" - def test_init_creates_the_catalog_at_the_current_version(self, cli_env): - """A fresh init applies every catalog migration and records the version. + def test_init_creates_the_catalog_at_the_current_migration(self, cli_env): + """A fresh init records every migration in both ledgers and the version. - The version is what the binary checks at startup, so a migration that - creates the table without moving the version, or the reverse, would leave - a deployment that cannot serve. + `serve` reads the status from the ledgers, so a migration that creates + its tables without recording them, or the reverse, would leave a + deployment that cannot serve. """ _init(cli_env) - assert _catalog_version(cli_env) == CURRENT_CATALOG_VERSION + assert _catalog_version(cli_env) == str(_latest()) assert _vector_table_exists(cli_env) is True assert _backups_has_vector_column(cli_env) is True - assert _backup_definitions_table_exists(cli_env) is True - - conn = _catalog_conn(cli_env) + assert _query( + cli_env["db_name"], "SELECT to_regclass('public.backup_definitions') IS NOT NULL" + ) is True + assert _ledger_ids(cli_env["db_name"]) == list(range(1, _latest() + 1)) + assert _ids_in("catalog") == list(range(1, _latest() + 1)) + data_ids = _ledger_ids(_data_db(cli_env)) + assert data_ids == _ids_in("data") + # Migrations 1 and 4 have bodies in the data database. + assert data_ids[:2] == [1, 4], data_ids + + # Migration 1 also writes the legacy ledger rows, so a release before + # 1.0 started against this catalog finds no legacy file to replay. + conn = _conn(cli_env["db_name"]) try: with conn.cursor() as cur: cur.execute("SELECT filename FROM schema_history ORDER BY filename") tracked = {row[0] for row in cur.fetchall()} finally: conn.close() - # Recorded, so a later migrate does not walk the file list again on a - # deployment that is already current. The statements are idempotent too, - # so a replay after a crash between applying and recording is harmless. assert VECTOR_MIGRATION in tracked, tracked assert BACKUP_DEFINITIONS_MIGRATION in tracked, tracked - def test_migrate_upgrades_a_pre_vector_deployment(self, cli_env): - """A 0.0.2 deployment is refused, upgraded by migrate, and then serves.""" + def test_migrate_upgrades_a_catalog_a_release_before_1_0_created(self, cli_env): + """A v0.1.10 deployment, from before the vector index schema, is + refused, upgraded by migrate, and then serves. + + The deployment is that release's committed fixture: no migration + ledger, catalog version 0.0.2, no vector tables. A fresh install with + its ledger removed is not the same state: it also holds what later + migrations added, which migration 1's check of its result refuses as + not part of the v1.0 schema. + """ _init(cli_env) _patch_config_port(cli_env["config_path"], cli_env["port"]) - _roll_catalog_back_to_pre_vector(cli_env) + _load_released_state(cli_env, PRE_VECTOR_RELEASE) assert _vector_table_exists(cli_env) is False - # The binary must refuse to serve a catalog it was not built for, rather - # than starting and failing on the first request that reads the table. - try: - refused_serve = _run_extenddb( - "serve", "--foreground", - config=cli_env["config_path"], - check=False, - timeout=20, - ) - except subprocess.TimeoutExpired: - _run_extenddb("stop", config=cli_env["config_path"], check=False) - raise AssertionError( - "serve started against a 0.0.2 catalog; the version gate did not hold" - ) from None - combined = refused_serve.stdout + refused_serve.stderr - assert refused_serve.returncode != 0, combined - assert CURRENT_CATALOG_VERSION in combined and "0.0.2" in combined, combined - - # Without --yes, migrate reports the pending upgrade and changes nothing. + # The binary refuses a catalog it was not built for (exit 3: migrations + # pending) rather than starting and failing on the first request that + # reads the table. + refused = _serve_refused(cli_env) + combined = refused.stdout + refused.stderr + assert refused.returncode == 3, combined + assert "no migration ledger" in combined, combined + assert f"needs migration {_latest()}" in combined, combined + + # Without --yes, migrate reports the pending migrations and changes + # nothing. pending = _run_extenddb( "migrate", *_pg_args(), config=cli_env["config_path"], check=False ) pending_output = pending.stdout + pending.stderr - assert pending.returncode != 0, pending_output - assert f"0.0.2 -> {CURRENT_CATALOG_VERSION}" in pending_output, pending_output + assert pending.returncode == 3, pending_output + assert "1 baseline (catalog, breaking)" in pending_output, pending_output + assert "1 baseline (data, breaking)" in pending_output, pending_output assert _vector_table_exists(cli_env) is False assert _catalog_version(cli_env) == "0.0.2" @@ -185,12 +306,14 @@ def test_migrate_upgrades_a_pre_vector_deployment(self, cli_env): "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False ) assert applied.returncode == 0, applied.stdout + applied.stderr - assert _catalog_version(cli_env) == CURRENT_CATALOG_VERSION + assert f"Catalog: no ledger -> migration {_latest()}" in applied.stdout, applied.stdout + assert _catalog_version(cli_env) == str(_latest()) assert _vector_table_exists(cli_env) is True assert _backups_has_vector_column(cli_env) is True + assert _ledger_ids(cli_env["db_name"]) == _ids_in("catalog") + assert _ledger_ids(_data_db(cli_env)) == _ids_in("data") - # The upgraded deployment serves, which is the assertion the version and - # the table shape are both really for. + # The upgraded deployment serves. served = _run_extenddb("serve", config=cli_env["config_path"]) assert served.returncode == 0, served.stdout + served.stderr try: @@ -199,8 +322,7 @@ def test_migrate_upgrades_a_pre_vector_deployment(self, cli_env): _run_extenddb("stop", config=cli_env["config_path"], check=False) time.sleep(1) - # A second migrate is a no-op: the ledger row stops the non-idempotent - # migration from running twice. + # A second migrate finds every migration recorded. again = _run_extenddb( "migrate", *_pg_args(), config=cli_env["config_path"], check=False ) @@ -208,110 +330,73 @@ def test_migrate_upgrades_a_pre_vector_deployment(self, cli_env): assert again.returncode == 0, again_output assert "up to date" in again_output.lower(), again_output - def test_migrate_survives_an_applied_but_unrecorded_migration(self, cli_env): - """A replay of the vector migration must succeed, not block the deployment. - - The runner applies a migration and records it in `schema_history` as two - separate commits, so a crash in between leaves the schema applied and the - ledger short a row. That state is silent at first, because the catalog - version moved with the schema and the startup gate is satisfied. It bites - when the next catalog migration lands: the runner then walks every file, - finds this one unrecorded, and applies it a second time. - - This reproduces that second run. The version is set to a value that makes - the runner walk the list, with the vector schema still present and its - ledger row missing, which is exactly the state a crash leaves behind. - Without idempotent statements the replay fails on "relation already - exists" and no later migration can ever be applied. + def test_reconverge_runs_every_convergent_migration_again_on_a_current_catalog( + self, cli_env + ): + """`migrate --yes --reconverge` runs each convergent migration again on + a catalog at the latest migration, names each one that is not + convergent as skipped, and leaves the catalog current, with its ledger + rows as they were. + + This is the command an operator runs after a release before 1.0 wrote + to a migrated catalog. Migration 1's check of its result does not run + again, because a later migration may have changed what it created. + Migration 2 is not convergent: it would rewrite policies written + through the API since it ran. """ _init(cli_env) - assert _vector_table_exists(cli_env) is True + stores = (cli_env["db_name"], _data_db(cli_env)) + rows_before = [_ledger_rows(database) for database in stores] - conn = _catalog_conn(cli_env) - try: - conn.autocommit = True - with conn.cursor() as cur: - # Ledger row gone, schema left in place: the crash window. - cur.execute( - "DELETE FROM schema_history WHERE filename = %s", (VECTOR_MIGRATION,) - ) - # Force the runner to walk the file list, the way a later - # migration would. - cur.execute( - "UPDATE settings SET value = '0.0.2' WHERE key = 'catalog_version'" - ) - finally: - conn.close() - - replayed = _run_extenddb( - "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False + rerun = _run_extenddb( + "migrate", "--yes", "--reconverge", *_pg_args(), + config=cli_env["config_path"], + check=False, ) - output = replayed.stdout + replayed.stderr - assert replayed.returncode == 0, output - # The file was re-applied rather than skipped by the ledger, which is what - # makes this a replay and not a no-op. - assert f"Applying {VECTOR_MIGRATION}" in output, output - # PostgreSQL emits "already exists, skipping" notices here, which is the - # guards working. What must not appear is the runner's failure line. - assert f"Migration {VECTOR_MIGRATION} failed" not in output, output - - # The replay leaves the same end state, and the ledger is repaired. - assert _catalog_version(cli_env) == CURRENT_CATALOG_VERSION - assert _vector_table_exists(cli_env) is True - assert _backups_has_vector_column(cli_env) is True - conn = _catalog_conn(cli_env) - try: - with conn.cursor() as cur: - cur.execute( - "SELECT COUNT(*) FROM schema_history WHERE filename = %s", - (VECTOR_MIGRATION,), - ) - assert cur.fetchone()[0] == 1 - finally: - conn.close() - - def test_serve_refuses_a_catalog_newer_than_the_binary(self, cli_env): - """The version gate must refuse a catalog from a future release too. - - The upgrade direction (new binary, old catalog) is covered above. This is - the other one, which matters during a rolling upgrade: a replica still - running the old build must refuse a catalog a newer replica has already - migrated, rather than serving against a schema it does not understand. - The check is an exact-equality comparison, so both directions come from - the same line, and pinning them both is what keeps that true. - - Simulated by moving the stored version forward, which needs no second - binary. + output = rerun.stdout + rerun.stderr + assert rerun.returncode == 0, output + assert 2 in _not_convergent(), _not_convergent() + for i in _ids_in("catalog"): + if i in _not_convergent(): + assert re.search(rf"^\s*Skipped {i} \S+: ", output, re.M), output + assert f"Ran again {i} " not in output, output + else: + assert f"Ran again {i} " in output, output + assert [_ledger_rows(database) for database in stores] == rows_before + assert _catalog_version(cli_env) == str(_latest()) + + verify = _run_extenddb("verify", config=cli_env["config_path"], check=False) + assert verify.returncode == 0, verify.stdout + verify.stderr + assert "OK: Current" in verify.stdout, verify.stdout + + def test_serve_refuses_a_catalog_ahead_by_a_breaking_migration(self, cli_env): + """A catalog with a breaking migration this binary does not carry. + + The upgrade direction (new binary, old catalog) is covered above. This + is the other one: a binary rolled back after a breaking migration must + refuse the catalog (exit 4) rather than write rows in the shape that + migration replaced. Simulated with a ledger row for the next id, which + needs no second binary. """ _init(cli_env) _patch_config_port(cli_env["config_path"], cli_env["port"]) - - conn = _catalog_conn(cli_env) + later = _latest() + 1 + conn = _conn(cli_env["db_name"]) try: - conn.autocommit = True with conn.cursor() as cur: cur.execute( - "UPDATE settings SET value = %s WHERE key = 'catalog_version'", - (FUTURE_CATALOG_VERSION,), + "INSERT INTO extenddb_migrations " + "(id, name, kind, checksum, outcome, applied_by) " + "VALUES (%s, 'later', 'breaking', 'x', 'applied', 'a later release')", + (later,), ) finally: conn.close() - try: - refused = _run_extenddb( - "serve", "--foreground", - config=cli_env["config_path"], - check=False, - timeout=20, - ) - except subprocess.TimeoutExpired: - _run_extenddb("stop", config=cli_env["config_path"], check=False) - raise AssertionError( - f"serve started against a {FUTURE_CATALOG_VERSION} catalog; " - "the version gate is not symmetric" - ) from None + refused = _serve_refused(cli_env) combined = refused.stdout + refused.stderr - assert refused.returncode != 0, combined - assert CURRENT_CATALOG_VERSION in combined and FUTURE_CATALOG_VERSION in combined, ( - combined - ) + assert refused.returncode == 4, combined + assert f"{later} later" in combined, combined + + verify = _run_extenddb("verify", config=cli_env["config_path"], check=False) + assert verify.returncode == 4, verify.stdout + verify.stderr diff --git a/tests/test_resource_tag_migration.py b/tests/test_resource_tag_migration.py index 38f682a1a..4a1252ee4 100644 --- a/tests/test_resource_tag_migration.py +++ b/tests/test_resource_tag_migration.py @@ -7,19 +7,24 @@ directory and port, independent of the server the integration job already started: - 1. init (a fresh catalog carries the migration marker) and serve; the first - start of a fresh catalog logs no migration line. - 2. store an inline user policy, a group policy, a permissions boundary and a - role trust policy using every operator shape of dynamodb:ResourceTag, - plus a conditional allow used for the decision check, through the - management API. - 3. stop, run `migrate` without --yes: it lists each rewrite with a count and - exits non-zero, and nothing on disk changes (read back with the sqlite3 - module). - 4. run `migrate --yes`, restart, and confirm a GetItem gated on the resource - tag that was denied before the migration is allowed after it. - 5. delete the marker row with the sqlite3 module and confirm serve refuses to - start, naming `extenddb migrate --yes`. + 1. init and serve; the first start of a fresh catalog logs no migration + line. + 2. store an inline user policy, a group policy and a permissions boundary + using every operator shape of dynamodb:ResourceTag, plus a conditional + allow used for the decision check, through the management API. + 3. stop, and make the catalog read as one a release before 1.0 left: an + empty migration ledger, a dotted catalog_version and the old marker row. + 4. run `migrate` without --yes: it lists the pending migrations, the + condition-key rewrite included, and exits 3, and nothing on disk changes + (read back with the sqlite3 module). + 5. run `migrate --yes`: it names each rewritten policy with its key count; + restart, and confirm a GetItem gated on the resource tag that was denied + before the migration is allowed after it. + 6. write the marker row again, as a release before 1.0 does, and confirm + serve refuses with exit 5 naming `--reconverge`, which repairs it. The + reconverge skips the condition-key rewrite, which is not convergent: a + policy written through the API after the upgrade keeps its + dynamodb:ResourceTag key, and the GetItem it would allow stays denied. The binary is found the way tests/lifecycle_helpers.py does, falling back to target/release/extenddb, so the SQLite integration job runs it against the @@ -52,6 +57,10 @@ MIGRATION_MARKER_KEY = "resource_tag_migration" +# Exit codes `serve` and `migrate` share. +EXIT_PENDING = 3 +EXIT_ATTENTION = 5 + # This test stands up its own deployment, so it must not inherit admin # credentials or an endpoint from an outer harness (devtools/run-tests and the # SQLite job export EXTENDDB_ADMIN_USER / EXTENDDB_ADMIN_PASSWORD / @@ -284,6 +293,38 @@ def deployment(tmp_path): server.stop() +def _as_released_catalog(sqlite_path: str) -> None: + """Make a catalog this binary created read as one a release before 1.0 + left: no ledger rows (the runner reads an empty ledger as it reads a + missing one), a dotted catalog_version, and the marker row every released + baseline seeded. The runner then applies every migration, the + condition-key rewrite included, as it does on a released catalog.""" + conn = sqlite3.connect(sqlite_path) + try: + conn.execute("DELETE FROM extenddb_migrations") + conn.execute( + "UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'" + ) + conn.execute( + "INSERT INTO settings (key, value) VALUES (?, 'applied')", + (MIGRATION_MARKER_KEY,), + ) + conn.commit() + finally: + conn.close() + + +def _sqlite_ledger_ids(sqlite_path: str) -> list[int]: + conn = sqlite3.connect(sqlite_path) + try: + return [ + r[0] + for r in conn.execute("SELECT id FROM extenddb_migrations ORDER BY id") + ] + finally: + conn.close() + + def _sqlite_policy_docs(sqlite_path: str) -> list[str]: conn = sqlite3.connect(sqlite_path) try: @@ -301,8 +342,8 @@ def test_resource_tag_migration_wire(deployment): sqlite_path = deployment["sqlite_path"] region = "us-east-1" - # 1. First start of a fresh catalog: marker is seeded, so serve starts and - # logs nothing about the migration. + # 1. The first start of a fresh catalog, which init migrated, logs + # nothing about the condition-key migration. server.start() log = server.read_log() assert "resource-tag condition-key migration" not in log, log @@ -365,56 +406,99 @@ def test_resource_tag_migration_wire(deployment): scoped_client.get_item(TableName=table, Key={"pk": {"S": "k1"}}) assert exc.value.response["Error"]["Code"] == "AccessDeniedException" - # 2. Stop and run migrate WITHOUT --yes: it lists the rewrites with counts - # and exits 3 (work pending, as distinct from an error), and nothing on - # disk changes. + # 2. Stop, and make the catalog read as one a release before 1.0 left. server.stop() + _as_released_catalog(sqlite_path) before_docs = _sqlite_policy_docs(sqlite_path) assert any("dynamodb:ResourceTag" in d for d in before_docs) + # 3. migrate WITHOUT --yes lists the pending migrations and exits 3 (work + # pending, as distinct from an error), and nothing on disk changes. dry = _run("migrate", "--config", deployment["config"]) - assert dry.returncode == 3, dry.stdout + dry.stderr out = dry.stdout + dry.stderr - assert "Rewrite:" in out, out - assert "resource-tag condition-key rewrites" in out, out + assert dry.returncode == EXIT_PENDING, out + assert "2 resource_tag_condition_keys (catalog, breaking)" in out, out assert "run again with --yes to apply" in out, out - # The listing names policies and key counts, never a policy body. - assert "key(s))" in out, out - # Nothing changed on disk. assert _sqlite_policy_docs(sqlite_path) == before_docs + assert _sqlite_ledger_ids(sqlite_path) == [] - # 3. migrate --yes rewrites and records the marker. + # 4. migrate --yes rewrites, naming each policy with its key count and + # never a policy body, and deletes the old marker. applied = _run("migrate", "--yes", "--config", deployment["config"]) - assert applied.returncode == 0, applied.stdout + applied.stderr + out = applied.stdout + applied.stderr + assert applied.returncode == 0, out + assert "rewrote account" in out and "key(s))" in out, out + assert "policy tagpolicy" in out and "policy shapes" in out, out after_docs = _sqlite_policy_docs(sqlite_path) assert all("dynamodb:ResourceTag" not in d for d in after_docs), after_docs assert any("aws:ResourceTag" in d for d in after_docs) + assert _sqlite_marker(sqlite_path) == [] - # 4. Restart and confirm the decision flipped: the rewritten aws:ResourceTag + # 5. Restart and confirm the decision flipped: the rewritten aws:ResourceTag # policy now resolves the tag, so GetItem on the env=prod table is allowed. server.start() scoped_client = _client(endpoint, scoped_ak, scoped_sk, region) got = scoped_client.get_item(TableName=table, Key={"pk": {"S": "k1"}}) assert got["Item"]["v"]["S"] == "hi" - # A second migrate is idempotent: nothing left to rewrite, marker present. + # A policy written through the API after the upgrade, on the + # dynamodb:ResourceTag form: the key is absent, as on the service, so the + # GetItem it would allow is denied. + assert mgmt.create_user(account_id, "later", "TestPass123!").status_code == 201 + later = mgmt.create_access_key(account_id, "later").json() + assert mgmt.put_user_policy( + account_id, "later", "laterpolicy", _policy("dynamodb:ResourceTag/env"), + ).status_code == 204 + later_client = _client( + endpoint, later["access_key_id"], later["secret_access_key"], region + ) + with pytest.raises(ClientError) as exc: + later_client.get_item(TableName=table, Key={"pk": {"S": "k1"}}) + assert exc.value.response["Error"]["Code"] == "AccessDeniedException" + + # A second migrate finds nothing to apply. server.stop() again = _run("migrate", "--config", deployment["config"]) assert again.returncode == 0, again.stdout + again.stderr - assert "No dynamodb:ResourceTag condition keys found." in (again.stdout + again.stderr) + assert "Up to date" in (again.stdout + again.stderr) - # 5. Delete the marker row with the sqlite3 module and confirm serve refuses. + # 6. A release before 1.0 writes the marker row again; serve refuses the + # catalog with exit 5 and names --reconverge, which repairs it. conn = sqlite3.connect(sqlite_path) try: - conn.execute("DELETE FROM settings WHERE key = ?", (MIGRATION_MARKER_KEY,)) + conn.execute( + "INSERT INTO settings (key, value) VALUES (?, 'applied')", + (MIGRATION_MARKER_KEY,), + ) conn.commit() finally: conn.close() refused = _run("serve", "--config", deployment["config"], "--foreground", timeout=60) - assert refused.returncode != 0 - assert "extenddb migrate --yes" in (refused.stdout + refused.stderr), ( - refused.stdout + refused.stderr + out = refused.stdout + refused.stderr + assert refused.returncode == EXIT_ATTENTION, out + assert "--yes --reconverge" in out, out + repaired = _run("migrate", "--yes", "--reconverge", "--config", deployment["config"]) + out = repaired.stdout + repaired.stderr + assert repaired.returncode == 0, out + assert _sqlite_marker(sqlite_path) == [] + + # The reconverge skipped the rewrite, which is not convergent: the policy + # written after the upgrade keeps its key, and its GetItem stays denied, + # while the policy the migration rewrote still allows it. + assert "Skipped 2 resource_tag_condition_keys:" in out, out + assert "dynamodb:ResourceTag/env" in json.dumps( + _sqlite_policy(sqlite_path, "laterpolicy") ) + server.start() + later_client = _client( + endpoint, later["access_key_id"], later["secret_access_key"], region + ) + with pytest.raises(ClientError) as exc: + later_client.get_item(TableName=table, Key={"pk": {"S": "k1"}}) + assert exc.value.response["Error"]["Code"] == "AccessDeniedException" + scoped_client = _client(endpoint, scoped_ak, scoped_sk, region) + got = scoped_client.get_item(TableName=table, Key={"pk": {"S": "k1"}}) + assert got["Item"]["v"]["S"] == "hi" def _sqlite_marker(sqlite_path: str) -> list: @@ -458,10 +542,10 @@ def test_resource_tag_migration_collisions(deployment): upgrade, so the two conditions are one condition. migrate drops the dynamodb: key and the Deny keeps applying. - Different values: the two cannot share one JSON key. migrate leaves the - policy unchanged, exits non-zero and does not record the marker, so serve - keeps refusing the catalog until the policy is fixed and migrate is run - again. Recording the marker here would change the decision: under + Different values: the two cannot share one JSON key. migrate finds the + collision before it writes anything and exits 5, and serve refuses the + catalog with exit 5 until the policy is fixed and migrate is run again. + Applying the migration here would change the decision: under StringNotEquals the dynamodb: half, now absent, always matches. """ from management_helpers import ManagementClient @@ -527,44 +611,41 @@ def test_resource_tag_migration_collisions(deployment): admin_client.put_item(TableName=table, Item={"pk": {"S": "k1"}}) server.stop() - # Simulate a catalog created before the migration existed. - conn = sqlite3.connect(sqlite_path) - try: - conn.execute("DELETE FROM settings WHERE key = ?", (MIGRATION_MARKER_KEY,)) - conn.commit() - finally: - conn.close() + # A catalog a release before 1.0 left, with both policies stored on it. + _as_released_catalog(sqlite_path) + same_before = _sqlite_policy(sqlite_path, "hedged-same") + # The collision is found before anything is written: no ledger rows, and + # both policies as they were, the mergeable one included. applied = _run("migrate", "--yes", "--config", deployment["config"]) out = applied.stdout + applied.stderr - assert applied.returncode != 0, out - assert "NEEDS ATTENTION" in out, out - attention = [line for line in out.splitlines() if "NEEDS ATTENTION" in line] - assert any("hedged-split" in line for line in attention), out - assert not any("hedged-same" in line for line in attention), out - assert _sqlite_marker(sqlite_path) == [] - # The different-value policy is unchanged; the same-value one keeps only - # the aws: key. + assert applied.returncode == EXIT_ATTENTION, out + assert "policy hedged-split" in out, out + assert "policy hedged-same" not in out, out + assert _sqlite_ledger_ids(sqlite_path) == [] assert _sqlite_policy(sqlite_path, "hedged-split") == split_doc - same = _sqlite_policy(sqlite_path, "hedged-same") - assert same["Statement"][1]["Condition"] == { - "StringEquals": {"aws:ResourceTag/env": "prod"} - } + assert _sqlite_policy(sqlite_path, "hedged-same") == same_before refused = _run("serve", "--config", deployment["config"], "--foreground", timeout=60) - assert refused.returncode != 0 - assert "extenddb migrate --yes" in (refused.stdout + refused.stderr), ( - refused.stdout + refused.stderr - ) + out = refused.stdout + refused.stderr + assert refused.returncode == EXIT_ATTENTION, out + assert "policy hedged-split" in out, out + assert "extenddb migrate --config" in out, out # Fix the different-value policy by hand (keep the aws: form) and run - # migrate again: it completes and records the marker. + # migrate again: it applies, and the same-value policy keeps only the + # aws: key. fixed = json.loads(json.dumps(split_doc)) del fixed["Statement"][0]["Condition"]["StringNotEquals"]["dynamodb:ResourceTag/env"] _sqlite_set_policy(sqlite_path, "hedged-split", fixed) again = _run("migrate", "--yes", "--config", deployment["config"]) assert again.returncode == 0, again.stdout + again.stderr - assert _sqlite_marker(sqlite_path) != [] + assert _sqlite_ledger_ids(sqlite_path) != [] + assert _sqlite_marker(sqlite_path) == [] + same = _sqlite_policy(sqlite_path, "hedged-same") + assert same["Statement"][1]["Condition"] == { + "StringEquals": {"aws:ResourceTag/env": "prod"} + } # The same-value Deny still applies after the upgrade. server.start()