feat(migrations): one list of catalog migrations with a ledger in every store, for SQLite and PostgreSQL

.github/workflows/integration.yml

@@ -89,6 +89,61 @@ jobs:
       - name: Install Python dependencies
         run: pip install -r requirements.txt
 
+      # A fresh install's schema, from the catalog init created above before
+      # any test ran, against the committed file. On a difference the job
+      # fails with the diff and uploads the regenerated file to commit.
+      - name: Check the current schema
+        run: |
+          devtools/schema-dump postgres postgresql://postgres:devpass@127.0.0.1:5432 \
+            extenddb_catalog extenddb > schema-postgres.txt
+          if ! diff -u docs/schema/postgres.txt schema-postgres.txt; then
+            echo "docs/schema/postgres.txt differs from a fresh install: commit the schema-postgres artifact as docs/schema/postgres.txt"
+            exit 1
+          fi
+
+      - name: Upload the regenerated schema
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: schema-postgres
+          path: schema-postgres.txt
+          if-no-files-found: ignore
+
+      # PgBouncer 1.26.1 in transaction mode in front of the job's server, for
+      # the lock script's pass through a connection pooler. It logs every role
+      # in through auth_query as the superuser, so the script's own role needs
+      # no entry of its own.
+      - name: Start PgBouncer
+        run: >-
+          docker run -d --name pgbouncer --network host
+          -e DB_HOST=127.0.0.1 -e DB_PORT=5432
+          -e DB_USER=postgres -e DB_PASSWORD=devpass -e AUTH_USER=postgres
+          -e AUTH_TYPE=scram-sha-256 -e POOL_MODE=transaction
+          -e LISTEN_ADDR=127.0.0.1 -e LISTEN_PORT=6432
+          edoburu/pgbouncer@sha256:8638aa563607256d3c7440437534195f1066a92b68015d89ae9d6b935b444a6d
+
+      # The migration lock and the serving registration between serve and
+      # migrate, on a deployment of its own (port 18452), then the commands
+      # and serve with the catalog reached through PgBouncer.
+      - name: Check serve and migrate on one catalog
+        env:
+          POOLER_URL: postgresql://127.0.0.1:6432
+        run: >-
+          devtools/check-postgres-catalog-lock ./target/release/extenddb
+          postgresql://postgres:devpass@127.0.0.1:5432 18452
+
+      # The last 0.1.x image's verify, migrate --yes and serve against
+      # catalogs this binary created and migrated: what each writes, and that
+      # this binary then reports WrittenByOlderRelease and --reconverge
+      # restores Current.
+      - name: Check the last 0.1.x release against migrated catalogs
+        run: |
+          docker pull extenddb/extenddb-postgres:0.1.14
+          devtools/check-postgres-older-binary ./target/release/extenddb \
+            postgresql://postgres:devpass@127.0.0.1:5432 \
+            crates/storage-postgres/tests/fixtures/released/0.1.14 \
+            extenddb/extenddb-postgres:0.1.14 18453
+
       - name: Run integration tests
         env:
           EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
@@ -120,6 +175,21 @@ jobs:
       - name: Build release (SQLite backend)
         run: cargo build --release -p extenddb --no-default-features --features sqlite
 
+      # The migration lock between serve and migrate, on a catalog of its own
+      # (port 18450), before the suite's server starts.
+      - name: Check serve and migrate on one catalog
+        run: devtools/check-sqlite-catalog-lock ./target/release/extenddb 18450
+
+      # The last 0.1.x dev image's serve and migrate --yes against catalogs
+      # this binary migrated: what each writes, and that this binary then
+      # reports WrittenByOlderRelease and --reconverge restores Current.
+      - name: Check the last 0.1.x release against migrated catalogs
+        run: |
+          docker pull extenddb/extenddb-dev:0.1.14
+          devtools/check-sqlite-older-binary ./target/release/extenddb \
+            crates/storage-sqlite/tests/fixtures/released/dev-image-0.1.14.sql \
+            extenddb/extenddb-dev:0.1.14 18451
+
       - name: Initialize ExtendDB
         id: init
         run: |
@@ -128,6 +198,24 @@ jobs:
           password=$(echo "$output" | grep -oP 'Password: \K\S+')
           echo "admin_password=$password" >> "$GITHUB_OUTPUT"
 
+      # A fresh install's schema against the committed file. On a difference
+      # the job fails with the diff and uploads the regenerated file to commit.
+      - name: Check the current schema
+        run: |
+          devtools/schema-dump sqlite extenddb.sqlite > schema-sqlite.txt
+          if ! diff -u docs/schema/sqlite.txt schema-sqlite.txt; then
+            echo "docs/schema/sqlite.txt differs from a fresh install: commit the schema-sqlite artifact as docs/schema/sqlite.txt"
+            exit 1
+          fi
+
+      - name: Upload the regenerated schema
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: schema-sqlite
+          path: schema-sqlite.txt
+          if-no-files-found: ignore
+
       - name: Start ExtendDB
         run: |
           # --write-pid-file so devtools/run-tests can restart the server with
@@ -234,6 +322,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
+        with:
+          # migrations_tag_blobs reads the release tags.
+          fetch-depth: 0
 
       - uses: dtolnay/rust-toolchain@stable
 
@@ -303,7 +394,10 @@ jobs:
       # cases need catalog rows the current binary would not write; all need
       # direct access. The online GSI build tests reorder queue rows by hand and
       # pause writers with a test hook. They build their own throwaway databases;
-      # the connection string is the server, not a database.
+      # the connection string is the server, not a database. `--tests` runs
+      # every target of the crate, so a new test file needs no edit here, and
+      # EXTENDDB_TEST_PG_REQUIRED turns a test's skip for a missing connection
+      # string into a failure.
       - name: Run PostgreSQL storage-level tests
         # table_snapshot: the SnapshotEngine consistency check; it needs the
         # shared page-size hook, compiled only into the crate's own test
@@ -312,11 +406,7 @@ jobs:
           EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
           # table_snapshot fails rather than skips without the string above.
           EXTENDDB_TEST_PG_REQUIRED: "1"
-        run: >-
-          cargo test --release -p extenddb-storage-postgres
-          --test vector_control_plane --test put_create_race --test twi_conflict
-          --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test backup_restore
-          --test gsi_online_build --test gsi_stale_writer --test table_snapshot
+        run: cargo test --release -p extenddb-storage-postgres --tests
 
       # The daemonized server logs to syslog; dump it so server-side failures
       # are diagnosable from the job log.
@@ -393,16 +483,24 @@ jobs:
           devtools/run-tests --extenddb --rust-integration --release
           --filter vector_index_unsupported
 
+      # The migration tests on a server without pgvector: the released vector
+      # states skip, naming pgvector, and every other case runs. The other
+      # storage-level tests build extension-free scratch databases and behave
+      # the same on either image, so the job above covers them.
+      - name: Run the PostgreSQL migration tests without pgvector
+        env:
+          EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+          EXTENDDB_TEST_PG_REQUIRED: "1"
+        run: >-
+          cargo test --release -p extenddb-storage-postgres
+          --test migrations_fresh --test migrations_released --test migrations_crash
+          --test migrations_golden --test migrations_guards --test migrations_runs
+
       # The daemonized server logs to syslog; dump it so server-side failures
       # are diagnosable from the job log.
       - name: Dump server syslog on failure
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
-
-      # No storage-level step here on purpose. Those tests build their own
-      # extension-free scratch databases, so they behave identically on either
-      # image and the job above already runs them; repeating them here would buy
-      # a second release build and no coverage.
   run-rust-integration-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40

.github/workflows/npm-publish.yml

@@ -164,6 +164,25 @@ jobs:
             echo "sha=$SHA"
           } >> "$GITHUB_OUTPUT"
 
+      - name: Check out the tagged commit for the migration gate
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          ref: ${{ steps.meta.outputs.sha }}
+          persist-credentials: false
+
+      - name: Gate - the release freezes the catalog migrations it ships
+        # The platform packages ship the sqlite,dev-mode build of the tag,
+        # whose dev server applies SQLite migrations to a catalog behind it.
+        # As in release-image.yml: RELEASED_THROUGH must reach the last
+        # migration id. A tag before the migration framework passes.
+        run: |
+          set -euo pipefail
+          if [[ -e crates/storage/src/migrations/registry.rs ]]; then
+            python3 devtools/check-released-through
+          else
+            echo "the tagged commit has no catalog migration registry"
+          fi
+
   build:
     needs: gate
     # The matrix lives in npm-build-matrix.yml, shared with the pull-request

.github/workflows/release-dev-image.yml

@@ -137,6 +137,17 @@ jobs:
           cargo install --locked --version 0.9.0 --features cli cargo-about
           ./devtools/generate-dev-license-notices --check
 
+      - name: Gate - the release freezes the catalog migrations it ships
+        # As in release-image.yml: RELEASED_THROUGH must reach the last
+        # migration id. A tag before the migration framework passes.
+        run: |
+          set -euo pipefail
+          if [[ -e crates/storage/src/migrations/registry.rs ]]; then
+            python3 devtools/check-released-through
+          else
+            echo "the tagged commit has no catalog migration registry"
+          fi
+
   build:
     needs: gate
     strategy:

.github/workflows/release-image.yml

@@ -210,6 +210,18 @@ jobs:
           ./devtools/generate-software-license-notices --check \
             --backend '${{ steps.meta.outputs.license_backend }}'
 
+      - name: Gate - the release freezes the catalog migrations it ships
+        # RELEASED_THROUGH must reach the last migration id, so each
+        # backend's checksum test pins every body this tag ships. A tag
+        # before the migration framework has no registry and passes.
+        run: |
+          set -euo pipefail
+          if [[ -e crates/storage/src/migrations/registry.rs ]]; then
+            python3 devtools/check-released-through
+          else
+            echo "the tagged commit has no catalog migration registry"
+          fi
+
   build:
     needs: gate
     # If one architecture fails, cancel the other: nothing is published either

.github/workflows/scripts-test.yml

@@ -1,15 +1,18 @@
 name: Scripts test
 # Unit tests for the JavaScript under .github/scripts (Node's built-in runner,
-# no dependencies to install).
+# no dependencies to install), and for the Python helpers under devtools that
+# the catalog check scripts report through (the standard library's unittest).
 on:
   pull_request:
     paths:
       - '.github/scripts/**'
+      - 'devtools/**'
       - '.github/workflows/scripts-test.yml'
   push:
     branches: [main]
     paths:
       - '.github/scripts/**'
+      - 'devtools/**'
       - '.github/workflows/scripts-test.yml'
 
 permissions:
@@ -33,3 +36,18 @@ jobs:
       - name: Run tests
         # The shell expands the glob, so this works on every Node version.
         run: node --test .github/scripts/*.test.js
+
+  devtools:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Check out devtools
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          sparse-checkout: devtools
+          persist-credentials: false
+
+      - name: Run tests
+        # A check that passed on its exit code alone, or on its output
+        # alone, would let the catalog check scripts pass a broken binary.
+        run: python3 -m unittest discover -s devtools -p '*_test.py' -v