From c432ad19742c5e8961ace04a70f1d72767b60df2 Mon Sep 17 00:00:00 2001 From: Manuel Parra Date: Fri, 18 Sep 2026 17:32:37 -0400 Subject: [PATCH 1/2] ci: run SQLite regression suite on pushes and pull requests --- .github/workflows/regressions.yml | 30 ++++++++++++++++++++++++++++++ README.md | 23 +++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/regressions.yml diff --git a/.github/workflows/regressions.yml b/.github/workflows/regressions.yml new file mode 100644 index 0000000..0ccc306 --- /dev/null +++ b/.github/workflows/regressions.yml @@ -0,0 +1,30 @@ +name: SQLite regressions + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + sqlite-regressions: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@eb1897b8dc4b5d5bfe39a428a8f2304605e0983c # v7.0.0 + with: + version: "0.11.19" + checksum: "7035608168e106375b36d0c818d537a889c51a8625fe7f8f7cad5e62b947c368" + enable-cache: false + - name: Install pinned Python + run: uv python install 3.13.7 + - name: Check whitespace + run: git show --format= --check HEAD + - name: Check Python syntax + run: uv run --python 3.13.7 --no-project --no-sync --offline python -m compileall -q scripts tests + - name: Run SQLite regression suite + run: uv run --python 3.13.7 --no-project --no-sync --offline python -m unittest discover -s tests -v diff --git a/README.md b/README.md index f05bd27..2ad5dbd 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,29 @@ Only `raw` and non-gold tool-assisted runs should be considered for fair ranking The July 3 result is a pilot evaluation, not a leaderboard claim. +## Regression tests and CI + +The `SQLite regressions` workflow runs on every push and pull request with +Python 3.13.7 and uv 0.11.19. Actions are pinned to commit SHAs, and the Linux +uv archive is verified against its release SHA-256. After runtime setup, checks +run offline with no third-party Python packages or benchmark downloads. + +Run the same suite from the repository root with the pinned runtime installed: + +```sh +uv run --python 3.13.7 --no-project --no-sync --offline python -m compileall -q scripts tests +uv run --python 3.13.7 --no-project --no-sync --offline python -m unittest discover -s tests -v +``` + +The 12 tests execute the real SQLite evaluator and comparator against temporary +fixture databases. They cover denied mutations and unchanged database contents, +safe literals/comments/identifiers, instruction limits and connection cleanup, +ordered versus unordered results, duplicate counts, types, NULLs, and the +versioned evaluation/report contract. A failed test exits nonzero and fails CI. +These are local fixture regressions, not measured model accuracy or acceptance +of downloaded benchmarks. CI also checks Python syntax and commit whitespace; +the repository currently has no configured formatter, linter, or type checker. + ## Documentation - `docs/HARNESS.md` explains the run and evaluation flow. From 34eee79e3ca797da0aeafc4e60212abc7f887ce8 Mon Sep 17 00:00:00 2001 From: Manuel Parra Date: Fri, 18 Sep 2026 17:34:12 -0400 Subject: [PATCH 2/2] ci: scope whitespace validation to the checked commit --- .github/workflows/regressions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regressions.yml b/.github/workflows/regressions.yml index 0ccc306..ab9e4e6 100644 --- a/.github/workflows/regressions.yml +++ b/.github/workflows/regressions.yml @@ -14,6 +14,7 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: + fetch-depth: 2 persist-credentials: false - uses: astral-sh/setup-uv@eb1897b8dc4b5d5bfe39a428a8f2304605e0983c # v7.0.0 with: @@ -23,7 +24,7 @@ jobs: - name: Install pinned Python run: uv python install 3.13.7 - name: Check whitespace - run: git show --format= --check HEAD + run: git diff --check HEAD^ HEAD - name: Check Python syntax run: uv run --python 3.13.7 --no-project --no-sync --offline python -m compileall -q scripts tests - name: Run SQLite regression suite