diff --git a/README.md b/README.md index 2ad5dbd..e35577d 100644 --- a/README.md +++ b/README.md @@ -94,11 +94,13 @@ uv run --python 3.13.7 --no-project --no-sync --offline python -m compileall -q uv run --python 3.13.7 --no-project --no-sync --offline python -m unittest discover -s tests -v ``` -The 12 tests execute the real SQLite evaluator and comparator against temporary -fixture databases. They cover denied mutations and unchanged database contents, +The regression tests execute the real SQLite evaluator and comparator against +temporary fixture databases. They cover denied mutations and unchanged database contents, safe literals/comments/identifiers, instruction limits and connection cleanup, ordered versus unordered results, duplicate counts, types, NULLs, and the -versioned evaluation/report contract. A failed test exits nonzero and fails CI. +versioned evaluation/report contract. Fairness regressions cover short gold SQL +leaks such as `SELECT 1`, clean prompts, and score eligibility in saved audits +and reports. A failed test exits nonzero and fails CI. These are local fixture regressions, not measured model accuracy or acceptance of downloaded benchmarks. CI also checks Python syntax and commit whitespace; the repository currently has no configured formatter, linter, or type checker. diff --git a/scripts/eval_sql_benchmark.py b/scripts/eval_sql_benchmark.py index 234a6b9..344d1f4 100644 --- a/scripts/eval_sql_benchmark.py +++ b/scripts/eval_sql_benchmark.py @@ -146,10 +146,11 @@ def audit_run(run_dir: Path, cases: list[dict[str, Any]], run_metadata: dict[str if not case: continue gold_sql = normalize_sql(case.get("gold_sql", "")) - if len(gold_sql) < 40: + if not gold_sql: continue prompt_norm = normalize_sql(prompt_path.read_text(encoding="utf-8", errors="ignore")) - if gold_sql and gold_sql in prompt_norm: + # Check short queries too, without matching SELECT 1 inside SELECT 10 or SELECT 1.0. + if re.search(rf"(?