Add missing-only validation continuation

docs/application-validation.md

@@ -66,11 +66,13 @@ For a one-time backfill of every ranked user's current best submission, add
 `--all-users`:
 
 ```bash
-kernelbot-admin validate-top10 cholesky B200 --all-users
+kernelbot-admin validate-top10 cholesky B200 --all-users --only-missing
 ```
 
 This override applies only to the manual sweep. Scheduled sweeps remain capped
-at the top 10 users.
+at the top 10 users. `--only-missing` skips exact submissions that already have
+a result for the active validation contract, making it suitable for continuing
+a backfill after leaderboard changes.
 
 Roll out KernelBot's migration and runner first, then the reference-kernels
 contract, then the Kernelboard badge.

src/kernelbot/admin_cli.py

@@ -23,6 +23,8 @@ def _validate_top10(args: argparse.Namespace) -> int:
     params = {"wait": str(args.wait).lower()}
     if args.all_users:
         params["all_users"] = "true"
+    if args.only_missing:
+        params["only_missing"] = "true"
     response = requests.post(
         f"{api_url.rstrip('/')}{path}",
         headers={"Authorization": f"Bearer {token}"},
@@ -57,6 +59,11 @@ def _parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Validate every ranked user's best submission instead of only the top 10",
     )
+    validate.add_argument(
+        "--only-missing",
+        action="store_true",
+        help="Skip exact submissions that already have a result for this contract",
+    )
     validate.set_defaults(run=_validate_top10)
     return parser
 

src/kernelbot/api/main.py

@@ -492,6 +492,7 @@ async def admin_run_application_validation(
     _: Annotated[None, Depends(require_admin)],
     wait: bool = Query(False),
     all_users: bool = Query(False),
+    only_missing: bool = Query(False),
 ) -> dict:
     if application_validation_service is None:
         raise HTTPException(
@@ -504,17 +505,20 @@ async def admin_run_application_validation(
             gpu_type,
             scheduled_for=None,
             all_users=all_users,
+            only_missing=only_missing,
         )
     application_validation_service.enqueue_manual_sweep(
         leaderboard_name,
         gpu_type,
         all_users=all_users,
+        only_missing=only_missing,
     )
     return {
         "status": "accepted",
         "leaderboard": leaderboard_name,
         "gpu_type": gpu_type,
         "all_users": all_users,
+        "only_missing": only_missing,
     }
 
 

src/libkernelbot/application_validation.py

@@ -103,13 +103,15 @@ def enqueue_manual_sweep(
         gpu_type: str,
         *,
         all_users: bool = False,
+        only_missing: bool = False,
     ) -> None:
         task = asyncio.create_task(
             self.run_sweep(
                 leaderboard_name,
                 gpu_type,
                 scheduled_for=None,
                 all_users=all_users,
+                only_missing=only_missing,
             ),
             name=f"manual-application-validation-{leaderboard_name}-{gpu_type}",
         )
@@ -233,6 +235,7 @@ async def run_sweep(
         *,
         scheduled_for: datetime.date | None,
         all_users: bool = False,
+        only_missing: bool = False,
     ) -> dict[str, Any]:
         with self.backend.db as db:
             leaderboard = db.get_leaderboard(leaderboard_name)
@@ -265,6 +268,17 @@ async def run_sweep(
                 gpu_type,
                 limit=None if all_users else TOP_K,
             )
+            if only_missing:
+                validated_ids = db.get_submission_validation_ids(
+                    [entry["submission_id"] for entry in submissions],
+                    gpu_type=gpu_type,
+                    contract_version=validation.version,
+                )
+                submissions = [
+                    entry
+                    for entry in submissions
+                    if entry["submission_id"] not in validated_ids
+                ]
 
         logger.info(
             "Running application validation sweep %s for leaderboard=%s gpu=%s submissions=%s",
@@ -297,6 +311,7 @@ async def run_sweep(
                 "scheduled_for": scheduled_for,
                 "contract_version": validation.version,
                 "all_users": all_users,
+                "only_missing": only_missing,
                 "results": results,
             }
         except Exception as exc:

src/libkernelbot/leaderboard_db.py

@@ -958,6 +958,28 @@ def get_submission_code_for_validation(self, submission_id: int) -> str:
             raise KernelBotError(f"Submission {submission_id} does not exist", code=404)
         return bytes(row[0]).decode("utf-8")
 
+    def get_submission_validation_ids(
+        self,
+        submission_ids: list[int],
+        *,
+        gpu_type: str,
+        contract_version: str,
+    ) -> set[int]:
+        """Return submissions that already have a result for this exact contract."""
+        if not submission_ids:
+            return set()
+        self.cursor.execute(
+            """
+            SELECT submission_id
+            FROM leaderboard.submission_validation
+            WHERE submission_id = ANY(%s)
+              AND gpu_type = %s
+              AND contract_version = %s
+            """,
+            (submission_ids, gpu_type, contract_version),
+        )
+        return {row[0] for row in self.cursor.fetchall()}
+
     def upsert_submission_validation(
         self,
         *,

tests/test_admin_api.py

@@ -128,6 +128,7 @@ def test_manual_validation_can_wait_for_debug_result(
             "B200",
             scheduled_for=None,
             all_users=False,
+            only_missing=False,
         )
 
     def test_manual_validation_is_async_by_default(
@@ -146,11 +147,13 @@ def test_manual_validation_is_async_by_default(
             "leaderboard": "cholesky",
             "gpu_type": "B200",
             "all_users": False,
+            "only_missing": False,
         }
         mock_validation_service.enqueue_manual_sweep.assert_called_once_with(
             "cholesky",
             "B200",
             all_users=False,
+            only_missing=False,
         )
 
     def test_manual_validation_can_enqueue_all_users(
@@ -159,7 +162,8 @@ def test_manual_validation_can_enqueue_all_users(
         mock_validation_service,
     ):
         response = test_client.post(
-            "/admin/application-validations/cholesky/B200?all_users=true",
+            "/admin/application-validations/cholesky/B200"
+            "?all_users=true&only_missing=true",
             headers={"Authorization": "Bearer test_token"},
         )
 
@@ -169,11 +173,13 @@ def test_manual_validation_can_enqueue_all_users(
             "leaderboard": "cholesky",
             "gpu_type": "B200",
             "all_users": True,
+            "only_missing": True,
         }
         mock_validation_service.enqueue_manual_sweep.assert_called_once_with(
             "cholesky",
             "B200",
             all_users=True,
+            only_missing=True,
         )
 
 class TestRunnerQueue:

tests/test_admin_cli.py

@@ -86,6 +86,7 @@ def test_validate_top10_can_override_scope_to_all_users():
                 "cholesky",
                 "B200",
                 "--all-users",
+                "--only-missing",
             ],
         ),
     ):
@@ -94,6 +95,7 @@ def test_validate_top10_can_override_scope_to_all_users():
     assert post.call_args.kwargs["params"] == {
         "wait": "false",
         "all_users": "true",
+        "only_missing": "true",
     }
 
 

tests/test_application_validation.py

@@ -45,6 +45,7 @@ def __init__(self):
         self.saved = []
         self.sweeps = []
         self.requested_limits = []
+        self.validated_ids = set()
 
     def __enter__(self):
         return self
@@ -80,6 +81,17 @@ def get_leaderboard_submissions(self, _name, _gpu, limit):
     def get_submission_code_for_validation(self, submission_id):
         return f"# submission {submission_id}"
 
+    def get_submission_validation_ids(
+        self,
+        submission_ids,
+        *,
+        gpu_type,
+        contract_version,
+    ):
+        assert gpu_type == "B200"
+        assert contract_version == "v1"
+        return set(submission_ids) & self.validated_ids
+
     def upsert_submission_validation(self, **values):
         self.saved.append(values)
 
@@ -170,6 +182,38 @@ async def test_manual_sweep_can_validate_every_ranked_user():
     assert launcher.max_active == 2
 
 
+@pytest.mark.asyncio
+async def test_manual_sweep_can_validate_only_missing_users():
+    database = FakeDB()
+    database.validated_ids = {1, 2, 4, 8}
+    launcher = FakeLauncher()
+    backend = SimpleNamespace(db=database, launcher_map={"B200": launcher})
+    service = ApplicationValidationService(backend)
+
+    summary = await service.run_sweep(
+        "cholesky",
+        "B200",
+        scheduled_for=None,
+        all_users=True,
+        only_missing=True,
+    )
+
+    assert summary["status"] == "completed"
+    assert summary["only_missing"] is True
+    assert [result["submission_id"] for result in summary["results"]] == [
+        3,
+        5,
+        6,
+        7,
+        9,
+        10,
+        11,
+        12,
+    ]
+    assert database.requested_limits == [None]
+    assert launcher.max_active == 2
+
+
 @pytest.mark.asyncio
 async def test_nightly_sweep_is_claimed_once():
     database = FakeDB()

tests/test_leaderboard_db.py

@@ -126,6 +126,11 @@ def test_application_validation_persistence(database, submit_leaderboard):
             user_name="validator",
         )
         assert db.get_submission_code_for_validation(submission_id) == source
+        assert db.get_submission_validation_ids(
+            [submission_id],
+            gpu_type="B200",
+            contract_version="v1",
+        ) == set()
 
         scheduled_for = datetime.date(2026, 7, 26)
         sweep_id = db.claim_validation_sweep(
@@ -153,6 +158,16 @@ def test_application_validation_persistence(database, submit_leaderboard):
             geomean_sync_wall_speedup=1.2,
             result={"passed_shapes": 7, "total_shapes": 8},
         )
+        assert db.get_submission_validation_ids(
+            [submission_id],
+            gpu_type="B200",
+            contract_version="v1",
+        ) == {submission_id}
+        assert db.get_submission_validation_ids(
+            [submission_id],
+            gpu_type="B200",
+            contract_version="v2",
+        ) == set()
         db.cursor.execute(
             """
             SELECT passed_shapes, total_shapes, fully_validated,