From e1446407f784203a4c805b4be2319157b0f3a739 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sun, 26 Jul 2026 11:32:44 -0700 Subject: [PATCH] Add missing-only validation continuation --- docs/application-validation.md | 6 ++- src/kernelbot/admin_cli.py | 7 ++++ src/kernelbot/api/main.py | 4 ++ src/libkernelbot/application_validation.py | 15 ++++++++ src/libkernelbot/leaderboard_db.py | 22 +++++++++++ tests/test_admin_api.py | 8 +++- tests/test_admin_cli.py | 2 + tests/test_application_validation.py | 44 ++++++++++++++++++++++ tests/test_leaderboard_db.py | 15 ++++++++ 9 files changed, 120 insertions(+), 3 deletions(-) diff --git a/docs/application-validation.md b/docs/application-validation.md index 7c3fd519..d58a4aa6 100644 --- a/docs/application-validation.md +++ b/docs/application-validation.md @@ -66,11 +66,13 @@ For a one-time backfill of every ranked user's current best submission, add `--all-users`: ```bash -kernelbot-admin validate-top10 cholesky B200 --all-users +kernelbot-admin validate-top10 cholesky B200 --all-users --only-missing ``` This override applies only to the manual sweep. Scheduled sweeps remain capped -at the top 10 users. +at the top 10 users. `--only-missing` skips exact submissions that already have +a result for the active validation contract, making it suitable for continuing +a backfill after leaderboard changes. Roll out KernelBot's migration and runner first, then the reference-kernels contract, then the Kernelboard badge. diff --git a/src/kernelbot/admin_cli.py b/src/kernelbot/admin_cli.py index f5c3aa5b..150b966b 100644 --- a/src/kernelbot/admin_cli.py +++ b/src/kernelbot/admin_cli.py @@ -23,6 +23,8 @@ def _validate_top10(args: argparse.Namespace) -> int: params = {"wait": str(args.wait).lower()} if args.all_users: params["all_users"] = "true" + if args.only_missing: + params["only_missing"] = "true" response = requests.post( f"{api_url.rstrip('/')}{path}", headers={"Authorization": f"Bearer {token}"}, @@ -57,6 +59,11 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="Validate every ranked user's best submission instead of only the top 10", ) + validate.add_argument( + "--only-missing", + action="store_true", + help="Skip exact submissions that already have a result for this contract", + ) validate.set_defaults(run=_validate_top10) return parser diff --git a/src/kernelbot/api/main.py b/src/kernelbot/api/main.py index a926d2df..6526ec9f 100644 --- a/src/kernelbot/api/main.py +++ b/src/kernelbot/api/main.py @@ -492,6 +492,7 @@ async def admin_run_application_validation( _: Annotated[None, Depends(require_admin)], wait: bool = Query(False), all_users: bool = Query(False), + only_missing: bool = Query(False), ) -> dict: if application_validation_service is None: raise HTTPException( @@ -504,17 +505,20 @@ async def admin_run_application_validation( gpu_type, scheduled_for=None, all_users=all_users, + only_missing=only_missing, ) application_validation_service.enqueue_manual_sweep( leaderboard_name, gpu_type, all_users=all_users, + only_missing=only_missing, ) return { "status": "accepted", "leaderboard": leaderboard_name, "gpu_type": gpu_type, "all_users": all_users, + "only_missing": only_missing, } diff --git a/src/libkernelbot/application_validation.py b/src/libkernelbot/application_validation.py index b876e169..7aee76f3 100644 --- a/src/libkernelbot/application_validation.py +++ b/src/libkernelbot/application_validation.py @@ -103,6 +103,7 @@ def enqueue_manual_sweep( gpu_type: str, *, all_users: bool = False, + only_missing: bool = False, ) -> None: task = asyncio.create_task( self.run_sweep( @@ -110,6 +111,7 @@ def enqueue_manual_sweep( gpu_type, scheduled_for=None, all_users=all_users, + only_missing=only_missing, ), name=f"manual-application-validation-{leaderboard_name}-{gpu_type}", ) @@ -233,6 +235,7 @@ async def run_sweep( *, scheduled_for: datetime.date | None, all_users: bool = False, + only_missing: bool = False, ) -> dict[str, Any]: with self.backend.db as db: leaderboard = db.get_leaderboard(leaderboard_name) @@ -265,6 +268,17 @@ async def run_sweep( gpu_type, limit=None if all_users else TOP_K, ) + if only_missing: + validated_ids = db.get_submission_validation_ids( + [entry["submission_id"] for entry in submissions], + gpu_type=gpu_type, + contract_version=validation.version, + ) + submissions = [ + entry + for entry in submissions + if entry["submission_id"] not in validated_ids + ] logger.info( "Running application validation sweep %s for leaderboard=%s gpu=%s submissions=%s", @@ -297,6 +311,7 @@ async def run_sweep( "scheduled_for": scheduled_for, "contract_version": validation.version, "all_users": all_users, + "only_missing": only_missing, "results": results, } except Exception as exc: diff --git a/src/libkernelbot/leaderboard_db.py b/src/libkernelbot/leaderboard_db.py index 06357e7b..9c0d0de8 100644 --- a/src/libkernelbot/leaderboard_db.py +++ b/src/libkernelbot/leaderboard_db.py @@ -958,6 +958,28 @@ def get_submission_code_for_validation(self, submission_id: int) -> str: raise KernelBotError(f"Submission {submission_id} does not exist", code=404) return bytes(row[0]).decode("utf-8") + def get_submission_validation_ids( + self, + submission_ids: list[int], + *, + gpu_type: str, + contract_version: str, + ) -> set[int]: + """Return submissions that already have a result for this exact contract.""" + if not submission_ids: + return set() + self.cursor.execute( + """ + SELECT submission_id + FROM leaderboard.submission_validation + WHERE submission_id = ANY(%s) + AND gpu_type = %s + AND contract_version = %s + """, + (submission_ids, gpu_type, contract_version), + ) + return {row[0] for row in self.cursor.fetchall()} + def upsert_submission_validation( self, *, diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index f51a4927..2b0a6352 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -128,6 +128,7 @@ def test_manual_validation_can_wait_for_debug_result( "B200", scheduled_for=None, all_users=False, + only_missing=False, ) def test_manual_validation_is_async_by_default( @@ -146,11 +147,13 @@ def test_manual_validation_is_async_by_default( "leaderboard": "cholesky", "gpu_type": "B200", "all_users": False, + "only_missing": False, } mock_validation_service.enqueue_manual_sweep.assert_called_once_with( "cholesky", "B200", all_users=False, + only_missing=False, ) def test_manual_validation_can_enqueue_all_users( @@ -159,7 +162,8 @@ def test_manual_validation_can_enqueue_all_users( mock_validation_service, ): response = test_client.post( - "/admin/application-validations/cholesky/B200?all_users=true", + "/admin/application-validations/cholesky/B200" + "?all_users=true&only_missing=true", headers={"Authorization": "Bearer test_token"}, ) @@ -169,11 +173,13 @@ def test_manual_validation_can_enqueue_all_users( "leaderboard": "cholesky", "gpu_type": "B200", "all_users": True, + "only_missing": True, } mock_validation_service.enqueue_manual_sweep.assert_called_once_with( "cholesky", "B200", all_users=True, + only_missing=True, ) class TestRunnerQueue: diff --git a/tests/test_admin_cli.py b/tests/test_admin_cli.py index 9f034131..3c7b5fab 100644 --- a/tests/test_admin_cli.py +++ b/tests/test_admin_cli.py @@ -86,6 +86,7 @@ def test_validate_top10_can_override_scope_to_all_users(): "cholesky", "B200", "--all-users", + "--only-missing", ], ), ): @@ -94,6 +95,7 @@ def test_validate_top10_can_override_scope_to_all_users(): assert post.call_args.kwargs["params"] == { "wait": "false", "all_users": "true", + "only_missing": "true", } diff --git a/tests/test_application_validation.py b/tests/test_application_validation.py index 79108d11..0aed4fc7 100644 --- a/tests/test_application_validation.py +++ b/tests/test_application_validation.py @@ -45,6 +45,7 @@ def __init__(self): self.saved = [] self.sweeps = [] self.requested_limits = [] + self.validated_ids = set() def __enter__(self): return self @@ -80,6 +81,17 @@ def get_leaderboard_submissions(self, _name, _gpu, limit): def get_submission_code_for_validation(self, submission_id): return f"# submission {submission_id}" + def get_submission_validation_ids( + self, + submission_ids, + *, + gpu_type, + contract_version, + ): + assert gpu_type == "B200" + assert contract_version == "v1" + return set(submission_ids) & self.validated_ids + def upsert_submission_validation(self, **values): self.saved.append(values) @@ -170,6 +182,38 @@ async def test_manual_sweep_can_validate_every_ranked_user(): assert launcher.max_active == 2 +@pytest.mark.asyncio +async def test_manual_sweep_can_validate_only_missing_users(): + database = FakeDB() + database.validated_ids = {1, 2, 4, 8} + launcher = FakeLauncher() + backend = SimpleNamespace(db=database, launcher_map={"B200": launcher}) + service = ApplicationValidationService(backend) + + summary = await service.run_sweep( + "cholesky", + "B200", + scheduled_for=None, + all_users=True, + only_missing=True, + ) + + assert summary["status"] == "completed" + assert summary["only_missing"] is True + assert [result["submission_id"] for result in summary["results"]] == [ + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 12, + ] + assert database.requested_limits == [None] + assert launcher.max_active == 2 + + @pytest.mark.asyncio async def test_nightly_sweep_is_claimed_once(): database = FakeDB() diff --git a/tests/test_leaderboard_db.py b/tests/test_leaderboard_db.py index 063f66ed..2e786813 100644 --- a/tests/test_leaderboard_db.py +++ b/tests/test_leaderboard_db.py @@ -126,6 +126,11 @@ def test_application_validation_persistence(database, submit_leaderboard): user_name="validator", ) assert db.get_submission_code_for_validation(submission_id) == source + assert db.get_submission_validation_ids( + [submission_id], + gpu_type="B200", + contract_version="v1", + ) == set() scheduled_for = datetime.date(2026, 7, 26) sweep_id = db.claim_validation_sweep( @@ -153,6 +158,16 @@ def test_application_validation_persistence(database, submit_leaderboard): geomean_sync_wall_speedup=1.2, result={"passed_shapes": 7, "total_shapes": 8}, ) + assert db.get_submission_validation_ids( + [submission_id], + gpu_type="B200", + contract_version="v1", + ) == {submission_id} + assert db.get_submission_validation_ids( + [submission_id], + gpu_type="B200", + contract_version="v2", + ) == set() db.cursor.execute( """ SELECT passed_shapes, total_shapes, fully_validated,