diff --git a/docs/application-validation.md b/docs/application-validation.md index 46a9b9f4..7c3fd519 100644 --- a/docs/application-validation.md +++ b/docs/application-validation.md @@ -62,5 +62,15 @@ Omit `--wait` to enqueue the sweep and return immediately. The command reads `ADMIN_TOKEN` and, unless `--api-url` is supplied, `DISCORD_CLUSTER_MANAGER_API_BASE_URL` from the environment. +For a one-time backfill of every ranked user's current best submission, add +`--all-users`: + +```bash +kernelbot-admin validate-top10 cholesky B200 --all-users +``` + +This override applies only to the manual sweep. Scheduled sweeps remain capped +at the top 10 users. + Roll out KernelBot's migration and runner first, then the reference-kernels contract, then the Kernelboard badge. diff --git a/src/kernelbot/admin_cli.py b/src/kernelbot/admin_cli.py index 8a605d24..f5c3aa5b 100644 --- a/src/kernelbot/admin_cli.py +++ b/src/kernelbot/admin_cli.py @@ -20,10 +20,13 @@ def _validate_top10(args: argparse.Namespace) -> int: quote(args.leaderboard, safe=""), quote(args.gpu, safe=""), ) + params = {"wait": str(args.wait).lower()} + if args.all_users: + params["all_users"] = "true" response = requests.post( f"{api_url.rstrip('/')}{path}", headers={"Authorization": f"Bearer {token}"}, - params={"wait": str(args.wait).lower()}, + params=params, timeout=None if args.wait else 30, ) response.raise_for_status() @@ -49,6 +52,11 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="Wait for all validation jobs and print their results", ) + validate.add_argument( + "--all-users", + action="store_true", + help="Validate every ranked user's best submission instead of only the top 10", + ) validate.set_defaults(run=_validate_top10) return parser diff --git a/src/kernelbot/api/main.py b/src/kernelbot/api/main.py index e68d0451..a926d2df 100644 --- a/src/kernelbot/api/main.py +++ b/src/kernelbot/api/main.py @@ -491,6 +491,7 @@ async def admin_run_application_validation( gpu_type: str, _: Annotated[None, Depends(require_admin)], wait: bool = Query(False), + all_users: bool = Query(False), ) -> dict: if application_validation_service is None: raise HTTPException( @@ -502,15 +503,18 @@ async def admin_run_application_validation( leaderboard_name, gpu_type, scheduled_for=None, + all_users=all_users, ) application_validation_service.enqueue_manual_sweep( leaderboard_name, gpu_type, + all_users=all_users, ) return { "status": "accepted", "leaderboard": leaderboard_name, "gpu_type": gpu_type, + "all_users": all_users, } diff --git a/src/libkernelbot/application_validation.py b/src/libkernelbot/application_validation.py index 203a23df..b876e169 100644 --- a/src/libkernelbot/application_validation.py +++ b/src/libkernelbot/application_validation.py @@ -101,9 +101,16 @@ def enqueue_manual_sweep( self, leaderboard_name: str, gpu_type: str, + *, + all_users: bool = False, ) -> None: task = asyncio.create_task( - self.run_sweep(leaderboard_name, gpu_type, scheduled_for=None), + self.run_sweep( + leaderboard_name, + gpu_type, + scheduled_for=None, + all_users=all_users, + ), name=f"manual-application-validation-{leaderboard_name}-{gpu_type}", ) self._manual_tasks.add(task) @@ -225,6 +232,7 @@ async def run_sweep( gpu_type: str, *, scheduled_for: datetime.date | None, + all_users: bool = False, ) -> dict[str, Any]: with self.backend.db as db: leaderboard = db.get_leaderboard(leaderboard_name) @@ -255,7 +263,7 @@ async def run_sweep( submissions = db.get_leaderboard_submissions( leaderboard_name, gpu_type, - limit=TOP_K, + limit=None if all_users else TOP_K, ) logger.info( @@ -288,6 +296,7 @@ async def run_sweep( "gpu_type": gpu_type, "scheduled_for": scheduled_for, "contract_version": validation.version, + "all_users": all_users, "results": results, } except Exception as exc: diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index 5b2feb47..f51a4927 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -127,6 +127,7 @@ def test_manual_validation_can_wait_for_debug_result( "cholesky", "B200", scheduled_for=None, + all_users=False, ) def test_manual_validation_is_async_by_default( @@ -144,10 +145,35 @@ def test_manual_validation_is_async_by_default( "status": "accepted", "leaderboard": "cholesky", "gpu_type": "B200", + "all_users": False, } mock_validation_service.enqueue_manual_sweep.assert_called_once_with( "cholesky", "B200", + all_users=False, + ) + + def test_manual_validation_can_enqueue_all_users( + self, + test_client, + mock_validation_service, + ): + response = test_client.post( + "/admin/application-validations/cholesky/B200?all_users=true", + headers={"Authorization": "Bearer test_token"}, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": "accepted", + "leaderboard": "cholesky", + "gpu_type": "B200", + "all_users": True, + } + mock_validation_service.enqueue_manual_sweep.assert_called_once_with( + "cholesky", + "B200", + all_users=True, ) class TestRunnerQueue: diff --git a/tests/test_admin_cli.py b/tests/test_admin_cli.py index ce784d6b..9f034131 100644 --- a/tests/test_admin_cli.py +++ b/tests/test_admin_cli.py @@ -65,6 +65,38 @@ def test_validate_top10_can_wait_and_url_encodes_names(): ) +def test_validate_top10_can_override_scope_to_all_users(): + response = Mock() + response.json.return_value = {"status": "accepted", "all_users": True} + + with ( + patch.dict( + "os.environ", + { + "DISCORD_CLUSTER_MANAGER_API_BASE_URL": "https://kernelbot.test", + "ADMIN_TOKEN": "secret", + }, + ), + patch("kernelbot.admin_cli.requests.post", return_value=response) as post, + patch( + "sys.argv", + [ + "kernelbot-admin", + "validate-top10", + "cholesky", + "B200", + "--all-users", + ], + ), + ): + assert admin_cli.main() == 0 + + assert post.call_args.kwargs["params"] == { + "wait": "false", + "all_users": "true", + } + + def test_validate_top10_requires_admin_token(): with ( patch.dict( diff --git a/tests/test_application_validation.py b/tests/test_application_validation.py index 1002be7b..79108d11 100644 --- a/tests/test_application_validation.py +++ b/tests/test_application_validation.py @@ -44,6 +44,7 @@ def __init__(self): self.claims = set() self.saved = [] self.sweeps = [] + self.requested_limits = [] def __enter__(self): return self @@ -73,6 +74,7 @@ def claim_validation_sweep( return len(self.claims) def get_leaderboard_submissions(self, _name, _gpu, limit): + self.requested_limits.append(limit) return self.submissions[:limit] def get_submission_code_for_validation(self, submission_id): @@ -143,6 +145,29 @@ async def test_sweep_validates_top_ten_with_bounded_concurrency(): assert database.sweeps == [(1, "completed", None)] assert database.saved[0]["fully_validated"] is True assert database.saved[0]["result"]["results"][0]["unexpected_private_field"] == "drop me" + assert database.requested_limits == [10] + + +@pytest.mark.asyncio +async def test_manual_sweep_can_validate_every_ranked_user(): + database = FakeDB() + launcher = FakeLauncher() + backend = SimpleNamespace(db=database, launcher_map={"B200": launcher}) + service = ApplicationValidationService(backend) + + summary = await service.run_sweep( + "cholesky", + "B200", + scheduled_for=None, + all_users=True, + ) + + assert summary["status"] == "completed" + assert summary["all_users"] is True + assert len(summary["results"]) == 12 + assert len(database.saved) == 12 + assert database.requested_limits == [None] + assert launcher.max_active == 2 @pytest.mark.asyncio