diff --git a/README.md b/README.md index aff59cb1..41881509 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ yoyo apply src/migrations -d postgresql://user:password@localhost/clusterdev See [docs/database.md](docs/database.md) for migration patterns and creating new migrations. For production incident triage, see [docs/production-debugging.md](docs/production-debugging.md). -For nightly end-to-end kernel checks, see +For admin-triggered end-to-end kernel checks, see [docs/application-validation.md](docs/application-validation.md). ### Environment Variables @@ -54,7 +54,6 @@ PROBLEM_DEV_DIR=examples DISABLE_SSL=1 # Set for local development GITHUB_TOKEN_BACKUP= # Fallback token for rate limiting ADMIN_TOKEN= # Token for admin API endpoints -APPLICATION_VALIDATION_ENABLED=true # Discord bot (only needed if testing Discord integration) # See docs/discord.md for setup instructions diff --git a/docs/application-validation.md b/docs/application-validation.md index d58a4aa6..b0a93ee6 100644 --- a/docs/application-validation.md +++ b/docs/application-validation.md @@ -26,16 +26,13 @@ JSON result: } ``` -## Schedule +## Manual batches -Every day at 22:00 `America/Los_Angeles`, one KernelBot replica claims each -`(leaderboard, GPU, contract version, local date)` sweep. It snapshots the -current best submission from each of the top 10 users and runs at most two -Modal jobs concurrently. The database claim prevents duplicate sweeps. - -Set `APPLICATION_VALIDATION_ENABLED=false` to disable the scheduler. A failed -job is recorded as `VALIDATION ERROR`; a completed job is stored as `X/Y -VALIDATED`. +Application validation never runs automatically. An admin explicitly starts a +batch for one leaderboard and GPU. By default it snapshots the current best +submission from each of the top 10 users and runs at most two Modal jobs +concurrently. A failed job is recorded as `VALIDATION ERROR`; a completed job +is stored as `X/Y VALIDATED`. ## Local debug @@ -47,7 +44,6 @@ modal environment create cholesky-validation-debug modal deploy --env cholesky-validation-debug src/runners/modal_runner_archs.py MODAL_ENVIRONMENT=cholesky-validation-debug \ -APPLICATION_VALIDATION_ENABLED=false \ python src/kernelbot/main.py --api-only --debug ``` @@ -69,10 +65,9 @@ For a one-time backfill of every ranked user's current best submission, add kernelbot-admin validate-top10 cholesky B200 --all-users --only-missing ``` -This override applies only to the manual sweep. Scheduled sweeps remain capped -at the top 10 users. `--only-missing` skips exact submissions that already have -a result for the active validation contract, making it suitable for continuing -a backfill after leaderboard changes. +`--only-missing` skips exact submissions that already have a result for the +active validation contract, making it suitable for continuing a backfill after +leaderboard changes. Roll out KernelBot's migration and runner first, then the reference-kernels contract, then the Kernelboard badge. diff --git a/src/kernelbot/api/main.py b/src/kernelbot/api/main.py index 6526ec9f..f646bda2 100644 --- a/src/kernelbot/api/main.py +++ b/src/kernelbot/api/main.py @@ -503,7 +503,6 @@ async def admin_run_application_validation( return await application_validation_service.run_sweep( leaderboard_name, gpu_type, - scheduled_for=None, all_users=all_users, only_missing=only_missing, ) diff --git a/src/kernelbot/env.py b/src/kernelbot/env.py index 1d945660..0c3d11f1 100644 --- a/src/kernelbot/env.py +++ b/src/kernelbot/env.py @@ -42,10 +42,6 @@ # PostgreSQL-specific constants env.DATABASE_URL = os.getenv("DATABASE_URL") env.DISABLE_SSL = os.getenv("DISABLE_SSL") -env.APPLICATION_VALIDATION_ENABLED = os.getenv( - "APPLICATION_VALIDATION_ENABLED", - "true", -).lower() in {"1", "true", "yes"} def init_environment(skip_discord: bool = False): diff --git a/src/kernelbot/main.py b/src/kernelbot/main.py index 64de6b26..5367916a 100644 --- a/src/kernelbot/main.py +++ b/src/kernelbot/main.py @@ -57,12 +57,8 @@ async def run_api_server(backend: KernelBackend): manager = init_background_submission_manager(BackgroundSubmissionManager(backend)) await manager.start() validation_service = init_application_validation_service( - ApplicationValidationService( - backend, - enabled=env.APPLICATION_VALIDATION_ENABLED, - ) + ApplicationValidationService(backend) ) - await validation_service.start() server = create_uvicorn_server() try: @@ -273,12 +269,8 @@ async def start_bot_and_api(debug_mode: bool): manager = init_background_submission_manager(BackgroundSubmissionManager(bot_instance.backend)) await manager.start() validation_service = init_application_validation_service( - ApplicationValidationService( - bot_instance.backend, - enabled=env.APPLICATION_VALIDATION_ENABLED, - ) + ApplicationValidationService(bot_instance.backend) ) - await validation_service.start() server = create_uvicorn_server() try: diff --git a/src/libkernelbot/application_validation.py b/src/libkernelbot/application_validation.py index 7aee76f3..2999924b 100644 --- a/src/libkernelbot/application_validation.py +++ b/src/libkernelbot/application_validation.py @@ -1,13 +1,11 @@ -"""Nightly application-level validation for ranked kernel submissions.""" +"""Admin-triggered application-level validation for ranked kernel submissions.""" from __future__ import annotations import asyncio import contextlib -import datetime import math from typing import Any -from zoneinfo import ZoneInfo from libkernelbot.consts import get_gpu_by_name from libkernelbot.task import LeaderboardTask, build_validation_config @@ -15,44 +13,17 @@ logger = setup_logging(__name__) -SCHEDULE = datetime.time(22, 0) -TIMEZONE = ZoneInfo("America/Los_Angeles") TOP_K = 10 MAX_CONCURRENCY = 2 -def _due_date(now: datetime.datetime) -> datetime.date | None: - if now.tzinfo is None: - now = now.replace(tzinfo=datetime.timezone.utc) - local_now = now.astimezone(TIMEZONE) - if local_now.time() < SCHEDULE: - return None - return local_now.date() - - class ApplicationValidationService: - def __init__(self, backend, *, enabled: bool = False, poll_seconds: int = 60): + def __init__(self, backend): self.backend = backend - self.enabled = enabled - self.poll_seconds = poll_seconds - self._scheduler_task: asyncio.Task | None = None self._manual_tasks: set[asyncio.Task] = set() - async def start(self) -> None: - if not self.enabled or self._scheduler_task is not None: - return - logger.info("Starting application validation scheduler") - self._scheduler_task = asyncio.create_task( - self._scheduler_loop(), - name="application-validation-scheduler", - ) - async def stop(self) -> None: tasks = list(self._manual_tasks) - if self._scheduler_task is not None: - self._scheduler_task.cancel() - tasks.append(self._scheduler_task) - self._scheduler_task = None for task in tasks: task.cancel() for task in tasks: @@ -60,43 +31,6 @@ async def stop(self) -> None: await task self._manual_tasks.clear() - async def _scheduler_loop(self) -> None: - while True: - try: - await self.run_due_once() - except asyncio.CancelledError: - raise - except Exception: - logger.exception("Application validation scheduler iteration failed") - await asyncio.sleep(self.poll_seconds) - - async def run_due_once( - self, - now: datetime.datetime | None = None, - ) -> list[dict[str, Any]]: - scheduled_for = _due_date( - now or datetime.datetime.now(datetime.timezone.utc) - ) - if scheduled_for is None: - return [] - - with self.backend.db as db: - leaderboards = db.get_leaderboards() - - summaries = [] - for leaderboard in leaderboards: - if leaderboard["task"].validation is None: - continue - for gpu_type in leaderboard["gpu_types"]: - summaries.append( - await self.run_sweep( - leaderboard["name"], - gpu_type, - scheduled_for=scheduled_for, - ) - ) - return summaries - def enqueue_manual_sweep( self, leaderboard_name: str, @@ -109,7 +43,6 @@ def enqueue_manual_sweep( self.run_sweep( leaderboard_name, gpu_type, - scheduled_for=None, all_users=all_users, only_missing=only_missing, ), @@ -233,7 +166,6 @@ async def run_sweep( leaderboard_name: str, gpu_type: str, *, - scheduled_for: datetime.date | None, all_users: bool = False, only_missing: bool = False, ) -> dict[str, Any]: @@ -254,15 +186,8 @@ async def run_sweep( leaderboard_id=leaderboard["id"], gpu_type=gpu_type, contract_version=validation.version, - scheduled_for=scheduled_for, + scheduled_for=None, ) - if sweep_id is None: - return { - "status": "already_claimed", - "leaderboard": leaderboard_name, - "gpu_type": gpu_type, - "scheduled_for": scheduled_for, - } submissions = db.get_leaderboard_submissions( leaderboard_name, gpu_type, @@ -308,7 +233,6 @@ async def run_sweep( "sweep_id": sweep_id, "leaderboard": leaderboard_name, "gpu_type": gpu_type, - "scheduled_for": scheduled_for, "contract_version": validation.version, "all_users": all_users, "only_missing": only_missing, diff --git a/src/libkernelbot/leaderboard_db.py b/src/libkernelbot/leaderboard_db.py index 9c0d0de8..ec17c90b 100644 --- a/src/libkernelbot/leaderboard_db.py +++ b/src/libkernelbot/leaderboard_db.py @@ -888,10 +888,10 @@ def claim_validation_sweep( contract_version: str, scheduled_for: Optional[datetime.date], ) -> Optional[int]: - """Atomically claim a nightly validation sweep. + """Record the start of an application-validation batch. - ``scheduled_for=None`` is reserved for explicit admin-triggered sweeps and - intentionally permits more than one run. + Admin-triggered batches use ``scheduled_for=None``, which intentionally + permits more than one run. """ try: self.cursor.execute( diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index 2b0a6352..2f35d918 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -126,7 +126,6 @@ def test_manual_validation_can_wait_for_debug_result( mock_validation_service.run_sweep.assert_awaited_once_with( "cholesky", "B200", - scheduled_for=None, all_users=False, only_missing=False, ) diff --git a/tests/test_application_validation.py b/tests/test_application_validation.py index 0aed4fc7..1425cfed 100644 --- a/tests/test_application_validation.py +++ b/tests/test_application_validation.py @@ -1,13 +1,9 @@ import asyncio -import datetime from types import SimpleNamespace import pytest -from libkernelbot.application_validation import ( - ApplicationValidationService, - _due_date, -) +from libkernelbot.application_validation import ApplicationValidationService from libkernelbot.task import LeaderboardTask @@ -53,9 +49,6 @@ def __enter__(self): def __exit__(self, *_args): return None - def get_leaderboards(self): - return [self.leaderboard] - def get_leaderboard(self, name): assert name == "cholesky" return self.leaderboard @@ -128,15 +121,6 @@ async def run_validation(self, config, _gpu): } -def test_due_date_uses_problem_timezone(): - assert _due_date( - datetime.datetime(2026, 7, 27, 4, 59, tzinfo=datetime.timezone.utc), - ) is None - assert _due_date( - datetime.datetime(2026, 7, 27, 5, 0, tzinfo=datetime.timezone.utc), - ) == datetime.date(2026, 7, 26) - - @pytest.mark.asyncio async def test_sweep_validates_top_ten_with_bounded_concurrency(): database = FakeDB() @@ -147,7 +131,6 @@ async def test_sweep_validates_top_ten_with_bounded_concurrency(): summary = await service.run_sweep( "cholesky", "B200", - scheduled_for=datetime.date(2026, 7, 26), ) assert summary["status"] == "completed" @@ -170,7 +153,6 @@ async def test_manual_sweep_can_validate_every_ranked_user(): summary = await service.run_sweep( "cholesky", "B200", - scheduled_for=None, all_users=True, ) @@ -193,7 +175,6 @@ async def test_manual_sweep_can_validate_only_missing_users(): summary = await service.run_sweep( "cholesky", "B200", - scheduled_for=None, all_users=True, only_missing=True, ) @@ -212,19 +193,3 @@ async def test_manual_sweep_can_validate_only_missing_users(): ] assert database.requested_limits == [None] assert launcher.max_active == 2 - - -@pytest.mark.asyncio -async def test_nightly_sweep_is_claimed_once(): - database = FakeDB() - launcher = FakeLauncher() - backend = SimpleNamespace(db=database, launcher_map={"B200": launcher}) - service = ApplicationValidationService(backend) - now = datetime.datetime(2026, 7, 27, 5, 1, tzinfo=datetime.timezone.utc) - - first = await service.run_due_once(now) - second = await service.run_due_once(now) - - assert first[0]["status"] == "completed" - assert second[0]["status"] == "already_claimed" - assert len(database.saved) == 10