Make application validation manual-only

README.md

@@ -34,7 +34,7 @@ yoyo apply src/migrations -d postgresql://user:password@localhost/clusterdev
 See [docs/database.md](docs/database.md) for migration patterns and creating new migrations.
 
 For production incident triage, see [docs/production-debugging.md](docs/production-debugging.md).
-For nightly end-to-end kernel checks, see
+For admin-triggered end-to-end kernel checks, see
 [docs/application-validation.md](docs/application-validation.md).
 
 ### Environment Variables
@@ -54,7 +54,6 @@ PROBLEM_DEV_DIR=examples
 DISABLE_SSL=1           # Set for local development
 GITHUB_TOKEN_BACKUP=    # Fallback token for rate limiting
 ADMIN_TOKEN=            # Token for admin API endpoints
-APPLICATION_VALIDATION_ENABLED=true
 
 # Discord bot (only needed if testing Discord integration)
 # See docs/discord.md for setup instructions

docs/application-validation.md

@@ -26,16 +26,13 @@ JSON result:
 }
 ```
 
-## Schedule
+## Manual batches
 
-Every day at 22:00 `America/Los_Angeles`, one KernelBot replica claims each
-`(leaderboard, GPU, contract version, local date)` sweep. It snapshots the
-current best submission from each of the top 10 users and runs at most two
-Modal jobs concurrently. The database claim prevents duplicate sweeps.
-
-Set `APPLICATION_VALIDATION_ENABLED=false` to disable the scheduler. A failed
-job is recorded as `VALIDATION ERROR`; a completed job is stored as `X/Y
-VALIDATED`.
+Application validation never runs automatically. An admin explicitly starts a
+batch for one leaderboard and GPU. By default it snapshots the current best
+submission from each of the top 10 users and runs at most two Modal jobs
+concurrently. A failed job is recorded as `VALIDATION ERROR`; a completed job
+is stored as `X/Y VALIDATED`.
 
 ## Local debug
 
@@ -47,7 +44,6 @@ modal environment create cholesky-validation-debug
 modal deploy --env cholesky-validation-debug src/runners/modal_runner_archs.py
 
 MODAL_ENVIRONMENT=cholesky-validation-debug \
-APPLICATION_VALIDATION_ENABLED=false \
 python src/kernelbot/main.py --api-only --debug
 ```
 
@@ -69,10 +65,9 @@ For a one-time backfill of every ranked user's current best submission, add
 kernelbot-admin validate-top10 cholesky B200 --all-users --only-missing
 ```
 
-This override applies only to the manual sweep. Scheduled sweeps remain capped
-at the top 10 users. `--only-missing` skips exact submissions that already have
-a result for the active validation contract, making it suitable for continuing
-a backfill after leaderboard changes.
+`--only-missing` skips exact submissions that already have a result for the
+active validation contract, making it suitable for continuing a backfill after
+leaderboard changes.
 
 Roll out KernelBot's migration and runner first, then the reference-kernels
 contract, then the Kernelboard badge.

src/kernelbot/api/main.py

@@ -503,7 +503,6 @@ async def admin_run_application_validation(
         return await application_validation_service.run_sweep(
             leaderboard_name,
             gpu_type,
-            scheduled_for=None,
             all_users=all_users,
             only_missing=only_missing,
         )

src/kernelbot/env.py

@@ -42,10 +42,6 @@
 # PostgreSQL-specific constants
 env.DATABASE_URL = os.getenv("DATABASE_URL")
 env.DISABLE_SSL = os.getenv("DISABLE_SSL")
-env.APPLICATION_VALIDATION_ENABLED = os.getenv(
-    "APPLICATION_VALIDATION_ENABLED",
-    "true",
-).lower() in {"1", "true", "yes"}
 
 
 def init_environment(skip_discord: bool = False):

src/kernelbot/main.py

@@ -57,12 +57,8 @@ async def run_api_server(backend: KernelBackend):
     manager = init_background_submission_manager(BackgroundSubmissionManager(backend))
     await manager.start()
     validation_service = init_application_validation_service(
-        ApplicationValidationService(
-            backend,
-            enabled=env.APPLICATION_VALIDATION_ENABLED,
-        )
+        ApplicationValidationService(backend)
     )
-    await validation_service.start()
 
     server = create_uvicorn_server()
     try:
@@ -273,12 +269,8 @@ async def start_bot_and_api(debug_mode: bool):
     manager = init_background_submission_manager(BackgroundSubmissionManager(bot_instance.backend))
     await manager.start()
     validation_service = init_application_validation_service(
-        ApplicationValidationService(
-            bot_instance.backend,
-            enabled=env.APPLICATION_VALIDATION_ENABLED,
-        )
+        ApplicationValidationService(bot_instance.backend)
     )
-    await validation_service.start()
 
     server = create_uvicorn_server()
     try:

src/libkernelbot/application_validation.py

@@ -1,102 +1,36 @@
-"""Nightly application-level validation for ranked kernel submissions."""
+"""Admin-triggered application-level validation for ranked kernel submissions."""
 
 from __future__ import annotations
 
 import asyncio
 import contextlib
-import datetime
 import math
 from typing import Any
-from zoneinfo import ZoneInfo
 
 from libkernelbot.consts import get_gpu_by_name
 from libkernelbot.task import LeaderboardTask, build_validation_config
 from libkernelbot.utils import KernelBotError, setup_logging
 
 logger = setup_logging(__name__)
 
-SCHEDULE = datetime.time(22, 0)
-TIMEZONE = ZoneInfo("America/Los_Angeles")
 TOP_K = 10
 MAX_CONCURRENCY = 2
 
 
-def _due_date(now: datetime.datetime) -> datetime.date | None:
-    if now.tzinfo is None:
-        now = now.replace(tzinfo=datetime.timezone.utc)
-    local_now = now.astimezone(TIMEZONE)
-    if local_now.time() < SCHEDULE:
-        return None
-    return local_now.date()
-
-
 class ApplicationValidationService:
-    def __init__(self, backend, *, enabled: bool = False, poll_seconds: int = 60):
+    def __init__(self, backend):
         self.backend = backend
-        self.enabled = enabled
-        self.poll_seconds = poll_seconds
-        self._scheduler_task: asyncio.Task | None = None
         self._manual_tasks: set[asyncio.Task] = set()
 
-    async def start(self) -> None:
-        if not self.enabled or self._scheduler_task is not None:
-            return
-        logger.info("Starting application validation scheduler")
-        self._scheduler_task = asyncio.create_task(
-            self._scheduler_loop(),
-            name="application-validation-scheduler",
-        )
-
     async def stop(self) -> None:
         tasks = list(self._manual_tasks)
-        if self._scheduler_task is not None:
-            self._scheduler_task.cancel()
-            tasks.append(self._scheduler_task)
-            self._scheduler_task = None
         for task in tasks:
             task.cancel()
         for task in tasks:
             with contextlib.suppress(asyncio.CancelledError):
                 await task
         self._manual_tasks.clear()
 
-    async def _scheduler_loop(self) -> None:
-        while True:
-            try:
-                await self.run_due_once()
-            except asyncio.CancelledError:
-                raise
-            except Exception:
-                logger.exception("Application validation scheduler iteration failed")
-            await asyncio.sleep(self.poll_seconds)
-
-    async def run_due_once(
-        self,
-        now: datetime.datetime | None = None,
-    ) -> list[dict[str, Any]]:
-        scheduled_for = _due_date(
-            now or datetime.datetime.now(datetime.timezone.utc)
-        )
-        if scheduled_for is None:
-            return []
-
-        with self.backend.db as db:
-            leaderboards = db.get_leaderboards()
-
-        summaries = []
-        for leaderboard in leaderboards:
-            if leaderboard["task"].validation is None:
-                continue
-            for gpu_type in leaderboard["gpu_types"]:
-                summaries.append(
-                    await self.run_sweep(
-                        leaderboard["name"],
-                        gpu_type,
-                        scheduled_for=scheduled_for,
-                    )
-                )
-        return summaries
-
     def enqueue_manual_sweep(
         self,
         leaderboard_name: str,
@@ -109,7 +43,6 @@ def enqueue_manual_sweep(
             self.run_sweep(
                 leaderboard_name,
                 gpu_type,
-                scheduled_for=None,
                 all_users=all_users,
                 only_missing=only_missing,
             ),
@@ -233,7 +166,6 @@ async def run_sweep(
         leaderboard_name: str,
         gpu_type: str,
         *,
-        scheduled_for: datetime.date | None,
         all_users: bool = False,
         only_missing: bool = False,
     ) -> dict[str, Any]:
@@ -254,15 +186,8 @@ async def run_sweep(
                 leaderboard_id=leaderboard["id"],
                 gpu_type=gpu_type,
                 contract_version=validation.version,
-                scheduled_for=scheduled_for,
+                scheduled_for=None,
             )
-            if sweep_id is None:
-                return {
-                    "status": "already_claimed",
-                    "leaderboard": leaderboard_name,
-                    "gpu_type": gpu_type,
-                    "scheduled_for": scheduled_for,
-                }
             submissions = db.get_leaderboard_submissions(
                 leaderboard_name,
                 gpu_type,
@@ -308,7 +233,6 @@ async def run_sweep(
                 "sweep_id": sweep_id,
                 "leaderboard": leaderboard_name,
                 "gpu_type": gpu_type,
-                "scheduled_for": scheduled_for,
                 "contract_version": validation.version,
                 "all_users": all_users,
                 "only_missing": only_missing,

src/libkernelbot/leaderboard_db.py

@@ -888,10 +888,10 @@ def claim_validation_sweep(
         contract_version: str,
         scheduled_for: Optional[datetime.date],
     ) -> Optional[int]:
-        """Atomically claim a nightly validation sweep.
+        """Record the start of an application-validation batch.
 
-        ``scheduled_for=None`` is reserved for explicit admin-triggered sweeps and
-        intentionally permits more than one run.
+        Admin-triggered batches use ``scheduled_for=None``, which intentionally
+        permits more than one run.
         """
         try:
             self.cursor.execute(

tests/test_admin_api.py

@@ -126,7 +126,6 @@ def test_manual_validation_can_wait_for_debug_result(
         mock_validation_service.run_sweep.assert_awaited_once_with(
             "cholesky",
             "B200",
-            scheduled_for=None,
             all_users=False,
             only_missing=False,
         )

tests/test_application_validation.py

@@ -1,13 +1,9 @@
 import asyncio
-import datetime
 from types import SimpleNamespace
 
 import pytest
 
-from libkernelbot.application_validation import (
-    ApplicationValidationService,
-    _due_date,
-)
+from libkernelbot.application_validation import ApplicationValidationService
 from libkernelbot.task import LeaderboardTask
 
 
@@ -53,9 +49,6 @@ def __enter__(self):
     def __exit__(self, *_args):
         return None
 
-    def get_leaderboards(self):
-        return [self.leaderboard]
-
     def get_leaderboard(self, name):
         assert name == "cholesky"
         return self.leaderboard
@@ -128,15 +121,6 @@ async def run_validation(self, config, _gpu):
         }
 
 
-def test_due_date_uses_problem_timezone():
-    assert _due_date(
-        datetime.datetime(2026, 7, 27, 4, 59, tzinfo=datetime.timezone.utc),
-    ) is None
-    assert _due_date(
-        datetime.datetime(2026, 7, 27, 5, 0, tzinfo=datetime.timezone.utc),
-    ) == datetime.date(2026, 7, 26)
-
-
 @pytest.mark.asyncio
 async def test_sweep_validates_top_ten_with_bounded_concurrency():
     database = FakeDB()
@@ -147,7 +131,6 @@ async def test_sweep_validates_top_ten_with_bounded_concurrency():
     summary = await service.run_sweep(
         "cholesky",
         "B200",
-        scheduled_for=datetime.date(2026, 7, 26),
     )
 
     assert summary["status"] == "completed"
@@ -170,7 +153,6 @@ async def test_manual_sweep_can_validate_every_ranked_user():
     summary = await service.run_sweep(
         "cholesky",
         "B200",
-        scheduled_for=None,
         all_users=True,
     )
 
@@ -193,7 +175,6 @@ async def test_manual_sweep_can_validate_only_missing_users():
     summary = await service.run_sweep(
         "cholesky",
         "B200",
-        scheduled_for=None,
         all_users=True,
         only_missing=True,
     )
@@ -212,19 +193,3 @@ async def test_manual_sweep_can_validate_only_missing_users():
     ]
     assert database.requested_limits == [None]
     assert launcher.max_active == 2
-
-
-@pytest.mark.asyncio
-async def test_nightly_sweep_is_claimed_once():
-    database = FakeDB()
-    launcher = FakeLauncher()
-    backend = SimpleNamespace(db=database, launcher_map={"B200": launcher})
-    service = ApplicationValidationService(backend)
-    now = datetime.datetime(2026, 7, 27, 5, 1, tzinfo=datetime.timezone.utc)
-
-    first = await service.run_due_once(now)
-    second = await service.run_due_once(now)
-
-    assert first[0]["status"] == "completed"
-    assert second[0]["status"] == "already_claimed"
-    assert len(database.saved) == 10