diff --git a/.env.example b/.env.example index 5a041fe26..0a4e96cc2 100644 --- a/.env.example +++ b/.env.example @@ -84,7 +84,9 @@ CELERY_ENABLE_UTC=true # India Standard Time (UTC+05:30) CELERY_TIMEZONE=Asia/Kolkata - +HEALTH_PROBE_API_KEY= +HEALTH_PROBE_LLM_CALL_URL=http://127.0.0.1:8000/api/v1/llm/call +DISCORD_STATS_WEBHOOK_URL= # Callback Timeouts and size limit(in seconds and MB respectively) CALLBACK_CONNECT_TIMEOUT = 3 CALLBACK_READ_TIMEOUT = 10 diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index c31288ce8..b6115c26b 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,4 +1,5 @@ import logging +from typing import Any import sentry_sdk from fastapi import APIRouter, Depends @@ -8,6 +9,7 @@ from app.api.permissions import Permission, require_permission from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations +from app.services.health_probes import run_health_probe_tick from app.services.job_monitoring import monitor_pending_jobs logger = logging.getLogger(__name__) @@ -15,13 +17,11 @@ router = APIRouter(tags=["Cron"]) EVALUATION_CRON_MONITOR_CONFIG: MonitorConfig = { - # Expected cadence: a check-in every CRON_INTERVAL_MINUTES minutes. "schedule": { "type": "interval", "value": settings.CRON_INTERVAL_MINUTES, "unit": "minute", }, - # Timezone for the schedule (only affects crontab-style schedules). "timezone": "UTC", # Grace period (minutes) before a late check-in is marked as missed. "checkin_margin": 2, @@ -123,6 +123,27 @@ async def evaluation_cron_job( raise +@router.get( + "/cron/health-probes", + include_in_schema=False, + dependencies=[Depends(require_permission(Permission.SUPERUSER))], +) +def health_probes_cron_job() -> dict[str, Any]: + # Runs synchronously (no Celery task of its own) — the probe makes an + # authenticated HTTP call to this app's own `/llm/call` endpoint. + try: + result = run_health_probe_tick() + logger.info( + f"[health_probes_cron_job] Tick complete | job_id: {result.get('job_id')}, " + f"probe_index: {result.get('probe_index')}" + ) + return result + except Exception as e: + logger.error(f"[health_probes_cron_job] Tick failed: {e}", exc_info=True) + sentry_sdk.capture_exception(e) + raise + + @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/assets/health_probe_hello.ogg b/backend/app/assets/health_probe_hello.ogg new file mode 100644 index 000000000..933c0ef6f Binary files /dev/null and b/backend/app/assets/health_probe_hello.ogg differ diff --git a/backend/app/celery/tasks/job_execution.py b/backend/app/celery/tasks/job_execution.py index 000c6025a..fcf7b1cfe 100644 --- a/backend/app/celery/tasks/job_execution.py +++ b/backend/app/celery/tasks/job_execution.py @@ -18,7 +18,6 @@ from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.propagate import extract - from app.celery.celery_app import celery_app from app.celery.utils import gevent_timeout from app.core.config import settings diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 337f2bd4f..8fdb4043b 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -193,6 +193,11 @@ def AWS_S3_BUCKET(self) -> str: EVAL_FAST_STALL_THRESHOLD_MINUTES: int = 15 PENDING_JOB_QUERY_TIMEOUT_MS: int = 1000 + HEALTH_PROBE_INTERVAL_MINUTES: int = 3 + HEALTH_PROBE_API_KEY: str | None = None + # Full URL, e.g. http://localhost:8000/api/v1/llm/call + HEALTH_PROBE_LLM_CALL_URL: str | None = None + # AI-assisted prompt improvement settings. # See docs/srd-ai-prompt-improvement.md for the full design rationale. # Platform-owned Anthropic key shared by every org/project for this feature, diff --git a/backend/app/core/telemetry.py b/backend/app/core/telemetry.py index 99d2fc959..95f64f13f 100644 --- a/backend/app/core/telemetry.py +++ b/backend/app/core/telemetry.py @@ -190,11 +190,15 @@ def setup_telemetry(service_name: str | None = None) -> None: resource = _build_resource(service_name) tracer_provider = TracerProvider(resource=resource) - # Bridge OTel spans into Sentry as Sentry transactions and spans, with full attribute and error capture. if settings.SENTRY_DSN: - from sentry_sdk.integrations.opentelemetry import SentrySpanProcessor + from opentelemetry.propagate import set_global_textmap + from sentry_sdk.integrations.opentelemetry import ( + SentryPropagator, + SentrySpanProcessor, + ) tracer_provider.add_span_processor(SentrySpanProcessor()) + set_global_textmap(SentryPropagator()) trace.set_tracer_provider(tracer_provider) diff --git a/backend/app/services/health_probes.py b/backend/app/services/health_probes.py new file mode 100644 index 000000000..42e0b30a1 --- /dev/null +++ b/backend/app/services/health_probes.py @@ -0,0 +1,268 @@ +import base64 +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal +from uuid import UUID + +import httpx +import redis +import sentry_sdk +from sentry_sdk.crons import MonitorStatus, capture_checkin +from sentry_sdk.types import MonitorConfig +from sqlmodel import Session + +from app.core.config import settings +from app.core.db import engine +from app.models.job import Job, JobStatus + +logger = logging.getLogger(__name__) + +PROBE_INPUT = "ping" + +# ~1sec "Hello" clip used as the STT probe's input audio. +STT_AUDIO_PATH = ( + Path(__file__).resolve().parents[1] / "assets" / "health_probe_hello.ogg" +) +STT_AUDIO_MIME = "audio/ogg" +stt_audio_cache: dict[str, str | None] = {"value": None} + +HEALTH_PROBES_MONITOR_SLUG = "health-probes-cron-job" +HEALTH_PROBES_MONITOR_CONFIG: MonitorConfig = { + "schedule": { + "type": "interval", + "value": settings.HEALTH_PROBE_INTERVAL_MINUTES, + "unit": "minute", + }, + "timezone": "UTC", + "checkin_margin": 2, + "max_runtime": 2 * settings.HEALTH_PROBE_INTERVAL_MINUTES, + "failure_issue_threshold": 2, + "recovery_threshold": 1, +} + +# Rotation state lives in Redis +LAST_JOB_ID_KEY = "health_probe:last_job_id" +INDEX_KEY = "health_probe:index" + +redis_client: redis.Redis = redis.from_url(settings.REDIS_URL, decode_responses=True) + +Modality = Literal["text", "tts", "stt"] + + +@dataclass(frozen=True) +class Probe: + provider: str + model: str + modality: Modality + # Extra Kaapi params a provider's mapper requires beyond `model` + # (Sarvam TTS needs `language`, ElevenLabs TTS needs `voice`). + params: dict[str, Any] = field(default_factory=dict) + + +PROBES: list[Probe] = [ + # Text + Probe(provider="openai", model="gpt-4o-mini", modality="text"), + Probe(provider="google", model="gemini-2.5-flash", modality="text"), + Probe(provider="google", model="gemini-2.5-pro", modality="text"), + Probe(provider="anthropic", model="claude-sonnet-4-6", modality="text"), + # TTS + Probe(provider="google", model="gemini-2.5-flash-preview-tts", modality="tts"), + Probe(provider="google", model="gemini-3.1-flash-tts-preview", modality="tts"), + Probe(provider="google", model="gemini-2.5-pro-preview-tts", modality="tts"), + Probe( + provider="sarvamai", + model="bulbul:v3", + modality="tts", + params={"voice": "simran", "language": "en-IN"}, + ), + Probe( + provider="elevenlabs", + model="eleven_v3", + modality="tts", + params={"voice": "Sarah"}, + ), + # STT + Probe(provider="google", model="gemini-2.5-pro", modality="stt"), + Probe(provider="google", model="gemini-2.5-flash", modality="stt"), + Probe(provider="google", model="gemini-3.1-pro-preview", modality="stt"), + Probe(provider="sarvamai", model="saaras:v3", modality="stt"), + Probe(provider="elevenlabs", model="scribe_v2", modality="stt"), +] + + +def load_stt_audio_b64() -> str: + if stt_audio_cache["value"] is None: + with open(STT_AUDIO_PATH, "rb") as f: + stt_audio_cache["value"] = base64.b64encode(f.read()).decode("ascii") + return stt_audio_cache["value"] + + +def build_probe_payload(probe: Probe, index: int) -> dict[str, Any]: + if probe.modality == "stt": + query_input: Any = { + "type": "audio", + "content": { + "format": "base64", + "value": load_stt_audio_b64(), + "mime_type": STT_AUDIO_MIME, + }, + } + else: + query_input = PROBE_INPUT + + # No callback_url: the probe reads its result by polling `Job.status` on + # the next tick, so callback delivery (a best-effort side channel that + # never affects Job.status) buys nothing here. + return { + "query": {"input": query_input}, + "config": { + "blob": { + "completion": { + "provider": probe.provider, + "type": probe.modality, + "params": {"model": probe.model, **probe.params}, + } + } + }, + "include_provider_raw_response": False, + "request_metadata": { + "health_probe": True, + "probe_index": index, + "provider": probe.provider, + "modality": probe.modality, + "model": probe.model, + }, + } + + +def call_llm_probe(payload: dict[str, Any]) -> UUID: + if not settings.HEALTH_PROBE_API_KEY or not settings.HEALTH_PROBE_LLM_CALL_URL: + raise RuntimeError( + "[call_llm_probe] HEALTH_PROBE_API_KEY and HEALTH_PROBE_LLM_CALL_URL " + "must both be set to fire a health probe" + ) + + headers = {"X-API-KEY": settings.HEALTH_PROBE_API_KEY} + response = httpx.post( + settings.HEALTH_PROBE_LLM_CALL_URL, + json=payload, + headers=headers, + timeout=30.0, + ) + response.raise_for_status() + body = response.json() + job_id_str = body["data"]["job_id"] + return UUID(job_id_str) + + +def check_previous_probe() -> JobStatus | None: + # Missing key (first run, Redis eviction) means skip the check-in, not fail the tick. + try: + last_job_id = redis_client.get(LAST_JOB_ID_KEY) + except redis.RedisError as e: + logger.warning(f"[check_previous_probe] Redis GET failed | error: {e}") + sentry_sdk.capture_exception(e, level="warning") + return None + + if last_job_id is None: + logger.warning( + f"[check_previous_probe] {LAST_JOB_ID_KEY} missing — skipping check-in" + ) + sentry_sdk.capture_message( + f"[check_previous_probe] {LAST_JOB_ID_KEY} missing — skipping check-in", + level="warning", + ) + return None + + try: + job_uuid = UUID(str(last_job_id)) + except ValueError as e: + logger.warning( + f"[check_previous_probe] Malformed job id | value: {last_job_id!r}, error: {e}" + ) + sentry_sdk.capture_exception(e, level="warning") + return None + + with Session(engine) as session: + job = session.get(Job, job_uuid) + + if job is None: + logger.warning(f"[check_previous_probe] Job not found | job_id: {last_job_id}") + sentry_sdk.capture_message( + f"[check_previous_probe] Job not found | job_id: {last_job_id}", + level="warning", + ) + return None + + return job.status + + +def capture_previous_result_checkin(previous_status: JobStatus | None) -> None: + if previous_status is None: + return + # SUCCESS means the probe actually ran and returned; FAILED or a + # still-PENDING/PROCESSING job both mean a real failure. + monitor_status = ( + MonitorStatus.OK + if previous_status == JobStatus.SUCCESS + else MonitorStatus.ERROR + ) + capture_checkin( + monitor_slug=HEALTH_PROBES_MONITOR_SLUG, + status=monitor_status, + monitor_config=HEALTH_PROBES_MONITOR_CONFIG, + ) + + +def claim_next_probe_index() -> int: + # Redis INCR is atomic, so two overlapping ticks can never claim the same + # slot (which would fire the same probe twice while skipping another). + try: + count = redis_client.incr(INDEX_KEY) + except redis.RedisError as e: + logger.warning(f"[claim_next_probe_index] Redis INCR failed | error: {e}") + sentry_sdk.capture_exception(e, level="warning") + return 0 + + try: + return (int(count) - 1) % len(PROBES) + except (TypeError, ValueError) as e: + logger.warning(f"[claim_next_probe_index] Non-integer index | raw: {count!r}") + sentry_sdk.capture_exception(e, level="warning") + return 0 + + +def store_last_job_id(job_id: UUID) -> None: + try: + redis_client.set(LAST_JOB_ID_KEY, str(job_id)) + except redis.RedisError as e: + logger.error( + f"[store_last_job_id] Redis SET failed | job_id: {job_id}, error: {e}" + ) + sentry_sdk.capture_exception(e) + + +def run_health_probe_tick() -> dict[str, Any]: + previous_status = check_previous_probe() + capture_previous_result_checkin(previous_status) + + index = claim_next_probe_index() + probe = PROBES[index] + payload = build_probe_payload(probe, index) + + job_id = call_llm_probe(payload) + + store_last_job_id(job_id) + + logger.info( + f"[run_health_probe_tick] Fired probe | provider: {probe.provider}, " + f"model: {probe.model}, modality: {probe.modality}, job_id: {job_id}, " + f"previous_job_status: {previous_status}" + ) + return { + "enqueued": True, + "job_id": job_id, + "probe_index": index, + "previous_job_status": previous_status.value if previous_status else None, + } diff --git a/backend/app/tests/api/routes/test_cron_health_probes.py b/backend/app/tests/api/routes/test_cron_health_probes.py new file mode 100644 index 000000000..8ccdc897a --- /dev/null +++ b/backend/app/tests/api/routes/test_cron_health_probes.py @@ -0,0 +1,74 @@ +from typing import Any +from unittest.mock import patch +from uuid import uuid4 + +from fastapi.testclient import TestClient + +from app.core.config import settings +from app.tests.utils.auth import TestAuthContext + + +def _fake_tick_result(**overrides: Any) -> dict[str, Any]: + result = { + "enqueued": True, + "job_id": str(uuid4()), + "probe_index": 4, + "previous_job_status": "SUCCESS", + } + result.update(overrides) + return result + + +def test_health_probes_cron_runs_tick_and_returns_result( + client: TestClient, + superuser_api_key: TestAuthContext, +) -> None: + canned = _fake_tick_result() + + with patch( + "app.api.routes.cron.run_health_probe_tick", return_value=canned + ) as tick_mock: + response = client.get( + f"{settings.API_V1_STR}/cron/health-probes", + headers={"X-API-KEY": superuser_api_key.key}, + ) + + assert response.status_code == 200 + assert response.json() == canned + tick_mock.assert_called_once_with() + + +def test_health_probes_cron_requires_superuser( + client: TestClient, + user_api_key: TestAuthContext, +) -> None: + with patch( + "app.api.routes.cron.run_health_probe_tick", + return_value=_fake_tick_result(), + ) as tick_mock: + response = client.get( + f"{settings.API_V1_STR}/cron/health-probes", + headers={"X-API-KEY": user_api_key.key}, + ) + + assert response.status_code == 403 + assert "Insufficient permissions" in response.json()["error"] + tick_mock.assert_not_called() + + +def test_health_probes_cron_requires_authentication(client: TestClient) -> None: + with patch( + "app.api.routes.cron.run_health_probe_tick", + return_value=_fake_tick_result(), + ) as tick_mock: + response = client.get(f"{settings.API_V1_STR}/cron/health-probes") + + assert response.status_code in (401, 403) + tick_mock.assert_not_called() + + +def test_health_probes_cron_not_in_openapi_schema(client: TestClient) -> None: + response = client.get(f"{settings.API_V1_STR}/openapi.json") + assert response.status_code == 200 + paths = response.json().get("paths", {}) + assert f"{settings.API_V1_STR}/cron/health-probes" not in paths diff --git a/backend/app/tests/services/test_health_probes.py b/backend/app/tests/services/test_health_probes.py new file mode 100644 index 000000000..86af5065d --- /dev/null +++ b/backend/app/tests/services/test_health_probes.py @@ -0,0 +1,357 @@ +import threading +from types import TracebackType +from typing import Any +from unittest.mock import MagicMock, Mock, patch +from uuid import UUID, uuid4 + +import httpx +import pytest +import redis +from sqlmodel import Session + +from app.core.config import settings +from app.crud.jobs import JobCrud +from app.models.job import JobStatus, JobType, JobUpdate +from app.models.llm.constants import CompletionType +from app.models.llm.request import KaapiCompletionConfig +from app.services import health_probes +from app.services.health_probes import PROBES, build_probe_payload +from app.services.llm.mappers import transform_kaapi_config_to_native +from app.tests.utils.test_data import create_test_project + + +class _NonClosingSession: + """Stands in for `Session(engine)` inside health_probes, backed by the + conftest transactional `db` session instead of closing a real one.""" + + def __init__(self, session: Session): + self._session = session + + def __enter__(self) -> Session: + return self._session + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool: + return False + + +class _FakeRedis: + """In-memory stand-in for `redis_client`. `incr` is lock-protected so it + actually replicates Redis's atomicity for the concurrency test below.""" + + def __init__(self) -> None: + self.store: dict[str, str] = {} + self._lock = threading.Lock() + + def get(self, key: str) -> str | None: + return self.store.get(key) + + def set(self, key: str, value: Any) -> None: + self.store[key] = str(value) + + def incr(self, key: str) -> int: + with self._lock: + count = int(self.store.get(key, "0")) + 1 + self.store[key] = str(count) + return count + + +@pytest.fixture +def probe_env(db: Session, monkeypatch: pytest.MonkeyPatch): + project = create_test_project(db) + monkeypatch.setattr( + health_probes, "Session", lambda _engine: _NonClosingSession(db) + ) + fake_redis = _FakeRedis() + monkeypatch.setattr(health_probes, "redis_client", fake_redis) + return project, fake_redis + + +def _create_job(db: Session, project_id: int, status: JobStatus) -> UUID: + job = JobCrud(session=db).create(job_type=JobType.LLM_API, project_id=project_id) + JobCrud(session=db).update(job_id=job.id, job_update=JobUpdate(status=status)) + return job.id + + +class TestFiresExactlyOneProbe: + def test_tick_enqueues_exactly_one_job( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + _, _fake_redis = probe_env + expected_job_id = uuid4() + + with patch( + "app.services.health_probes.call_llm_probe", return_value=expected_job_id + ) as call_llm_probe_mock: + result = health_probes.run_health_probe_tick() + + call_llm_probe_mock.assert_called_once() + assert result["enqueued"] is True + assert result["job_id"] == expected_job_id + + +class TestRoundRobinRotation: + def test_index_advances_and_wraps_across_full_registry( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + seen_indexes = [] + with patch( + "app.services.health_probes.call_llm_probe", + side_effect=lambda _payload: uuid4(), + ): + for _ in range(len(PROBES)): + result = health_probes.run_health_probe_tick() + seen_indexes.append(result["probe_index"]) + + assert seen_indexes == list(range(len(PROBES))) + + wrapped = health_probes.run_health_probe_tick() + assert wrapped["probe_index"] == 0 + + def test_concurrent_claims_never_collide_or_skip_a_slot( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + # N concurrent claims must land on N distinct, evenly spread slots. + from concurrent.futures import ThreadPoolExecutor + + rounds = 5 + total_claims = rounds * len(PROBES) + with ThreadPoolExecutor(max_workers=16) as pool: + claimed = list( + pool.map( + lambda _: health_probes.claim_next_probe_index(), + range(total_claims), + ) + ) + + counts = {i: claimed.count(i) for i in range(len(PROBES))} + assert all(count == rounds for count in counts.values()), counts + + +class TestMissingRedisKeysDoNotFailTick: + def test_missing_index_key_defaults_to_zero( + self, db: Session, probe_env: tuple[Any, _FakeRedis] + ) -> None: + project, fake_redis = probe_env + # last_job_id present (not a first-ever run), index key absent. + existing_job_id = _create_job(db, project.id, JobStatus.SUCCESS) + fake_redis.store[health_probes.LAST_JOB_ID_KEY] = str(existing_job_id) + + with patch( + "app.services.health_probes.call_llm_probe", return_value=uuid4() + ) as call_llm_probe_mock: + result = health_probes.run_health_probe_tick() + + call_llm_probe_mock.assert_called_once() + assert result["enqueued"] is True + assert result["probe_index"] == 0 + + def test_missing_last_job_id_key_skips_checkin_but_still_fires( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + _, fake_redis = probe_env + fake_redis.store[health_probes.INDEX_KEY] = "2" + + with ( + patch( + "app.services.health_probes.call_llm_probe", return_value=uuid4() + ) as call_llm_probe_mock, + patch("app.services.health_probes.capture_checkin") as checkin_mock, + ): + result = health_probes.run_health_probe_tick() + + call_llm_probe_mock.assert_called_once() + checkin_mock.assert_not_called() + assert result["enqueued"] is True + assert result["probe_index"] == 2 + assert result["previous_job_status"] is None + + +class TestPreviousProbeCheckin: + def test_previous_job_failed_reports_error_checkin( + self, db: Session, probe_env: tuple[Any, _FakeRedis] + ) -> None: + project, fake_redis = probe_env + failed_job_id = _create_job(db, project.id, JobStatus.FAILED) + fake_redis.store[health_probes.LAST_JOB_ID_KEY] = str(failed_job_id) + + with ( + patch("app.services.health_probes.call_llm_probe", return_value=uuid4()), + patch("app.services.health_probes.capture_checkin") as checkin_mock, + ): + result = health_probes.run_health_probe_tick() + + checkin_mock.assert_called_once() + assert ( + checkin_mock.call_args.kwargs["status"] == health_probes.MonitorStatus.ERROR + ) + assert result["previous_job_status"] == JobStatus.FAILED.value + + def test_previous_job_success_reports_ok_checkin( + self, db: Session, probe_env: tuple[Any, _FakeRedis] + ) -> None: + project, fake_redis = probe_env + success_job_id = _create_job(db, project.id, JobStatus.SUCCESS) + fake_redis.store[health_probes.LAST_JOB_ID_KEY] = str(success_job_id) + + with ( + patch("app.services.health_probes.call_llm_probe", return_value=uuid4()), + patch("app.services.health_probes.capture_checkin") as checkin_mock, + ): + result = health_probes.run_health_probe_tick() + + checkin_mock.assert_called_once() + assert checkin_mock.call_args.kwargs["status"] == health_probes.MonitorStatus.OK + assert result["previous_job_status"] == JobStatus.SUCCESS.value + + +class TestProbeRegistryPayloadsAreValid: + # FR-8/FR-10/FR-11: every registry entry must resolve through the real + # Kaapi -> native mapper with zero warnings (catches e.g. Sarvam TTS + # missing `language`, ElevenLabs TTS missing `voice`). + + @pytest.mark.parametrize( + "probe", + PROBES, + ids=[f"{p.provider}-{p.model}-{p.modality}" for p in PROBES], + ) + def test_probe_config_resolves_with_no_mapper_warnings( + self, db: Session, probe: health_probes.Probe + ) -> None: + payload = build_probe_payload(probe, index=0) + + completion = payload["config"]["blob"]["completion"] + completion_type = { + "text": CompletionType.TEXT, + "tts": CompletionType.TTS, + "stt": CompletionType.STT, + }[completion["type"]] + kaapi_config = KaapiCompletionConfig( + provider=completion["provider"], + type=completion_type, + params=completion["params"], + ) + + _native_config, warnings = transform_kaapi_config_to_native( + session=db, kaapi_config=kaapi_config + ) + + assert warnings == [] + + +class TestCallLlmProbe: + def test_missing_api_key_raises_clear_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(settings, "HEALTH_PROBE_API_KEY", None) + monkeypatch.setattr( + settings, + "HEALTH_PROBE_LLM_CALL_URL", + "http://localhost:8000/api/v1/llm/call", + ) + + with pytest.raises(RuntimeError, match="HEALTH_PROBE_API_KEY"): + health_probes.call_llm_probe({}) + + def test_missing_url_raises_clear_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(settings, "HEALTH_PROBE_API_KEY", "test-key") + monkeypatch.setattr(settings, "HEALTH_PROBE_LLM_CALL_URL", None) + + with pytest.raises(RuntimeError, match="HEALTH_PROBE_LLM_CALL_URL"): + health_probes.call_llm_probe({}) + + def test_success_returns_job_id(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "HEALTH_PROBE_API_KEY", "test-key") + monkeypatch.setattr( + settings, + "HEALTH_PROBE_LLM_CALL_URL", + "http://localhost:8000/api/v1/llm/call", + ) + expected_job_id = uuid4() + fake_response = MagicMock() + fake_response.json.return_value = {"data": {"job_id": str(expected_job_id)}} + + with patch( + "app.services.health_probes.httpx.post", return_value=fake_response + ) as post_mock: + job_id = health_probes.call_llm_probe({"query": {"input": "ping"}}) + + assert job_id == expected_job_id + fake_response.raise_for_status.assert_called_once() + post_mock.assert_called_once_with( + "http://localhost:8000/api/v1/llm/call", + json={"query": {"input": "ping"}}, + headers={"X-API-KEY": "test-key"}, + timeout=30.0, + ) + + def test_non_2xx_response_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "HEALTH_PROBE_API_KEY", "test-key") + monkeypatch.setattr( + settings, + "HEALTH_PROBE_LLM_CALL_URL", + "http://localhost:8000/api/v1/llm/call", + ) + fake_response = MagicMock() + fake_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "422 Unprocessable Entity", request=Mock(), response=Mock() + ) + + with patch("app.services.health_probes.httpx.post", return_value=fake_response): + with pytest.raises(httpx.HTTPStatusError): + health_probes.call_llm_probe({}) + + +class TestCheckPreviousProbeEdgeCases: + def test_redis_get_failure_returns_none( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + broken_redis = Mock() + broken_redis.get.side_effect = redis.RedisError("boom") + monkeypatch.setattr(health_probes, "redis_client", broken_redis) + + assert health_probes.check_previous_probe() is None + + def test_malformed_job_id_returns_none( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + _, fake_redis = probe_env + fake_redis.store[health_probes.LAST_JOB_ID_KEY] = "not-a-uuid" + + assert health_probes.check_previous_probe() is None + + def test_job_not_found_returns_none( + self, probe_env: tuple[Any, _FakeRedis] + ) -> None: + _, fake_redis = probe_env + fake_redis.store[health_probes.LAST_JOB_ID_KEY] = str(uuid4()) + + assert health_probes.check_previous_probe() is None + + +class TestClaimNextProbeIndexErrorHandling: + def test_redis_incr_failure_defaults_to_zero( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + broken_redis = Mock() + broken_redis.incr.side_effect = redis.RedisError("boom") + monkeypatch.setattr(health_probes, "redis_client", broken_redis) + + assert health_probes.claim_next_probe_index() == 0 + + +class TestStoreLastJobIdErrorHandling: + def test_redis_set_failure_is_swallowed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + broken_redis = Mock() + broken_redis.set.side_effect = redis.RedisError("boom") + monkeypatch.setattr(health_probes, "redis_client", broken_redis) + + health_probes.store_last_job_id(uuid4()) diff --git a/docs/wiki/INDEX.md b/docs/wiki/INDEX.md index 9bc299d34..4417c9d28 100644 --- a/docs/wiki/INDEX.md +++ b/docs/wiki/INDEX.md @@ -22,7 +22,7 @@ Deep design narrative lives in `docs/architecture/*.md`; open those only for des - [modules/responses.md](modules/responses.md) — OpenAI Responses API integration, conversations, threads, assistants. No deep-dive doc yet. - [modules/assessment.md](modules/assessment.md) — assessments and assessment runs. No deep-dive doc yet. - [modules/tenancy.md](modules/tenancy.md) — users, orgs, projects, API keys, onboarding, login. -- [modules/platform.md](modules/platform.md) — analytics, notifications, feature flags, languages, credentials, model config, cron. +- [modules/platform.md](modules/platform.md) — analytics, notifications, feature flags, languages, credentials, model config, cron (incl. round-robin health probes through the real `/llm/call` pipeline). ### Cross-cutting - [cross-cutting/auth.md](cross-cutting/auth.md) — JWT, API keys, org/project permission model. diff --git a/docs/wiki/modules/platform.md b/docs/wiki/modules/platform.md index 0f34e5d87..961394ef6 100644 --- a/docs/wiki/modules/platform.md +++ b/docs/wiki/modules/platform.md @@ -12,5 +12,5 @@ All paths relative to `backend/app/`. | Languages | `api/routes/languages.py` | `global.languages` (`models/language.py`) | `crud/language.py` | | Credentials | `api/routes/credentials.py` | `credential` (`models/credentials.py`) | `crud/credentials.py`; provider keys per org/project | | Model config | `api/routes/model_config.py` | `model_config` (`models/model_config.py`) | `crud/model_config.py` | -| Cron | `api/routes/cron.py` | — | triggers batch polling (`crud/evaluations/cron.py`) | +| Cron | `api/routes/cron.py` | — | triggers batch polling (`crud/evaluations/cron.py`); health probes (`services/health_probes.py`) fire one round-robin probe per tick through the real `/llm/call` pipeline (`services/llm/jobs.py::start_job`), rotation state in Redis (`health_probe:index`, `health_probe:last_job_id`), no dedicated table — result rides the `job` table | | Jobs | — | `job` (`models/job.py`), `batch_job` (`models/batch_job.py`) | `crud/jobs.py`, `crud/job/`, `services/job_monitoring.py` | diff --git a/features/health-probes/assets/flow-a.mmd b/features/health-probes/assets/flow-a.mmd new file mode 100644 index 000000000..ce38c8a81 --- /dev/null +++ b/features/health-probes/assets/flow-a.mmd @@ -0,0 +1,27 @@ +sequenceDiagram + participant Scheduler as External Scheduler + participant Cron as Kaapi backend (cron route) + participant Redis + participant DB as Postgres (job table) + participant Sentry + participant Celery as LLM call pipeline (Celery) + participant Provider as LLM Provider + + Scheduler->>Cron: GET /cron/health-probes + Cron->>Redis: GET last_job_id + alt key present + Cron->>DB: look up previous Job by id + DB-->>Cron: JobStatus (SUCCESS / FAILED / PENDING) + Cron->>Sentry: check-in reflecting previous probe's real status + else key missing + Cron->>Sentry: log entry, continue (no flow failure) + end + Cron->>Redis: INCR probe_index (atomic claim) + Cron->>Cron: pick probe = registry[index mod len(registry)] + Cron->>Celery: start_job(LLMCallRequest for probe, no callback_url) + Celery-->>Cron: job_id + Cron->>Redis: SET last_job_id + Cron-->>Scheduler: 200 OK (enqueued) + Celery->>Provider: resolved config + input (config -> mapper -> provider) + Provider-->>Celery: response or provider error + Celery->>DB: update Job status (SUCCESS / FAILED) diff --git a/features/health-probes/assets/flow-a.png b/features/health-probes/assets/flow-a.png new file mode 100644 index 000000000..a2bf8ee6a Binary files /dev/null and b/features/health-probes/assets/flow-a.png differ diff --git a/scripts/python/invoke-cron.py b/scripts/python/invoke-cron.py index 64df37b25..64b409573 100644 --- a/scripts/python/invoke-cron.py +++ b/scripts/python/invoke-cron.py @@ -18,6 +18,7 @@ ENDPOINTS = [ "/api/v1/cron/evaluations", "/api/v1/cron/pending-jobs", + "/api/v1/cron/health-probes", ] REQUEST_TIMEOUT = 30 # Timeout for requests in seconds