Health Probes: For llm_call endpoints

.env.example

@@ -84,7 +84,9 @@ CELERY_ENABLE_UTC=true
 # India Standard Time (UTC+05:30)
 CELERY_TIMEZONE=Asia/Kolkata
 
-
+HEALTH_PROBE_API_KEY=
+HEALTH_PROBE_LLM_CALL_URL=http://127.0.0.1:8000/api/v1/llm/call
+DISCORD_STATS_WEBHOOK_URL=
 # Callback Timeouts and size limit(in seconds and MB respectively)
 CALLBACK_CONNECT_TIMEOUT = 3
 CALLBACK_READ_TIMEOUT = 10

backend/app/api/routes/cron.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 
 import sentry_sdk
 from fastapi import APIRouter, Depends
@@ -8,20 +9,19 @@
 from app.api.permissions import Permission, require_permission
 from app.core.config import settings
 from app.crud.evaluations import process_all_pending_evaluations
+from app.services.health_probes import run_health_probe_tick
 from app.services.job_monitoring import monitor_pending_jobs
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(tags=["Cron"])
 
 EVALUATION_CRON_MONITOR_CONFIG: MonitorConfig = {
-    # Expected cadence: a check-in every CRON_INTERVAL_MINUTES minutes.
     "schedule": {
         "type": "interval",
         "value": settings.CRON_INTERVAL_MINUTES,
         "unit": "minute",
     },
-    # Timezone for the schedule (only affects crontab-style schedules).
     "timezone": "UTC",
     # Grace period (minutes) before a late check-in is marked as missed.
     "checkin_margin": 2,
@@ -123,6 +123,27 @@ async def evaluation_cron_job(
         raise
 
 
+@router.get(
+    "/cron/health-probes",
+    include_in_schema=False,
+    dependencies=[Depends(require_permission(Permission.SUPERUSER))],
+)
+def health_probes_cron_job() -> dict[str, Any]:
+    # Runs synchronously (no Celery task of its own) — the probe makes an
+    # authenticated HTTP call to this app's own `/llm/call` endpoint.
+    try:
+        result = run_health_probe_tick()
+        logger.info(
+            f"[health_probes_cron_job] Tick complete | job_id: {result.get('job_id')}, "
+            f"probe_index: {result.get('probe_index')}"
+        )
+        return result
+    except Exception as e:
+        logger.error(f"[health_probes_cron_job] Tick failed: {e}", exc_info=True)
+        sentry_sdk.capture_exception(e)
+        raise
+
+
 @router.get(
     "/cron/pending-jobs",
     include_in_schema=False,

backend/app/celery/tasks/job_execution.py

@@ -18,7 +18,6 @@
 from opentelemetry import context as otel_context
 from opentelemetry import trace
 from opentelemetry.propagate import extract
-
 from app.celery.celery_app import celery_app
 from app.celery.utils import gevent_timeout
 from app.core.config import settings

backend/app/core/config.py

@@ -193,6 +193,11 @@ def AWS_S3_BUCKET(self) -> str:
     EVAL_FAST_STALL_THRESHOLD_MINUTES: int = 15
     PENDING_JOB_QUERY_TIMEOUT_MS: int = 1000
 
+    HEALTH_PROBE_INTERVAL_MINUTES: int = 3
+    HEALTH_PROBE_API_KEY: str | None = None
+    # Full URL, e.g. http://localhost:8000/api/v1/llm/call
+    HEALTH_PROBE_LLM_CALL_URL: str | None = None
+
     # AI-assisted prompt improvement settings.
     # See docs/srd-ai-prompt-improvement.md for the full design rationale.
     # Platform-owned Anthropic key shared by every org/project for this feature,

backend/app/core/telemetry.py

@@ -190,11 +190,15 @@ def setup_telemetry(service_name: str | None = None) -> None:
     resource = _build_resource(service_name)
     tracer_provider = TracerProvider(resource=resource)
 
-    # Bridge OTel spans into Sentry as Sentry transactions and spans, with full attribute and error capture.
     if settings.SENTRY_DSN:
-        from sentry_sdk.integrations.opentelemetry import SentrySpanProcessor
+        from opentelemetry.propagate import set_global_textmap
+        from sentry_sdk.integrations.opentelemetry import (
+            SentryPropagator,
+            SentrySpanProcessor,
+        )
 
         tracer_provider.add_span_processor(SentrySpanProcessor())
+        set_global_textmap(SentryPropagator())
 
     trace.set_tracer_provider(tracer_provider)
 

backend/app/services/health_probes.py

@@ -0,0 +1,268 @@
+import base64
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+from uuid import UUID
+
+import httpx
+import redis
+import sentry_sdk
+from sentry_sdk.crons import MonitorStatus, capture_checkin
+from sentry_sdk.types import MonitorConfig
+from sqlmodel import Session
+
+from app.core.config import settings
+from app.core.db import engine
+from app.models.job import Job, JobStatus
+
+logger = logging.getLogger(__name__)
+
+PROBE_INPUT = "ping"
+
+# ~1sec "Hello" clip used as the STT probe's input audio.
+STT_AUDIO_PATH = (
+    Path(__file__).resolve().parents[1] / "assets" / "health_probe_hello.ogg"
+)
+STT_AUDIO_MIME = "audio/ogg"
+stt_audio_cache: dict[str, str | None] = {"value": None}
+
+HEALTH_PROBES_MONITOR_SLUG = "health-probes-cron-job"
+HEALTH_PROBES_MONITOR_CONFIG: MonitorConfig = {
+    "schedule": {
+        "type": "interval",
+        "value": settings.HEALTH_PROBE_INTERVAL_MINUTES,
+        "unit": "minute",
+    },
+    "timezone": "UTC",
+    "checkin_margin": 2,
+    "max_runtime": 2 * settings.HEALTH_PROBE_INTERVAL_MINUTES,
+    "failure_issue_threshold": 2,
+    "recovery_threshold": 1,
+}
+
+# Rotation state lives in Redis
+LAST_JOB_ID_KEY = "health_probe:last_job_id"
+INDEX_KEY = "health_probe:index"
+
+redis_client: redis.Redis = redis.from_url(settings.REDIS_URL, decode_responses=True)
+
+Modality = Literal["text", "tts", "stt"]
+
+
+@dataclass(frozen=True)
+class Probe:
+    provider: str
+    model: str
+    modality: Modality
+    # Extra Kaapi params a provider's mapper requires beyond `model`
+    # (Sarvam TTS needs `language`, ElevenLabs TTS needs `voice`).
+    params: dict[str, Any] = field(default_factory=dict)
+
+
+PROBES: list[Probe] = [
+    # Text
+    Probe(provider="openai", model="gpt-4o-mini", modality="text"),
+    Probe(provider="google", model="gemini-2.5-flash", modality="text"),
+    Probe(provider="google", model="gemini-2.5-pro", modality="text"),
+    Probe(provider="anthropic", model="claude-sonnet-4-6", modality="text"),
+    # TTS
+    Probe(provider="google", model="gemini-2.5-flash-preview-tts", modality="tts"),
+    Probe(provider="google", model="gemini-3.1-flash-tts-preview", modality="tts"),
+    Probe(provider="google", model="gemini-2.5-pro-preview-tts", modality="tts"),
+    Probe(
+        provider="sarvamai",
+        model="bulbul:v3",
+        modality="tts",
+        params={"voice": "simran", "language": "en-IN"},
+    ),
+    Probe(
+        provider="elevenlabs",
+        model="eleven_v3",
+        modality="tts",
+        params={"voice": "Sarah"},
+    ),
+    # STT
+    Probe(provider="google", model="gemini-2.5-pro", modality="stt"),
+    Probe(provider="google", model="gemini-2.5-flash", modality="stt"),
+    Probe(provider="google", model="gemini-3.1-pro-preview", modality="stt"),
+    Probe(provider="sarvamai", model="saaras:v3", modality="stt"),
+    Probe(provider="elevenlabs", model="scribe_v2", modality="stt"),
+]
+
+
+def load_stt_audio_b64() -> str:
+    if stt_audio_cache["value"] is None:
+        with open(STT_AUDIO_PATH, "rb") as f:
+            stt_audio_cache["value"] = base64.b64encode(f.read()).decode("ascii")
+    return stt_audio_cache["value"]
+
+
+def build_probe_payload(probe: Probe, index: int) -> dict[str, Any]:
+    if probe.modality == "stt":
+        query_input: Any = {
+            "type": "audio",
+            "content": {
+                "format": "base64",
+                "value": load_stt_audio_b64(),
+                "mime_type": STT_AUDIO_MIME,
+            },
+        }
+    else:
+        query_input = PROBE_INPUT
+
+    # No callback_url: the probe reads its result by polling `Job.status` on
+    # the next tick, so callback delivery (a best-effort side channel that
+    # never affects Job.status) buys nothing here.
+    return {
+        "query": {"input": query_input},
+        "config": {
+            "blob": {
+                "completion": {
+                    "provider": probe.provider,
+                    "type": probe.modality,
+                    "params": {"model": probe.model, **probe.params},
+                }
+            }
+        },
+        "include_provider_raw_response": False,
+        "request_metadata": {
+            "health_probe": True,
+            "probe_index": index,
+            "provider": probe.provider,
+            "modality": probe.modality,
+            "model": probe.model,
+        },
+    }
+
+
+def call_llm_probe(payload: dict[str, Any]) -> UUID:
+    if not settings.HEALTH_PROBE_API_KEY or not settings.HEALTH_PROBE_LLM_CALL_URL:
+        raise RuntimeError(
+            "[call_llm_probe] HEALTH_PROBE_API_KEY and HEALTH_PROBE_LLM_CALL_URL "
+            "must both be set to fire a health probe"
+        )
+
+    headers = {"X-API-KEY": settings.HEALTH_PROBE_API_KEY}
+    response = httpx.post(
+        settings.HEALTH_PROBE_LLM_CALL_URL,
+        json=payload,
+        headers=headers,
+        timeout=30.0,
+    )
+    response.raise_for_status()
+    body = response.json()
+    job_id_str = body["data"]["job_id"]
+    return UUID(job_id_str)
+
+
+def check_previous_probe() -> JobStatus | None:
+    # Missing key (first run, Redis eviction) means skip the check-in, not fail the tick.
+    try:
+        last_job_id = redis_client.get(LAST_JOB_ID_KEY)
+    except redis.RedisError as e:
+        logger.warning(f"[check_previous_probe] Redis GET failed | error: {e}")
+        sentry_sdk.capture_exception(e, level="warning")
+        return None
+
+    if last_job_id is None:
+        logger.warning(
+            f"[check_previous_probe] {LAST_JOB_ID_KEY} missing — skipping check-in"
+        )
+        sentry_sdk.capture_message(
+            f"[check_previous_probe] {LAST_JOB_ID_KEY} missing — skipping check-in",
+            level="warning",
+        )
+        return None
+
+    try:
+        job_uuid = UUID(str(last_job_id))
+    except ValueError as e:
+        logger.warning(
+            f"[check_previous_probe] Malformed job id | value: {last_job_id!r}, error: {e}"
+        )
+        sentry_sdk.capture_exception(e, level="warning")
+        return None
+
+    with Session(engine) as session:
+        job = session.get(Job, job_uuid)
+
+    if job is None:
+        logger.warning(f"[check_previous_probe] Job not found | job_id: {last_job_id}")
+        sentry_sdk.capture_message(
+            f"[check_previous_probe] Job not found | job_id: {last_job_id}",
+            level="warning",
+        )
+        return None
+
+    return job.status
+
+
+def capture_previous_result_checkin(previous_status: JobStatus | None) -> None:
+    if previous_status is None:
+        return
+    # SUCCESS means the probe actually ran and returned; FAILED or a
+    # still-PENDING/PROCESSING job both mean a real failure.
+    monitor_status = (
+        MonitorStatus.OK
+        if previous_status == JobStatus.SUCCESS
+        else MonitorStatus.ERROR
+    )
+    capture_checkin(
+        monitor_slug=HEALTH_PROBES_MONITOR_SLUG,
+        status=monitor_status,
+        monitor_config=HEALTH_PROBES_MONITOR_CONFIG,
+    )
+
+
+def claim_next_probe_index() -> int:
+    # Redis INCR is atomic, so two overlapping ticks can never claim the same
+    # slot (which would fire the same probe twice while skipping another).
+    try:
+        count = redis_client.incr(INDEX_KEY)
+    except redis.RedisError as e:
+        logger.warning(f"[claim_next_probe_index] Redis INCR failed | error: {e}")
+        sentry_sdk.capture_exception(e, level="warning")
+        return 0
+
+    try:
+        return (int(count) - 1) % len(PROBES)
+    except (TypeError, ValueError) as e:
+        logger.warning(f"[claim_next_probe_index] Non-integer index | raw: {count!r}")
+        sentry_sdk.capture_exception(e, level="warning")
+        return 0
+
+
+def store_last_job_id(job_id: UUID) -> None:
+    try:
+        redis_client.set(LAST_JOB_ID_KEY, str(job_id))
+    except redis.RedisError as e:
+        logger.error(
+            f"[store_last_job_id] Redis SET failed | job_id: {job_id}, error: {e}"
+        )
+        sentry_sdk.capture_exception(e)
+
+
+def run_health_probe_tick() -> dict[str, Any]:
+    previous_status = check_previous_probe()
+    capture_previous_result_checkin(previous_status)
+
+    index = claim_next_probe_index()
+    probe = PROBES[index]
+    payload = build_probe_payload(probe, index)
+
+    job_id = call_llm_probe(payload)
+
+    store_last_job_id(job_id)
+
+    logger.info(
+        f"[run_health_probe_tick] Fired probe | provider: {probe.provider}, "
+        f"model: {probe.model}, modality: {probe.modality}, job_id: {job_id}, "
+        f"previous_job_status: {previous_status}"
+    )
+    return {
+        "enqueued": True,
+        "job_id": job_id,
+        "probe_index": index,
+        "previous_job_status": previous_status.value if previous_status else None,
+    }