Discord: Automate Stat Messages

.claude/agents/senior-engineer.md

@@ -46,6 +46,9 @@ to what the task needs.
    **Cross-cutting:** when a service or crud function wraps an external SDK or raw HTTP call, also
    Read `.claude/conventions/error-handling.md` and apply its source-tagged, fault-based pattern.
 
+   **Coding Style:** unroll loops for better readability and maintainability. Prefer clarity over clever
+   language tricks. Do not write one-liner loops that involves 1) nested loops 2) complex pydantic interfaces.
+
 4. **Before writing a helper, check it doesn't already exist.** Anything generic — wrapping an
    external SDK or its error handling, building/parsing a domain payload, hitting cloud storage,
    loading config — usually has a canonical version already, and it rarely lives in a neighbor file.

backend/app/api/routes/cron.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 
 import sentry_sdk
 from fastapi import APIRouter, Depends
@@ -9,6 +10,8 @@
 from app.core.config import settings
 from app.crud.evaluations import process_all_pending_evaluations
 from app.services.job_monitoring import monitor_pending_jobs
+from app.crud.stats import get_daily_stats
+from app.services.stats import format_sections, post_to_discord
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +36,16 @@
     "recovery_threshold": 1,
 }
 
+DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = {
+    "schedule": {"type": "crontab", "value": "0 9 * * *"},
+    "timezone": "UTC",
+    "checkin_margin": 5,
+    "max_runtime": 10,
+    "failure_issue_threshold": 1,
+    "recovery_threshold": 1,
+}
+
+
 PENDING_JOBS_CRON_MONITOR_CONFIG: MonitorConfig = {
     "schedule": {
         "type": "interval",
@@ -123,6 +136,29 @@ async def evaluation_cron_job(
         raise
 
 
+@router.get(
+    "/cron/daily-stats",
+    include_in_schema=False,
+    dependencies=[Depends(require_permission(Permission.SUPERUSER))],
+)
+@sentry_sdk.monitor(
+    monitor_slug="daily-stats-cron-job",
+    monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG,
+)
+def daily_stats_cron_job(session: SessionDep) -> dict[str, Any]:
+    try:
+        stats = get_daily_stats(session=session)
+        post_to_discord(format_sections(stats))
+        return stats
+    except Exception as e:
+        logger.error(
+            f"[daily_stats_cron_job] Error executing cron job: {e}",
+            exc_info=True,
+        )
+        sentry_sdk.capture_exception(e)
+        raise
+
+
 @router.get(
     "/cron/pending-jobs",
     include_in_schema=False,

backend/app/core/config.py

@@ -47,6 +47,7 @@ class Settings(BaseSettings):
     PROJECT_NAME: str
     API_VERSION: str = "0.5.0"
     SENTRY_DSN: HttpUrl | None = None
+    DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None
     POSTGRES_SERVER: str
     POSTGRES_PORT: int = 5432
     POSTGRES_USER: str

backend/app/crud/stats.py

@@ -0,0 +1,150 @@
+from typing import Any
+
+from sqlalchemy import text
+from sqlmodel import Session
+
+# Every query reports two rolling windows in one pass: the last 24 hours and the
+# last 7 days (168 hours), broken down per organization and per project. The
+# 24h FILTER count is a subset of the rows the 7d WHERE already selected.
+
+LLM_CALLS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours')  AS calls_24h,
+        COUNT(*) AS calls_7d
+    FROM llm_call l
+    INNER JOIN organization o ON l.organization_id = o.id
+    INNER JOIN project p ON l.project_id = p.id
+    WHERE l.inserted_at >= now() - INTERVAL '168 hours'
+      AND l.deleted_at IS NULL
+    GROUP BY o.name, p.name
+    ORDER BY calls_7d DESC
+"""
+
+LLM_TOKENS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        l.model AS model,
+        COALESCE(SUM((l.usage->>'total_tokens')::INTEGER)
+            FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours'), 0) AS tokens_24h,
+        COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS tokens_7d
+    FROM llm_call l
+    INNER JOIN organization o ON l.organization_id = o.id
+    INNER JOIN project p ON l.project_id = p.id
+    WHERE l.inserted_at >= now() - INTERVAL '168 hours'
+      AND l.deleted_at IS NULL
+    GROUP BY o.name, p.name, l.model
+    ORDER BY tokens_7d DESC
+"""
+
+LLM_MODALITY = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        CASE
+            WHEN l.input_type = 'text'  AND l.output_type = 'text'  THEN 'TEXT'
+            WHEN l.input_type = 'audio' AND l.output_type = 'text'  THEN 'STT'
+            WHEN l.input_type = 'text'  AND l.output_type = 'audio' THEN 'TTS'
+            ELSE 'OTHER'
+        END AS modality,
+        COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours')  AS calls_24h,
+        COUNT(*) AS calls_7d
+    FROM llm_call l
+    INNER JOIN organization o ON l.organization_id = o.id
+    INNER JOIN project p ON l.project_id = p.id
+    WHERE l.inserted_at >= now() - INTERVAL '168 hours'
+      AND l.deleted_at IS NULL
+    GROUP BY o.name, p.name, modality
+    ORDER BY o.name, p.name, modality
+"""
+
+JOBS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        j.job_type AS job_type,
+        COUNT(*) FILTER (WHERE j.inserted_at >= now() - INTERVAL '24 hours')  AS jobs_24h,
+        COUNT(*) AS jobs_7d
+    FROM job j
+    INNER JOIN project p ON j.project_id = p.id
+    INNER JOIN organization o ON p.organization_id = o.id
+    WHERE j.inserted_at >= now() - INTERVAL '168 hours'
+    GROUP BY o.name, p.name, j.job_type
+    ORDER BY o.name, p.name, j.job_type
+"""
+
+EVALUATION_RUNS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours')  AS count_24h,
+        COUNT(*) AS count_7d
+    FROM evaluation_run t
+    INNER JOIN organization o ON t.organization_id = o.id
+    INNER JOIN project p ON t.project_id = p.id
+    WHERE t.inserted_at >= now() - INTERVAL '168 hours'
+    GROUP BY o.name, p.name
+    ORDER BY count_7d DESC
+"""
+
+STT_RESULTS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours')  AS count_24h,
+        COUNT(*) AS count_7d
+    FROM stt_result t
+    INNER JOIN organization o ON t.organization_id = o.id
+    INNER JOIN project p ON t.project_id = p.id
+    WHERE t.inserted_at >= now() - INTERVAL '168 hours'
+    GROUP BY o.name, p.name
+    ORDER BY count_7d DESC
+"""
+
+TTS_RESULTS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours')  AS count_24h,
+        COUNT(*) AS count_7d
+    FROM tts_result t
+    INNER JOIN organization o ON t.organization_id = o.id
+    INNER JOIN project p ON t.project_id = p.id
+    WHERE t.inserted_at >= now() - INTERVAL '168 hours'
+    GROUP BY o.name, p.name
+    ORDER BY count_7d DESC
+"""
+
+ASSESSMENTS = """
+    SELECT
+        o.name AS organization,
+        p.name AS project,
+        COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours')  AS count_24h,
+        COUNT(*) AS count_7d
+    FROM assessment t
+    INNER JOIN organization o ON t.organization_id = o.id
+    INNER JOIN project p ON t.project_id = p.id
+    WHERE t.inserted_at >= now() - INTERVAL '168 hours'
+    GROUP BY o.name, p.name
+    ORDER BY count_7d DESC
+"""
+
+
+def _rows(session: Session, sql: str) -> list[dict[str, Any]]:
+    result = session.connection().execute(text(sql))
+    return [dict(row) for row in result.mappings().all()]
+
+
+def get_daily_stats(*, session: Session) -> dict[str, list[dict[str, Any]]]:
+    stats: dict[str, list[dict[str, Any]]] = {}
+    stats["LLM Calls"] = _rows(session, LLM_CALLS)
+    stats["LLM Tokens"] = _rows(session, LLM_TOKENS)
+    stats["LLM Modality"] = _rows(session, LLM_MODALITY)
+    stats["Jobs by Type"] = _rows(session, JOBS)
+    stats["Evaluation Runs"] = _rows(session, EVALUATION_RUNS)
+    stats["STT Results"] = _rows(session, STT_RESULTS)
+    stats["TTS Results"] = _rows(session, TTS_RESULTS)
+    stats["Assessments"] = _rows(session, ASSESSMENTS)
+    return stats

backend/app/services/stats.py

@@ -0,0 +1,83 @@
+import logging
+from typing import Any
+
+import requests
+
+from app.core.config import settings
+
+logger = logging.getLogger(__name__)
+
+DISCORD_LIMIT = 1900  # Discord caps message content at 2000; leave headroom.
+MAX_COL_WIDTH = 18  # Cap each column so wide tables don't wrap in Discord.
+
+
+def _clip(text: str) -> str:
+    if len(text) <= MAX_COL_WIDTH:
+        return text
+    return text[: MAX_COL_WIDTH - 1] + "…"
+
+
+def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[str]:
+    sections: list[str] = []
+    for title, rows in stats.items():
+        if not rows:
+            sections.append(f"**{title}**\n_no data_")
+            continue
+
+        columns = list(rows[0].keys())
+
+        # Numeric columns get thousands separators and are right-aligned so they
+        # read as a clean column; text columns stay left-aligned.
+        numeric = {
+            c: all(isinstance(row[c], (int, float)) for row in rows) for c in columns
+        }
+
+        def cell(column: str, row: dict[str, Any]) -> str:
+            value = row[column]
+            text = f"{value:,}" if numeric[column] else str(value)
+            return _clip(text)
+
+        widths = {}
+        for column in columns:
+            cell_lengths = [len(cell(column, row)) for row in rows]
+            widths[column] = max(len(_clip(column)), max(cell_lengths))
+
+        def align(text: str, column: str) -> str:
+            width = widths[column]
+            return text.rjust(width) if numeric[column] else text.ljust(width)
+
+        header = "  ".join(align(_clip(column), column) for column in columns)
+        lines = [header.rstrip()]
+        for row in rows:
+            line = "  ".join(align(cell(column, row), column) for column in columns)
+            lines.append(line.rstrip())
+
+        table = "\n".join(lines)
+        sections.append(f"**{title}**\n```\n{table}\n```")
+    return sections
+
+
+def post_to_discord(sections: list[str]) -> None:
+    url = settings.DISCORD_STATS_WEBHOOK_URL
+    if not url:
+        return
+
+    # Pack whole sections into messages under Discord's size cap so no code
+    # block is split across two posts.
+    chunk = "Daily Stats · last 24h and 7d (UTC)"
+    for section in sections:
+        if len(chunk) + len(section) + 2 > DISCORD_LIMIT:
+            _post(str(url), chunk)
+            chunk = ""
+        chunk = f"{chunk}\n\n{section}" if chunk else section
+    if chunk:
+        _post(str(url), chunk)
+
+
+def _post(url: str, content: str) -> None:
+    try:
+        response = requests.post(url, json={"content": content}, timeout=5)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        # Log only the exception type — the message can contain the webhook URL.
+        logger.warning(f"[_post] Webhook post failed: {type(e).__name__}")

backend/app/tests/api/routes/test_cron.py

@@ -1,7 +1,9 @@
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
+import pytest
 from fastapi.testclient import TestClient
 
+from app.api.routes import cron
 from app.core.config import settings
 from app.tests.utils.auth import TestAuthContext
 
@@ -272,6 +274,58 @@ def test_pending_jobs_cron_job_requires_superuser(
     assert "superuser" in response_data["error"].lower()
 
 
+def test_daily_stats_cron_job_success(
+    client: TestClient,
+    superuser_api_key: TestAuthContext,
+) -> None:
+    """Returns the collected stats and posts them to Discord."""
+    stats = {
+        "LLM Calls": [
+            {"organization": "Acme", "project": "Alpha", "calls_24h": 1, "calls_7d": 2}
+        ],
+    }
+    with (
+        patch("app.api.routes.cron.get_daily_stats", return_value=stats),
+        patch("app.api.routes.cron.post_to_discord") as post,
+    ):
+        response = client.get(
+            f"{settings.API_V1_STR}/cron/daily-stats",
+            headers={"X-API-KEY": superuser_api_key.key},
+        )
+
+    assert response.status_code == 200
+    assert response.json() == stats
+    post.assert_called_once()
+
+
+def test_daily_stats_cron_job_requires_superuser(
+    client: TestClient,
+    user_api_key: TestAuthContext,
+) -> None:
+    """Non-superuser cannot access the daily stats cron endpoint."""
+    response = client.get(
+        f"{settings.API_V1_STR}/cron/daily-stats",
+        headers={"X-API-KEY": user_api_key.key},
+    )
+
+    assert response.status_code == 403
+
+
+def test_daily_stats_cron_job_captures_and_reraises_on_error() -> None:
+    """On failure the job reports to Sentry and re-raises."""
+    with (
+        patch(
+            "app.api.routes.cron.get_daily_stats",
+            side_effect=RuntimeError("boom"),
+        ),
+        patch("app.api.routes.cron.sentry_sdk") as sentry,
+    ):
+        with pytest.raises(RuntimeError):
+            cron.daily_stats_cron_job(session=MagicMock())
+
+    sentry.capture_exception.assert_called_once()
+
+
 def test_evaluation_cron_job_not_in_schema(
     client: TestClient,
 ) -> None:
@@ -285,6 +339,7 @@ def test_evaluation_cron_job_not_in_schema(
     # Endpoint should not be in the schema due to include_in_schema=False
     assert f"{settings.API_V1_STR}/cron/evaluations" not in paths
     assert f"{settings.API_V1_STR}/cron/pending-jobs" not in paths
+    assert f"{settings.API_V1_STR}/cron/daily-stats" not in paths
 
 
 def test_cron_intervals_match_to_prevent_sentry_monitor_drift() -> None: