diff --git a/.claude/agents/senior-engineer.md b/.claude/agents/senior-engineer.md index 777a23f6f..856d11099 100644 --- a/.claude/agents/senior-engineer.md +++ b/.claude/agents/senior-engineer.md @@ -46,6 +46,9 @@ to what the task needs. **Cross-cutting:** when a service or crud function wraps an external SDK or raw HTTP call, also Read `.claude/conventions/error-handling.md` and apply its source-tagged, fault-based pattern. + **Coding Style:** unroll loops for better readability and maintainability. Prefer clarity over clever + language tricks. Do not write one-liner loops that involves 1) nested loops 2) complex pydantic interfaces. + 4. **Before writing a helper, check it doesn't already exist.** Anything generic — wrapping an external SDK or its error handling, building/parsing a domain payload, hitting cloud storage, loading config — usually has a canonical version already, and it rarely lives in a neighbor file. diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index c31288ce8..c285d0fe2 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,4 +1,5 @@ import logging +from typing import Any import sentry_sdk from fastapi import APIRouter, Depends @@ -9,6 +10,8 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs +from app.crud.stats import get_daily_stats +from app.services.stats import format_sections, post_to_discord logger = logging.getLogger(__name__) @@ -33,6 +36,16 @@ "recovery_threshold": 1, } +DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = { + "schedule": {"type": "crontab", "value": "0 9 * * *"}, + "timezone": "UTC", + "checkin_margin": 5, + "max_runtime": 10, + "failure_issue_threshold": 1, + "recovery_threshold": 1, +} + + PENDING_JOBS_CRON_MONITOR_CONFIG: MonitorConfig = { "schedule": { "type": "interval", @@ -123,6 +136,29 @@ async def evaluation_cron_job( raise +@router.get( + "/cron/daily-stats", + include_in_schema=False, + dependencies=[Depends(require_permission(Permission.SUPERUSER))], +) +@sentry_sdk.monitor( + monitor_slug="daily-stats-cron-job", + monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, +) +def daily_stats_cron_job(session: SessionDep) -> dict[str, Any]: + try: + stats = get_daily_stats(session=session) + post_to_discord(format_sections(stats)) + return stats + except Exception as e: + logger.error( + f"[daily_stats_cron_job] Error executing cron job: {e}", + exc_info=True, + ) + sentry_sdk.capture_exception(e) + raise + + @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 337f2bd4f..879458223 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -47,6 +47,7 @@ class Settings(BaseSettings): PROJECT_NAME: str API_VERSION: str = "0.5.0" SENTRY_DSN: HttpUrl | None = None + DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None POSTGRES_SERVER: str POSTGRES_PORT: int = 5432 POSTGRES_USER: str diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py new file mode 100644 index 000000000..dbae337a6 --- /dev/null +++ b/backend/app/crud/stats.py @@ -0,0 +1,150 @@ +from typing import Any + +from sqlalchemy import text +from sqlmodel import Session + +# Every query reports two rolling windows in one pass: the last 24 hours and the +# last 7 days (168 hours), broken down per organization and per project. The +# 24h FILTER count is a subset of the rows the 7d WHERE already selected. + +LLM_CALLS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours') AS calls_24h, + COUNT(*) AS calls_7d + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' + AND l.deleted_at IS NULL + GROUP BY o.name, p.name + ORDER BY calls_7d DESC +""" + +LLM_TOKENS = """ + SELECT + o.name AS organization, + p.name AS project, + l.model AS model, + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER) + FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours'), 0) AS tokens_24h, + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS tokens_7d + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' + AND l.deleted_at IS NULL + GROUP BY o.name, p.name, l.model + ORDER BY tokens_7d DESC +""" + +LLM_MODALITY = """ + SELECT + o.name AS organization, + p.name AS project, + CASE + WHEN l.input_type = 'text' AND l.output_type = 'text' THEN 'TEXT' + WHEN l.input_type = 'audio' AND l.output_type = 'text' THEN 'STT' + WHEN l.input_type = 'text' AND l.output_type = 'audio' THEN 'TTS' + ELSE 'OTHER' + END AS modality, + COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours') AS calls_24h, + COUNT(*) AS calls_7d + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' + AND l.deleted_at IS NULL + GROUP BY o.name, p.name, modality + ORDER BY o.name, p.name, modality +""" + +JOBS = """ + SELECT + o.name AS organization, + p.name AS project, + j.job_type AS job_type, + COUNT(*) FILTER (WHERE j.inserted_at >= now() - INTERVAL '24 hours') AS jobs_24h, + COUNT(*) AS jobs_7d + FROM job j + INNER JOIN project p ON j.project_id = p.id + INNER JOIN organization o ON p.organization_id = o.id + WHERE j.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name, j.job_type + ORDER BY o.name, p.name, j.job_type +""" + +EVALUATION_RUNS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM evaluation_run t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" + +STT_RESULTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM stt_result t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" + +TTS_RESULTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM tts_result t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" + +ASSESSMENTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM assessment t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" + + +def _rows(session: Session, sql: str) -> list[dict[str, Any]]: + result = session.connection().execute(text(sql)) + return [dict(row) for row in result.mappings().all()] + + +def get_daily_stats(*, session: Session) -> dict[str, list[dict[str, Any]]]: + stats: dict[str, list[dict[str, Any]]] = {} + stats["LLM Calls"] = _rows(session, LLM_CALLS) + stats["LLM Tokens"] = _rows(session, LLM_TOKENS) + stats["LLM Modality"] = _rows(session, LLM_MODALITY) + stats["Jobs by Type"] = _rows(session, JOBS) + stats["Evaluation Runs"] = _rows(session, EVALUATION_RUNS) + stats["STT Results"] = _rows(session, STT_RESULTS) + stats["TTS Results"] = _rows(session, TTS_RESULTS) + stats["Assessments"] = _rows(session, ASSESSMENTS) + return stats diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py new file mode 100644 index 000000000..bc8787315 --- /dev/null +++ b/backend/app/services/stats.py @@ -0,0 +1,83 @@ +import logging +from typing import Any + +import requests + +from app.core.config import settings + +logger = logging.getLogger(__name__) + +DISCORD_LIMIT = 1900 # Discord caps message content at 2000; leave headroom. +MAX_COL_WIDTH = 18 # Cap each column so wide tables don't wrap in Discord. + + +def _clip(text: str) -> str: + if len(text) <= MAX_COL_WIDTH: + return text + return text[: MAX_COL_WIDTH - 1] + "…" + + +def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[str]: + sections: list[str] = [] + for title, rows in stats.items(): + if not rows: + sections.append(f"**{title}**\n_no data_") + continue + + columns = list(rows[0].keys()) + + # Numeric columns get thousands separators and are right-aligned so they + # read as a clean column; text columns stay left-aligned. + numeric = { + c: all(isinstance(row[c], (int, float)) for row in rows) for c in columns + } + + def cell(column: str, row: dict[str, Any]) -> str: + value = row[column] + text = f"{value:,}" if numeric[column] else str(value) + return _clip(text) + + widths = {} + for column in columns: + cell_lengths = [len(cell(column, row)) for row in rows] + widths[column] = max(len(_clip(column)), max(cell_lengths)) + + def align(text: str, column: str) -> str: + width = widths[column] + return text.rjust(width) if numeric[column] else text.ljust(width) + + header = " ".join(align(_clip(column), column) for column in columns) + lines = [header.rstrip()] + for row in rows: + line = " ".join(align(cell(column, row), column) for column in columns) + lines.append(line.rstrip()) + + table = "\n".join(lines) + sections.append(f"**{title}**\n```\n{table}\n```") + return sections + + +def post_to_discord(sections: list[str]) -> None: + url = settings.DISCORD_STATS_WEBHOOK_URL + if not url: + return + + # Pack whole sections into messages under Discord's size cap so no code + # block is split across two posts. + chunk = "Daily Stats · last 24h and 7d (UTC)" + for section in sections: + if len(chunk) + len(section) + 2 > DISCORD_LIMIT: + _post(str(url), chunk) + chunk = "" + chunk = f"{chunk}\n\n{section}" if chunk else section + if chunk: + _post(str(url), chunk) + + +def _post(url: str, content: str) -> None: + try: + response = requests.post(url, json={"content": content}, timeout=5) + response.raise_for_status() + except requests.RequestException as e: + # Log only the exception type — the message can contain the webhook URL. + logger.warning(f"[_post] Webhook post failed: {type(e).__name__}") diff --git a/backend/app/tests/api/routes/test_cron.py b/backend/app/tests/api/routes/test_cron.py index 7b52ee4f9..399ed6eee 100644 --- a/backend/app/tests/api/routes/test_cron.py +++ b/backend/app/tests/api/routes/test_cron.py @@ -1,7 +1,9 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch +import pytest from fastapi.testclient import TestClient +from app.api.routes import cron from app.core.config import settings from app.tests.utils.auth import TestAuthContext @@ -272,6 +274,58 @@ def test_pending_jobs_cron_job_requires_superuser( assert "superuser" in response_data["error"].lower() +def test_daily_stats_cron_job_success( + client: TestClient, + superuser_api_key: TestAuthContext, +) -> None: + """Returns the collected stats and posts them to Discord.""" + stats = { + "LLM Calls": [ + {"organization": "Acme", "project": "Alpha", "calls_24h": 1, "calls_7d": 2} + ], + } + with ( + patch("app.api.routes.cron.get_daily_stats", return_value=stats), + patch("app.api.routes.cron.post_to_discord") as post, + ): + response = client.get( + f"{settings.API_V1_STR}/cron/daily-stats", + headers={"X-API-KEY": superuser_api_key.key}, + ) + + assert response.status_code == 200 + assert response.json() == stats + post.assert_called_once() + + +def test_daily_stats_cron_job_requires_superuser( + client: TestClient, + user_api_key: TestAuthContext, +) -> None: + """Non-superuser cannot access the daily stats cron endpoint.""" + response = client.get( + f"{settings.API_V1_STR}/cron/daily-stats", + headers={"X-API-KEY": user_api_key.key}, + ) + + assert response.status_code == 403 + + +def test_daily_stats_cron_job_captures_and_reraises_on_error() -> None: + """On failure the job reports to Sentry and re-raises.""" + with ( + patch( + "app.api.routes.cron.get_daily_stats", + side_effect=RuntimeError("boom"), + ), + patch("app.api.routes.cron.sentry_sdk") as sentry, + ): + with pytest.raises(RuntimeError): + cron.daily_stats_cron_job(session=MagicMock()) + + sentry.capture_exception.assert_called_once() + + def test_evaluation_cron_job_not_in_schema( client: TestClient, ) -> None: @@ -285,6 +339,7 @@ def test_evaluation_cron_job_not_in_schema( # Endpoint should not be in the schema due to include_in_schema=False assert f"{settings.API_V1_STR}/cron/evaluations" not in paths assert f"{settings.API_V1_STR}/cron/pending-jobs" not in paths + assert f"{settings.API_V1_STR}/cron/daily-stats" not in paths def test_cron_intervals_match_to_prevent_sentry_monitor_drift() -> None: diff --git a/backend/app/tests/crud/test_stats.py b/backend/app/tests/crud/test_stats.py new file mode 100644 index 000000000..b7453ed24 --- /dev/null +++ b/backend/app/tests/crud/test_stats.py @@ -0,0 +1,27 @@ +from unittest.mock import MagicMock + +from app.crud.stats import get_daily_stats + +EXPECTED_SECTIONS = { + "LLM Calls", + "LLM Tokens", + "LLM Modality", + "Jobs by Type", + "Evaluation Runs", + "STT Results", + "TTS Results", + "Assessments", +} + + +def test_get_daily_stats_runs_every_section_and_maps_rows(): + row = {"organization": "Acme", "project": "Alpha", "calls_7d": 5} + session = MagicMock() + execute = session.connection.return_value.execute + execute.return_value.mappings.return_value.all.return_value = [row] + + stats = get_daily_stats(session=session) + + assert set(stats) == EXPECTED_SECTIONS + assert execute.call_count == len(EXPECTED_SECTIONS) # one query per section + assert stats["LLM Calls"] == [row] # _rows unpacks each mapping into a dict diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py new file mode 100644 index 000000000..16fb2089c --- /dev/null +++ b/backend/app/tests/services/test_stats.py @@ -0,0 +1,94 @@ +from unittest.mock import MagicMock, patch + +import requests + +from app.services import stats as stats_mod +from app.services.stats import format_sections, post_to_discord + + +def _sample_stats() -> dict: + return { + "LLM Calls": [ + { + "organization": "Acme", + "project": "Alpha", + "calls_24h": 3, + "calls_7d": 15, + }, + ], + "STT Results": [], + } + + +def test_format_sections_renders_bold_title_and_aligned_table(): + sections = format_sections(_sample_stats()) + llm_section = next(s for s in sections if s.startswith("**LLM Calls**")) + assert "organization project calls_24h calls_7d" in llm_section + assert "Acme Alpha 3 15" in llm_section + assert llm_section.count("```") == 2 # wrapped in one code block + + +def test_format_sections_marks_empty_sections(): + sections = format_sections(_sample_stats()) + stt_section = next(s for s in sections if s.startswith("**STT Results**")) + assert stt_section == "**STT Results**\n_no data_" + + +def test_format_sections_clips_long_values_and_formats_numbers(): + stats = { + "LLM Tokens": [ + { + "organization": "Org", + "model": "gemini-3.1-flash-tts-preview", # 28 chars, over the cap + "tokens_7d": 89271, + }, + ], + } + section = format_sections(stats)[0] + assert "gemini-3.1-flash-…" in section # clipped to 17 chars + ellipsis + assert "gemini-3.1-flash-tts-preview" not in section + assert "89,271" in section # thousands separator applied + + +def test_post_to_discord_noop_when_webhook_unset(): + with patch.object(stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", None), patch( + "app.services.stats.requests.post" + ) as mock_post: + post_to_discord(["anything"]) + mock_post.assert_not_called() + + +def test_post_to_discord_packs_sections_under_size_limit(): + posted: list[str] = [] + + def fake_post(url, json, timeout): + posted.append(json["content"]) + return MagicMock() # provides raise_for_status() + + big_sections = ["x" * 1000 for _ in range(4)] + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", side_effect=fake_post): + post_to_discord(big_sections) + assert len(posted) >= 2 # split into multiple messages + assert all(len(content) <= 2000 for content in posted) + + +def test_post_to_discord_swallows_request_exception(): + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch( + "app.services.stats.requests.post", + side_effect=requests.ConnectionError("boom"), + ): + post_to_discord(["hello"]) # must not raise + + +def test_post_to_discord_swallows_non_success_status(): + response = MagicMock() + response.raise_for_status.side_effect = requests.HTTPError("429 Too Many Requests") + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", return_value=response): + post_to_discord(["hello"]) # must not raise + response.raise_for_status.assert_called_once() diff --git a/scripts/python/invoke-cron.py b/scripts/python/invoke-cron.py index 64df37b25..e589c65aa 100644 --- a/scripts/python/invoke-cron.py +++ b/scripts/python/invoke-cron.py @@ -18,6 +18,7 @@ ENDPOINTS = [ "/api/v1/cron/evaluations", "/api/v1/cron/pending-jobs", + "/api/v1/cron/daily-stats", ] REQUEST_TIMEOUT = 30 # Timeout for requests in seconds