feat(evaluation): Improve summary readability

backend/app/core/config.py

@@ -216,6 +216,8 @@ def AWS_S3_BUCKET(self) -> str:
     # One of: none | minimal | low | medium | high | xhigh.
     EVAL_JUDGE_REASONING_EFFORT: str = "medium"
 
+    EVAL_SUMMARY_MODEL: str = "gpt-5-mini"
+
     # Judge runs alone in the single aggregate task (no response calls competing),
     # so it uses a larger pool than the response stage to finish the max dataset
     # (EVAL_FAST_MAX_UNIQUE_ROWS x duplication) well under CELERY_TASK_SOFT_TIME_LIMIT.

backend/app/crud/evaluations/core.py

@@ -485,6 +485,8 @@ def save_score(
         # TODO: Evaluate whether this behaviour is needed or completely discard the storing data in db
         if score_trace_url:
             db_score = {"summary_scores": summary_score}
+            if score.get("overall") is not None:
+                db_score["overall"] = score["overall"]
         else:
             # fallback to store data in db if failed to store in s3
             db_score = score

backend/app/crud/evaluations/fast.py

@@ -49,6 +49,10 @@
     update_evaluation_run,
 )
 from app.crud.evaluations.cost import attach_cost
+from app.crud.evaluations.dataset import (
+    DATASET_META_DUPLICATION_FACTOR,
+    get_dataset_by_id,
+)
 from app.crud.evaluations.embeddings import (
     EMBEDDING_MODEL,
     calculate_cosine_similarity,
@@ -81,10 +85,13 @@
     UNSCOREABLE_EMPTY_GROUND_TRUTH,
     UNSCOREABLE_EMPTY_OUTPUT,
     EvaluationScore,
+    OverallSummary,
     TraceData,
     TraceScore,
+    compute_overall_summary,
     verdict_from_score,
 )
+from app.crud.evaluations.summary import generate_run_ai_summary
 from app.crud.job import (
     create_batch_job,
     delete_batch_job,
@@ -953,6 +960,7 @@ def _stage3_score_and_trace(
     unscoreable: dict[str, str] = {}  # {ref: reason}
     write_items: list[dict[str, Any]] = []
     summary_scores: list[dict[str, Any]] = []
+    overall: OverallSummary | None = None
 
     if is_judge_run:
         # v2: no cosine. A row is judgeable only with a non-empty generated AND
@@ -1109,6 +1117,40 @@ def _stage3_score_and_trace(
                 summary_scores=summary_scores,
             )
 
+        avg_by_name = {s["name"]: s["avg"] for s in summary_scores if "avg" in s}
+        metric_avgs = {
+            spec.key.value: avg_by_name[spec.score_name]
+            for spec in metrics
+            if spec.score_name in avg_by_name
+        }
+        overall = compute_overall_summary(
+            metric_avgs=metric_avgs,
+            metric_weights={spec.key.value: spec.weight for spec in metrics},
+            metric_names={spec.key.value: spec.score_name for spec in metrics},
+        )
+        if overall is not None:
+            # Falls back to 1 (no repetition) if the dataset/metadata can't be
+            # resolved, so the summary still generates.
+            dataset = get_dataset_by_id(
+                session=session,
+                dataset_id=eval_run.dataset_id,
+                organization_id=eval_run.organization_id,
+                project_id=eval_run.project_id,
+            )
+            metadata = dataset.dataset_metadata if dataset else None
+            duplication_factor = max(
+                1, int((metadata or {}).get(DATASET_META_DUPLICATION_FACTOR, 1))
+            )
+            overall["ai_summary"] = generate_run_ai_summary(
+                session=session,
+                openai_client=openai_client,
+                model=settings.EVAL_SUMMARY_MODEL,
+                overall=overall,
+                run_name=eval_run.run_name,
+                summary_scores=summary_scores,
+                duplication_factor=duplication_factor,
+            )
+
         # One combined call grades every metric, so its tokens can't be split per
         # metric — they land in a single "judge" cost stage.
         if judge_results and judge_model:
@@ -1230,6 +1272,8 @@ def _stage3_score_and_trace(
         "summary_scores": summary_scores,
         "traces": traces,
     }
+    if overall is not None:
+        score["overall"] = overall
     return eval_run, score, write_items
 
 
@@ -1312,7 +1356,12 @@ def run_fast_evaluation(
         eval_run=eval_run,
         update=EvaluationRunUpdate(
             status="completed",
-            score={"summary_scores": score["summary_scores"]},
+            # Persist the overall alongside the summary so GET run status shows the
+            # run-level score/verdict/breakdown without loading the S3 trace unit.
+            score={
+                "summary_scores": score["summary_scores"],
+                "overall": score.get("overall"),
+            },
             cost=eval_run.cost,
         ),
     )

backend/app/crud/evaluations/judge.py

@@ -82,6 +82,7 @@ class JudgeMetricSpec:
     score_name: str
     prompt_fragment: str
     required_inputs: tuple[JudgeInputEnum, ...]
+    weight: float
 
 
 # All metrics are graded by one combined call, so they share a single judge model
@@ -97,6 +98,7 @@ class JudgeMetricSpec:
             JudgeInputEnum.GENERATED_ANSWER,
             JudgeInputEnum.GOLDEN_ANSWER,
         ),
+        weight=0.5,
     ),
     JudgeMetricEnum.PROMPT: JudgeMetricSpec(
         key=JudgeMetricEnum.PROMPT,
@@ -107,6 +109,7 @@ class JudgeMetricSpec:
             JudgeInputEnum.QUESTION,
             JudgeInputEnum.GENERATED_ANSWER,
         ),
+        weight=0.2,
     ),
     JudgeMetricEnum.KNOWLEDGE_BASE: JudgeMetricSpec(
         key=JudgeMetricEnum.KNOWLEDGE_BASE,
@@ -117,6 +120,7 @@ class JudgeMetricSpec:
             JudgeInputEnum.GENERATED_ANSWER,
             JudgeInputEnum.RETRIEVED_CHUNKS,
         ),
+        weight=0.3,
     ),
 }
 

backend/app/crud/evaluations/score.py

@@ -235,8 +235,82 @@ class CategoricalSummaryScore(TypedDict):
 SummaryScore = NumericSummaryScore | CategoricalSummaryScore
 
 
+class OverallDimension(TypedDict):
+    """One judge metric's contribution to the run-level overall score."""
+
+    name: str
+    key: str
+    score: float
+    weight: float
+    delta: float
+    verdict: str
+
+
+class OverallSummary(TypedDict):
+    """Run-level weighted quality view for a v2 judge run.
+
+    `ai_summary` is filled by a best-effort LLM step after the deterministic
+    fields; it stays None when no summary was generated.
+    """
+
+    overall_score: float
+    verdict: str
+    ai_summary: str | None
+    breakdown: list[OverallDimension]
+
+
+def compute_overall_summary(
+    *,
+    metric_avgs: dict[str, float],
+    metric_weights: dict[str, float],
+    metric_names: dict[str, str],
+) -> OverallSummary | None:
+    """Weighted run-level overall score + per-dimension breakdown. No LLM.
+
+    All three dicts are keyed by metric key value. Only metrics present in
+    `metric_avgs` (i.e. that actually scored ≥1 row) count; a metric with no
+    scoreable rows is dropped and the remaining base weights are renormalized to
+    sum to 1, so a missing metric never drags the overall down. Returns None when
+    nothing scored. `ai_summary` is None here — the LLM step fills it later.
+    """
+    scored_keys = [key for key in metric_avgs if key in metric_weights]
+    weight_total = sum(metric_weights[key] for key in scored_keys)
+    if not scored_keys or weight_total <= 0:
+        return None
+
+    renorm_weights = {key: metric_weights[key] / weight_total for key in scored_keys}
+    # Round the overall once, then reuse it everywhere so the badge and the number
+    # (and every delta) are computed from the same value and can never disagree.
+    overall_score = round(
+        sum(renorm_weights[key] * metric_avgs[key] for key in scored_keys), 2
+    )
+    verdict = verdict_from_score(overall_score).value
+
+    breakdown: list[OverallDimension] = []
+    for key in scored_keys:
+        avg = round(metric_avgs[key], 2)
+        breakdown.append(
+            {
+                "name": metric_names.get(key, key),
+                "key": key,
+                "score": avg,
+                "weight": round(renorm_weights[key], 2),
+                "delta": round(avg - overall_score, 2),
+                "verdict": verdict_from_score(avg).value,
+            }
+        )
+
+    return {
+        "overall_score": overall_score,
+        "verdict": verdict,
+        "ai_summary": None,
+        "breakdown": breakdown,
+    }
+
+
 class EvaluationScore(TypedDict):
     """Complete evaluation score data with traces and summary statistics."""
 
     summary_scores: list[SummaryScore]
     traces: list[TraceData]
+    overall: NotRequired[OverallSummary]