feat(evals): Added verdict categorization system

backend/app/crud/evaluations/fast.py

@@ -83,6 +83,7 @@
     EvaluationScore,
     TraceData,
     TraceScore,
+    verdict_from_score,
 )
 from app.crud.job import (
     create_batch_job,
@@ -1171,12 +1172,14 @@ def _stage3_score_and_trace(
                     comment = metric_score.reasoning
                     if is_kb:
                         comment = f"{comment} | Top matches: {top_matches}"
+                    rounded_score = round(metric_score.score, 2)
                     trace_scores.append(
                         {
                             "name": spec.score_name,
-                            "value": round(metric_score.score, 2),
+                            "value": rounded_score,
                             "data_type": "NUMERIC",
                             "comment": comment,
+                            "verdict": verdict_from_score(rounded_score),
                         }
                     )
                 elif is_kb:

backend/app/crud/evaluations/score.py

@@ -5,10 +5,36 @@
 used throughout the evaluation system.
 """
 
+from enum import Enum
 from typing import NotRequired, TypedDict
 
 DEFAULT_CATEGORY: str = "Other"
 
+
+class VerdictEnum(str, Enum):
+    """Qualitative band derived from a 0–1 judge-metric score."""
+
+    NEEDS_IMPROVEMENT = "Needs Improvement"
+    NEEDS_REFINEMENT = "Needs Refinement"
+    GOOD = "Good"
+
+
+VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3
+VERDICT_GOOD_AT_OR_ABOVE: float = 0.6
+
+
+def verdict_from_score(score: float) -> VerdictEnum:
+    """Map a 0–1 judge-metric score to its verdict band.
+
+    Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good.
+    """
+    if score < VERDICT_NEEDS_IMPROVEMENT_BELOW:
+        return VerdictEnum.NEEDS_IMPROVEMENT
+    if score < VERDICT_GOOD_AT_OR_ABOVE:
+        return VerdictEnum.NEEDS_REFINEMENT
+    return VerdictEnum.GOOD
+
+
 # Canonical name/comment for the cosine-similarity score, centralized to avoid
 # import cycles.
 COSINE_SCORE_NAME: str = "Cosine Similarity"
@@ -152,6 +178,7 @@ class TraceScore(TypedDict):
     value: float | str
     data_type: str
     comment: NotRequired[str]
+    verdict: NotRequired[str]
     # True for placeholder scores on unscoreable items; excluded from summary stats.
     unscoreable: NotRequired[bool]
 

backend/app/tests/crud/evaluations/test_fast_judge.py

@@ -43,6 +43,7 @@
     JUDGE_FAILED_REASON,
     KNOWLEDGE_BASE_SCORE_NAME,
     PROMPT_SCORE_NAME,
+    verdict_from_score,
 )
 from app.models import Config, EvaluationDataset, EvaluationRun
 from app.models.batch_job import BatchJob
@@ -620,6 +621,7 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores(
                 "value": 0.25,
                 "data_type": "NUMERIC",
                 "comment": "answered in English",
+                "verdict": "Needs Improvement",
             }
             assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9
 
@@ -1082,6 +1084,107 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered(
         assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None
 
 
+class TestVerdictBandOnTraceScores:
+    """each scored judge metric on a v2 trace carries a verdict band;
+    the cosine (v1) score and the KB N/A placeholder never do."""
+
+    def test_each_scored_judge_metric_carries_its_verdict_band(
+        self, db: Session, user_api_key: TestAuthContext, _s3_store
+    ):
+        eval_run = _make_run(
+            db=db,
+            user_api_key=user_api_key,
+            is_judge_run=True,
+            instructions=BOT_INSTRUCTIONS,
+        )
+        row = _resp_result("item-1", "Q1", "golden-1")
+        row["retrieved_chunks"] = [
+            {"score": 0.9, "text": "supporting", "filename": "kb.pdf"}
+        ]
+        _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store)
+
+        # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement,
+        # 0.75 → Good, so the three metrics land in three different bands.
+        result, _ = _run_pipeline(
+            db=db,
+            eval_run=eval_run,
+            judge_side_effect=lambda _p: _raw_judge_response(
+                json.dumps(
+                    {
+                        "ground_truth": {"score": 0.2, "reasoning": "gt"},
+                        "prompt": {"score": 0.45, "reasoning": "p"},
+                        "knowledge_base": {"score": 0.75, "reasoning": "kb"},
+                    }
+                )
+            ),
+        )
+
+        trace = _trace_by_ref(result)["item-1"]
+        gt = _score_named(trace, GROUND_TRUTH_SCORE_NAME)
+        prompt = _score_named(trace, PROMPT_SCORE_NAME)
+        kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME)
+
+        assert gt["verdict"] == "Needs Improvement"
+        assert prompt["verdict"] == "Needs Refinement"
+        assert kb["verdict"] == "Good"
+
+        for score in (gt, prompt, kb):
+            assert score["verdict"] == verdict_from_score(score["value"])
+
+    def test_cosine_score_carries_no_verdict(
+        self, db: Session, user_api_key: TestAuthContext, _s3_store
+    ):
+        eval_run = _make_run(
+            db=db,
+            user_api_key=user_api_key,
+            is_judge_run=False,
+            instructions=BOT_INSTRUCTIONS,
+        )
+        _seed_chunk(
+            db=db,
+            eval_run=eval_run,
+            results=[_resp_result("item-1", "Q1", "golden-1")],
+            store=_s3_store,
+        )
+
+        result, _ = _run_pipeline(
+            db=db,
+            eval_run=eval_run,
+            judge_side_effect=lambda _p: _judge_response(0.9, "never runs"),
+        )
+
+        cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME)
+        assert cosine is not None
+        assert "verdict" not in cosine
+
+    def test_kb_na_placeholder_carries_no_verdict(
+        self, db: Session, user_api_key: TestAuthContext, _s3_store
+    ):
+        eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True)
+        plain = _resp_result("item-plain", "Q1", "golden-1")  # no retrieved_chunks
+        _seed_chunk(db=db, eval_run=eval_run, results=[plain], store=_s3_store)
+
+        def _judge(params):
+            if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]:
+                return _raw_judge_response(
+                    json.dumps(
+                        {
+                            "ground_truth": {"score": 0.8, "reasoning": "gt"},
+                            "knowledge_base": {"score": 0.6, "reasoning": "kb"},
+                        }
+                    )
+                )
+            return _judge_response(0.8, "gt only")
+
+        result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge)
+
+        kb = _score_named(
+            _trace_by_ref(result)["item-plain"], KNOWLEDGE_BASE_SCORE_NAME
+        )
+        assert kb["value"] == "N/A"
+        assert "verdict" not in kb
+
+
 class TestFormatTopKbMatches:
     """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments."""
 

backend/app/tests/crud/evaluations/test_score.py

@@ -0,0 +1,42 @@
+"""`verdict_from_score` — the per-metric verdict band on v2 judge runs.
+
+Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs
+Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the
+display string is part of the contract.
+"""
+
+import pytest
+
+from app.crud.evaluations.score import VerdictEnum, verdict_from_score
+
+
+class TestVerdictFromScore:
+    @pytest.mark.parametrize(
+        ("score", "expected"),
+        [
+            (0.0, VerdictEnum.NEEDS_IMPROVEMENT),
+            (0.29, VerdictEnum.NEEDS_IMPROVEMENT),
+            (0.3, VerdictEnum.NEEDS_REFINEMENT),
+            (0.59, VerdictEnum.NEEDS_REFINEMENT),
+            (0.6, VerdictEnum.GOOD),
+            (1.0, VerdictEnum.GOOD),
+        ],
+    )
+    def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None:
+        assert verdict_from_score(score) is expected
+
+    @pytest.mark.parametrize(
+        ("member", "display"),
+        [
+            (VerdictEnum.NEEDS_IMPROVEMENT, "Needs Improvement"),
+            (VerdictEnum.NEEDS_REFINEMENT, "Needs Refinement"),
+            (VerdictEnum.GOOD, "Good"),
+        ],
+    )
+    def test_display_string_is_the_serialized_value(
+        self, member: VerdictEnum, display: str
+    ) -> None:
+        assert member.value == display
+
+    def test_returns_verdict_enum_instance(self) -> None:
+        assert isinstance(verdict_from_score(0.5), VerdictEnum)

docs/wiki/modules/evaluations.md

@@ -22,7 +22,7 @@ All paths relative to `backend/app/`.
 | `batch_job` (BatchJob) | `models/batch_job.py` |
 
 Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`.
-v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config.
+v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none.
 
 ## Services / CRUD
 - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py`