diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index bbfe16912..de39f4a20 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -83,6 +83,7 @@ EvaluationScore, TraceData, TraceScore, + verdict_from_score, ) from app.crud.job import ( create_batch_job, @@ -1171,12 +1172,14 @@ def _stage3_score_and_trace( comment = metric_score.reasoning if is_kb: comment = f"{comment} | Top matches: {top_matches}" + rounded_score = round(metric_score.score, 2) trace_scores.append( { "name": spec.score_name, - "value": round(metric_score.score, 2), + "value": rounded_score, "data_type": "NUMERIC", "comment": comment, + "verdict": verdict_from_score(rounded_score), } ) elif is_kb: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 93dbc70a2..2ea27f567 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -5,10 +5,36 @@ used throughout the evaluation system. """ +from enum import Enum from typing import NotRequired, TypedDict DEFAULT_CATEGORY: str = "Other" + +class VerdictEnum(str, Enum): + """Qualitative band derived from a 0–1 judge-metric score.""" + + NEEDS_IMPROVEMENT = "Needs Improvement" + NEEDS_REFINEMENT = "Needs Refinement" + GOOD = "Good" + + +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 +VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 + + +def verdict_from_score(score: float) -> VerdictEnum: + """Map a 0–1 judge-metric score to its verdict band. + + Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + """ + if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: + return VerdictEnum.NEEDS_IMPROVEMENT + if score < VERDICT_GOOD_AT_OR_ABOVE: + return VerdictEnum.NEEDS_REFINEMENT + return VerdictEnum.GOOD + + # Canonical name/comment for the cosine-similarity score, centralized to avoid # import cycles. COSINE_SCORE_NAME: str = "Cosine Similarity" @@ -152,6 +178,7 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] + verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 81ee818b2..636d8c635 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -43,6 +43,7 @@ JUDGE_FAILED_REASON, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, + verdict_from_score, ) from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob @@ -620,6 +621,7 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( "value": 0.25, "data_type": "NUMERIC", "comment": "answered in English", + "verdict": "Needs Improvement", } assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 @@ -1082,6 +1084,107 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None +class TestVerdictBandOnTraceScores: + """each scored judge metric on a v2 trace carries a verdict band; + the cosine (v1) score and the KB N/A placeholder never do.""" + + def test_each_scored_judge_metric_carries_its_verdict_band( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, + # 0.75 → Good, so the three metrics land in three different bands. + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.2, "reasoning": "gt"}, + "prompt": {"score": 0.45, "reasoning": "p"}, + "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + } + ) + ), + ) + + trace = _trace_by_ref(result)["item-1"] + gt = _score_named(trace, GROUND_TRUTH_SCORE_NAME) + prompt = _score_named(trace, PROMPT_SCORE_NAME) + kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) + + assert gt["verdict"] == "Needs Improvement" + assert prompt["verdict"] == "Needs Refinement" + assert kb["verdict"] == "Good" + + for score in (gt, prompt, kb): + assert score["verdict"] == verdict_from_score(score["value"]) + + def test_cosine_score_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) + assert cosine is not None + assert "verdict" not in cosine + + def test_kb_na_placeholder_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + plain = _resp_result("item-plain", "Q1", "golden-1") # no retrieved_chunks + _seed_chunk(db=db, eval_run=eval_run, results=[plain], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + return _judge_response(0.8, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named( + _trace_by_ref(result)["item-plain"], KNOWLEDGE_BASE_SCORE_NAME + ) + assert kb["value"] == "N/A" + assert "verdict" not in kb + + class TestFormatTopKbMatches: """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py new file mode 100644 index 000000000..f1d2aa03e --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -0,0 +1,42 @@ +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs. + +Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs +Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the +display string is part of the contract. +""" + +import pytest + +from app.crud.evaluations.score import VerdictEnum, verdict_from_score + + +class TestVerdictFromScore: + @pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, VerdictEnum.NEEDS_IMPROVEMENT), + (0.29, VerdictEnum.NEEDS_IMPROVEMENT), + (0.3, VerdictEnum.NEEDS_REFINEMENT), + (0.59, VerdictEnum.NEEDS_REFINEMENT), + (0.6, VerdictEnum.GOOD), + (1.0, VerdictEnum.GOOD), + ], + ) + def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: + assert verdict_from_score(score) is expected + + @pytest.mark.parametrize( + ("member", "display"), + [ + (VerdictEnum.NEEDS_IMPROVEMENT, "Needs Improvement"), + (VerdictEnum.NEEDS_REFINEMENT, "Needs Refinement"), + (VerdictEnum.GOOD, "Good"), + ], + ) + def test_display_string_is_the_serialized_value( + self, member: VerdictEnum, display: str + ) -> None: + assert member.value == display + + def test_returns_verdict_enum_instance(self) -> None: + assert isinstance(verdict_from_score(0.5), VerdictEnum) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index cd0efcd3d..c0119546d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py`