From 46a3037dc2a8fa8d10a2beb5a05f74f46a3c0cb4 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:21:37 +0530 Subject: [PATCH 1/3] feat(evals): added the verdict corresponding to judge metric in each row --- backend/app/crud/evaluations/fast.py | 5 +- backend/app/crud/evaluations/score.py | 29 +++++ .../tests/crud/evaluations/test_fast_judge.py | 105 ++++++++++++++++++ .../app/tests/crud/evaluations/test_score.py | 42 +++++++ docs/wiki/modules/evaluations.md | 2 +- 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 backend/app/tests/crud/evaluations/test_score.py diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index bbfe16912..de39f4a20 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -83,6 +83,7 @@ EvaluationScore, TraceData, TraceScore, + verdict_from_score, ) from app.crud.job import ( create_batch_job, @@ -1171,12 +1172,14 @@ def _stage3_score_and_trace( comment = metric_score.reasoning if is_kb: comment = f"{comment} | Top matches: {top_matches}" + rounded_score = round(metric_score.score, 2) trace_scores.append( { "name": spec.score_name, - "value": round(metric_score.score, 2), + "value": rounded_score, "data_type": "NUMERIC", "comment": comment, + "verdict": verdict_from_score(rounded_score), } ) elif is_kb: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 93dbc70a2..df012b29e 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -5,10 +5,36 @@ used throughout the evaluation system. """ +from enum import Enum from typing import NotRequired, TypedDict DEFAULT_CATEGORY: str = "Other" + +class VerdictEnum(str, Enum): + """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + + NEEDS_IMPROVEMENT = "Needs Improvement" + NEEDS_REFINEMENT = "Needs Refinement" + GOOD = "Good" + + +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 +VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 + + +def verdict_from_score(score: float) -> VerdictEnum: + """Map a 0–1 judge-metric score to its verdict band. + + Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + """ + if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: + return VerdictEnum.NEEDS_IMPROVEMENT + if score < VERDICT_GOOD_AT_OR_ABOVE: + return VerdictEnum.NEEDS_REFINEMENT + return VerdictEnum.GOOD + + # Canonical name/comment for the cosine-similarity score, centralized to avoid # import cycles. COSINE_SCORE_NAME: str = "Cosine Similarity" @@ -152,6 +178,9 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] + # Verdict band; present only on numeric judge-metric scores (v2 runs), never + # on cosine or unscoreable/"N/A" entries. + verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 81ee818b2..ff9b20aac 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -43,6 +43,7 @@ JUDGE_FAILED_REASON, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, + verdict_from_score, ) from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob @@ -615,11 +616,13 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) + # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, "data_type": "NUMERIC", "comment": "answered in English", + "verdict": "Needs Improvement", } assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 @@ -1082,6 +1085,108 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None +class TestVerdictBandOnTraceScores: + """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + the cosine (v1) score and the KB N/A placeholder never do.""" + + def test_each_scored_judge_metric_carries_its_verdict_band( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, + # 0.75 → Good, so the three metrics land in three different bands. + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.2, "reasoning": "gt"}, + "prompt": {"score": 0.45, "reasoning": "p"}, + "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + } + ) + ), + ) + + trace = _trace_by_ref(result)["item-1"] + gt = _score_named(trace, GROUND_TRUTH_SCORE_NAME) + prompt = _score_named(trace, PROMPT_SCORE_NAME) + kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) + + # Expected bands are the #1092 spec, not a re-run of the mapper. + assert gt["verdict"] == "Needs Improvement" + assert prompt["verdict"] == "Needs Refinement" + assert kb["verdict"] == "Good" + + for score in (gt, prompt, kb): + assert score["verdict"] == verdict_from_score(score["value"]) + + def test_cosine_score_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) + assert cosine is not None + assert "verdict" not in cosine + + def test_kb_na_placeholder_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + plain = _resp_result("item-plain", "Q1", "golden-1") # no retrieved_chunks + _seed_chunk(db=db, eval_run=eval_run, results=[plain], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + return _judge_response(0.8, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named( + _trace_by_ref(result)["item-plain"], KNOWLEDGE_BASE_SCORE_NAME + ) + assert kb["value"] == "N/A" + assert "verdict" not in kb + + class TestFormatTopKbMatches: """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py new file mode 100644 index 000000000..dcf140c24 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -0,0 +1,42 @@ +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). + +Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs +Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the +display string is part of the contract. +""" + +import pytest + +from app.crud.evaluations.score import VerdictEnum, verdict_from_score + + +class TestVerdictFromScore: + @pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, VerdictEnum.NEEDS_IMPROVEMENT), + (0.29, VerdictEnum.NEEDS_IMPROVEMENT), + (0.3, VerdictEnum.NEEDS_REFINEMENT), + (0.59, VerdictEnum.NEEDS_REFINEMENT), + (0.6, VerdictEnum.GOOD), + (1.0, VerdictEnum.GOOD), + ], + ) + def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: + assert verdict_from_score(score) is expected + + @pytest.mark.parametrize( + ("member", "display"), + [ + (VerdictEnum.NEEDS_IMPROVEMENT, "Needs Improvement"), + (VerdictEnum.NEEDS_REFINEMENT, "Needs Refinement"), + (VerdictEnum.GOOD, "Good"), + ], + ) + def test_display_string_is_the_serialized_value( + self, member: VerdictEnum, display: str + ) -> None: + assert member.value == display + + def test_returns_verdict_enum_instance(self) -> None: + assert isinstance(verdict_from_score(0.5), VerdictEnum) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index cd0efcd3d..c0119546d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py` From 6cd310be49486551a523e879d38c4f8f06d3de9f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:23:35 +0530 Subject: [PATCH 2/3] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 +- backend/app/tests/crud/evaluations/test_fast_judge.py | 4 +--- backend/app/tests/crud/evaluations/test_score.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index df012b29e..dd0ec1ff7 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -12,7 +12,7 @@ class VerdictEnum(str, Enum): - """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + """Qualitative band derived from a 0–1 judge-metric score.""" NEEDS_IMPROVEMENT = "Needs Improvement" NEEDS_REFINEMENT = "Needs Refinement" diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index ff9b20aac..636d8c635 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -616,7 +616,6 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) - # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, @@ -1086,7 +1085,7 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( class TestVerdictBandOnTraceScores: - """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + """each scored judge metric on a v2 trace carries a verdict band; the cosine (v1) score and the KB N/A placeholder never do.""" def test_each_scored_judge_metric_carries_its_verdict_band( @@ -1125,7 +1124,6 @@ def test_each_scored_judge_metric_carries_its_verdict_band( prompt = _score_named(trace, PROMPT_SCORE_NAME) kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) - # Expected bands are the #1092 spec, not a re-run of the mapper. assert gt["verdict"] == "Needs Improvement" assert prompt["verdict"] == "Needs Refinement" assert kb["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py index dcf140c24..f1d2aa03e 100644 --- a/backend/app/tests/crud/evaluations/test_score.py +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -1,4 +1,4 @@ -"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs. Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the From 5ff0202d460904e6ea20e94854758d0d1fba377b Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:35:33 +0530 Subject: [PATCH 3/3] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index dd0ec1ff7..2ea27f567 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -178,8 +178,6 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] - # Verdict band; present only on numeric judge-metric scores (v2 runs), never - # on cosine or unscoreable/"N/A" entries. verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool]