From 46a3037dc2a8fa8d10a2beb5a05f74f46a3c0cb4 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:21:37 +0530 Subject: [PATCH 01/15] feat(evals): added the verdict corresponding to judge metric in each row --- backend/app/crud/evaluations/fast.py | 5 +- backend/app/crud/evaluations/score.py | 29 +++++ .../tests/crud/evaluations/test_fast_judge.py | 105 ++++++++++++++++++ .../app/tests/crud/evaluations/test_score.py | 42 +++++++ docs/wiki/modules/evaluations.md | 2 +- 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 backend/app/tests/crud/evaluations/test_score.py diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index bbfe16912..de39f4a20 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -83,6 +83,7 @@ EvaluationScore, TraceData, TraceScore, + verdict_from_score, ) from app.crud.job import ( create_batch_job, @@ -1171,12 +1172,14 @@ def _stage3_score_and_trace( comment = metric_score.reasoning if is_kb: comment = f"{comment} | Top matches: {top_matches}" + rounded_score = round(metric_score.score, 2) trace_scores.append( { "name": spec.score_name, - "value": round(metric_score.score, 2), + "value": rounded_score, "data_type": "NUMERIC", "comment": comment, + "verdict": verdict_from_score(rounded_score), } ) elif is_kb: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 93dbc70a2..df012b29e 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -5,10 +5,36 @@ used throughout the evaluation system. """ +from enum import Enum from typing import NotRequired, TypedDict DEFAULT_CATEGORY: str = "Other" + +class VerdictEnum(str, Enum): + """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + + NEEDS_IMPROVEMENT = "Needs Improvement" + NEEDS_REFINEMENT = "Needs Refinement" + GOOD = "Good" + + +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 +VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 + + +def verdict_from_score(score: float) -> VerdictEnum: + """Map a 0–1 judge-metric score to its verdict band. + + Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + """ + if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: + return VerdictEnum.NEEDS_IMPROVEMENT + if score < VERDICT_GOOD_AT_OR_ABOVE: + return VerdictEnum.NEEDS_REFINEMENT + return VerdictEnum.GOOD + + # Canonical name/comment for the cosine-similarity score, centralized to avoid # import cycles. COSINE_SCORE_NAME: str = "Cosine Similarity" @@ -152,6 +178,9 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] + # Verdict band; present only on numeric judge-metric scores (v2 runs), never + # on cosine or unscoreable/"N/A" entries. + verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 81ee818b2..ff9b20aac 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -43,6 +43,7 @@ JUDGE_FAILED_REASON, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, + verdict_from_score, ) from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob @@ -615,11 +616,13 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) + # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, "data_type": "NUMERIC", "comment": "answered in English", + "verdict": "Needs Improvement", } assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 @@ -1082,6 +1085,108 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None +class TestVerdictBandOnTraceScores: + """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + the cosine (v1) score and the KB N/A placeholder never do.""" + + def test_each_scored_judge_metric_carries_its_verdict_band( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, + # 0.75 → Good, so the three metrics land in three different bands. + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.2, "reasoning": "gt"}, + "prompt": {"score": 0.45, "reasoning": "p"}, + "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + } + ) + ), + ) + + trace = _trace_by_ref(result)["item-1"] + gt = _score_named(trace, GROUND_TRUTH_SCORE_NAME) + prompt = _score_named(trace, PROMPT_SCORE_NAME) + kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) + + # Expected bands are the #1092 spec, not a re-run of the mapper. + assert gt["verdict"] == "Needs Improvement" + assert prompt["verdict"] == "Needs Refinement" + assert kb["verdict"] == "Good" + + for score in (gt, prompt, kb): + assert score["verdict"] == verdict_from_score(score["value"]) + + def test_cosine_score_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) + assert cosine is not None + assert "verdict" not in cosine + + def test_kb_na_placeholder_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + plain = _resp_result("item-plain", "Q1", "golden-1") # no retrieved_chunks + _seed_chunk(db=db, eval_run=eval_run, results=[plain], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + return _judge_response(0.8, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named( + _trace_by_ref(result)["item-plain"], KNOWLEDGE_BASE_SCORE_NAME + ) + assert kb["value"] == "N/A" + assert "verdict" not in kb + + class TestFormatTopKbMatches: """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py new file mode 100644 index 000000000..dcf140c24 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -0,0 +1,42 @@ +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). + +Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs +Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the +display string is part of the contract. +""" + +import pytest + +from app.crud.evaluations.score import VerdictEnum, verdict_from_score + + +class TestVerdictFromScore: + @pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, VerdictEnum.NEEDS_IMPROVEMENT), + (0.29, VerdictEnum.NEEDS_IMPROVEMENT), + (0.3, VerdictEnum.NEEDS_REFINEMENT), + (0.59, VerdictEnum.NEEDS_REFINEMENT), + (0.6, VerdictEnum.GOOD), + (1.0, VerdictEnum.GOOD), + ], + ) + def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: + assert verdict_from_score(score) is expected + + @pytest.mark.parametrize( + ("member", "display"), + [ + (VerdictEnum.NEEDS_IMPROVEMENT, "Needs Improvement"), + (VerdictEnum.NEEDS_REFINEMENT, "Needs Refinement"), + (VerdictEnum.GOOD, "Good"), + ], + ) + def test_display_string_is_the_serialized_value( + self, member: VerdictEnum, display: str + ) -> None: + assert member.value == display + + def test_returns_verdict_enum_instance(self) -> None: + assert isinstance(verdict_from_score(0.5), VerdictEnum) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index cd0efcd3d..c0119546d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py` From 6cd310be49486551a523e879d38c4f8f06d3de9f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:23:35 +0530 Subject: [PATCH 02/15] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 +- backend/app/tests/crud/evaluations/test_fast_judge.py | 4 +--- backend/app/tests/crud/evaluations/test_score.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index df012b29e..dd0ec1ff7 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -12,7 +12,7 @@ class VerdictEnum(str, Enum): - """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + """Qualitative band derived from a 0–1 judge-metric score.""" NEEDS_IMPROVEMENT = "Needs Improvement" NEEDS_REFINEMENT = "Needs Refinement" diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index ff9b20aac..636d8c635 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -616,7 +616,6 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) - # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, @@ -1086,7 +1085,7 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( class TestVerdictBandOnTraceScores: - """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + """each scored judge metric on a v2 trace carries a verdict band; the cosine (v1) score and the KB N/A placeholder never do.""" def test_each_scored_judge_metric_carries_its_verdict_band( @@ -1125,7 +1124,6 @@ def test_each_scored_judge_metric_carries_its_verdict_band( prompt = _score_named(trace, PROMPT_SCORE_NAME) kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) - # Expected bands are the #1092 spec, not a re-run of the mapper. assert gt["verdict"] == "Needs Improvement" assert prompt["verdict"] == "Needs Refinement" assert kb["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py index dcf140c24..f1d2aa03e 100644 --- a/backend/app/tests/crud/evaluations/test_score.py +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -1,4 +1,4 @@ -"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs. Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the From 5ff0202d460904e6ea20e94854758d0d1fba377b Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:35:33 +0530 Subject: [PATCH 03/15] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index dd0ec1ff7..2ea27f567 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -178,8 +178,6 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] - # Verdict band; present only on numeric judge-metric scores (v2 runs), never - # on cosine or unscoreable/"N/A" entries. verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] From 194c8f58e2be1b9cc17b030d9960b481a913eca8 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:33:43 +0530 Subject: [PATCH 04/15] feat(*): implement the evals human readable summary generation --- backend/app/core/config.py | 2 + backend/app/crud/evaluations/core.py | 2 + backend/app/crud/evaluations/fast.py | 62 +++++- backend/app/crud/evaluations/judge.py | 4 + backend/app/crud/evaluations/score.py | 74 +++++++ backend/app/crud/evaluations/summary.py | 189 ++++++++++++++++++ .../app/services/evaluations/evaluation.py | 7 + .../tests/crud/evaluations/test_fast_judge.py | 171 +++++++++++++++- .../crud/evaluations/test_overall_summary.py | 127 ++++++++++++ .../crud/evaluations/test_run_ai_summary.py | 179 +++++++++++++++++ .../evaluations/test_evaluation_service_s3.py | 115 +++++++++++ 11 files changed, 930 insertions(+), 2 deletions(-) create mode 100644 backend/app/crud/evaluations/summary.py create mode 100644 backend/app/tests/crud/evaluations/test_overall_summary.py create mode 100644 backend/app/tests/crud/evaluations/test_run_ai_summary.py diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8ebc37b53..337f2bd4f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -216,6 +216,8 @@ def AWS_S3_BUCKET(self) -> str: # One of: none | minimal | low | medium | high | xhigh. EVAL_JUDGE_REASONING_EFFORT: str = "medium" + EVAL_SUMMARY_MODEL: str = "gpt-5-mini" + # Judge runs alone in the single aggregate task (no response calls competing), # so it uses a larger pool than the response stage to finish the max dataset # (EVAL_FAST_MAX_UNIQUE_ROWS x duplication) well under CELERY_TASK_SOFT_TIME_LIMIT. diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index ed15f720e..9a2a0e5e9 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -485,6 +485,8 @@ def save_score( # TODO: Evaluate whether this behaviour is needed or completely discard the storing data in db if score_trace_url: db_score = {"summary_scores": summary_score} + if score.get("overall") is not None: + db_score["overall"] = score["overall"] else: # fallback to store data in db if failed to store in s3 db_score = score diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index de39f4a20..22da3c23b 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -49,6 +49,10 @@ update_evaluation_run, ) from app.crud.evaluations.cost import attach_cost +from app.crud.evaluations.dataset import ( + DATASET_META_DUPLICATION_FACTOR, + get_dataset_by_id, +) from app.crud.evaluations.embeddings import ( EMBEDDING_MODEL, calculate_cosine_similarity, @@ -81,10 +85,13 @@ UNSCOREABLE_EMPTY_GROUND_TRUTH, UNSCOREABLE_EMPTY_OUTPUT, EvaluationScore, + OverallSummary, TraceData, TraceScore, + compute_overall_summary, verdict_from_score, ) +from app.crud.evaluations.summary import generate_run_ai_summary from app.crud.job import ( create_batch_job, delete_batch_job, @@ -900,6 +907,24 @@ def _attach_metric_scores( ) +def _resolve_duplication_factor(*, session: Session, eval_run: EvaluationRun) -> int: + """How many times each question was asked, from the run's dataset metadata. + + Feeds only the best-effort AI summary, so it never fails the run: an + unresolvable dataset or missing metadata falls back to 1 (no repetition). + """ + dataset = get_dataset_by_id( + session=session, + dataset_id=eval_run.dataset_id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + if dataset is None: + return 1 + metadata = dataset.dataset_metadata or {} + return max(1, int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1))) + + def _stage3_score_and_trace( *, session: Session, @@ -953,6 +978,7 @@ def _stage3_score_and_trace( unscoreable: dict[str, str] = {} # {ref: reason} write_items: list[dict[str, Any]] = [] summary_scores: list[dict[str, Any]] = [] + overall: OverallSummary | None = None if is_judge_run: # v2: no cosine. A row is judgeable only with a non-empty generated AND @@ -1109,6 +1135,33 @@ def _stage3_score_and_trace( summary_scores=summary_scores, ) + avg_by_name = {s["name"]: s["avg"] for s in summary_scores if "avg" in s} + metric_avgs = { + spec.key.value: avg_by_name[spec.score_name] + for spec in metrics + if spec.score_name in avg_by_name + } + overall = compute_overall_summary( + metric_avgs=metric_avgs, + metric_weights={spec.key.value: spec.weight for spec in metrics}, + metric_names={spec.key.value: spec.score_name for spec in metrics}, + ) + if overall is not None: + # Falls back to 1 (no repetition) if the dataset can't be resolved, so + # the summary still generates. + duplication_factor = _resolve_duplication_factor( + session=session, eval_run=eval_run + ) + overall["ai_summary"] = generate_run_ai_summary( + session=session, + openai_client=openai_client, + model=settings.EVAL_SUMMARY_MODEL, + overall=overall, + run_name=eval_run.run_name, + summary_scores=summary_scores, + duplication_factor=duplication_factor, + ) + # One combined call grades every metric, so its tokens can't be split per # metric — they land in a single "judge" cost stage. if judge_results and judge_model: @@ -1230,6 +1283,8 @@ def _stage3_score_and_trace( "summary_scores": summary_scores, "traces": traces, } + if overall is not None: + score["overall"] = overall return eval_run, score, write_items @@ -1312,7 +1367,12 @@ def run_fast_evaluation( eval_run=eval_run, update=EvaluationRunUpdate( status="completed", - score={"summary_scores": score["summary_scores"]}, + # Persist the overall alongside the summary so GET run status shows the + # run-level score/verdict/breakdown without loading the S3 trace unit. + score={ + "summary_scores": score["summary_scores"], + "overall": score.get("overall"), + }, cost=eval_run.cost, ), ) diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index 30db8e8eb..c228f447b 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -82,6 +82,7 @@ class JudgeMetricSpec: score_name: str prompt_fragment: str required_inputs: tuple[JudgeInputEnum, ...] + weight: float # All metrics are graded by one combined call, so they share a single judge model @@ -97,6 +98,7 @@ class JudgeMetricSpec: JudgeInputEnum.GENERATED_ANSWER, JudgeInputEnum.GOLDEN_ANSWER, ), + weight=0.5, ), JudgeMetricEnum.PROMPT: JudgeMetricSpec( key=JudgeMetricEnum.PROMPT, @@ -107,6 +109,7 @@ class JudgeMetricSpec: JudgeInputEnum.QUESTION, JudgeInputEnum.GENERATED_ANSWER, ), + weight=0.2, ), JudgeMetricEnum.KNOWLEDGE_BASE: JudgeMetricSpec( key=JudgeMetricEnum.KNOWLEDGE_BASE, @@ -117,6 +120,7 @@ class JudgeMetricSpec: JudgeInputEnum.GENERATED_ANSWER, JudgeInputEnum.RETRIEVED_CHUNKS, ), + weight=0.3, ), } diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 2ea27f567..43a8af5b3 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -235,8 +235,82 @@ class CategoricalSummaryScore(TypedDict): SummaryScore = NumericSummaryScore | CategoricalSummaryScore +class OverallDimension(TypedDict): + """One judge metric's contribution to the run-level overall score.""" + + name: str + key: str + score: float + weight: float + delta: float + verdict: str + + +class OverallSummary(TypedDict): + """Run-level weighted quality view for a v2 judge run. + + `ai_summary` is filled by a best-effort LLM step after the deterministic + fields; it stays None when no summary was generated. + """ + + overall_score: float + verdict: str + ai_summary: str | None + breakdown: list[OverallDimension] + + +def compute_overall_summary( + *, + metric_avgs: dict[str, float], + metric_weights: dict[str, float], + metric_names: dict[str, str], +) -> OverallSummary | None: + """Weighted run-level overall score + per-dimension breakdown. No LLM. + + All three dicts are keyed by metric key value. Only metrics present in + `metric_avgs` (i.e. that actually scored ≥1 row) count; a metric with no + scoreable rows is dropped and the remaining base weights are renormalized to + sum to 1, so a missing metric never drags the overall down. Returns None when + nothing scored. `ai_summary` is None here — the LLM step fills it later. + """ + scored_keys = [key for key in metric_avgs if key in metric_weights] + weight_total = sum(metric_weights[key] for key in scored_keys) + if not scored_keys or weight_total <= 0: + return None + + renorm_weights = {key: metric_weights[key] / weight_total for key in scored_keys} + # Round the overall once, then reuse it everywhere so the badge and the number + # (and every delta) are computed from the same value and can never disagree. + overall_score = round( + sum(renorm_weights[key] * metric_avgs[key] for key in scored_keys), 2 + ) + verdict = verdict_from_score(overall_score).value + + breakdown: list[OverallDimension] = [] + for key in scored_keys: + avg = round(metric_avgs[key], 2) + breakdown.append( + { + "name": metric_names.get(key, key), + "key": key, + "score": avg, + "weight": round(renorm_weights[key], 2), + "delta": round(avg - overall_score, 2), + "verdict": verdict_from_score(avg).value, + } + ) + + return { + "overall_score": overall_score, + "verdict": verdict, + "ai_summary": None, + "breakdown": breakdown, + } + + class EvaluationScore(TypedDict): """Complete evaluation score data with traces and summary statistics.""" summary_scores: list[SummaryScore] traces: list[TraceData] + overall: NotRequired[OverallSummary] diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py new file mode 100644 index 000000000..faef41e03 --- /dev/null +++ b/backend/app/crud/evaluations/summary.py @@ -0,0 +1,189 @@ +"""Human-readable AI summary of a v2 judge run's overall quality. +""" + +import logging +from typing import Any + +import openai +from openai import OpenAI +from sqlmodel import Session + +from app.core.config import settings +from app.crud.evaluations.judge import JudgeMetricEnum +from app.crud.evaluations.response_parsing import extract_response_text +from app.crud.evaluations.score import OverallSummary +from app.services.llm.mappers import map_kaapi_to_openai_params + +logger = logging.getLogger(__name__) + +# Reasoning tokens count against this cap, so it needs headroom beyond the visible +# note — too low and reasoning consumes it all, yielding empty output. Length is +# bounded by the "1 to 3 sentences" instruction, not by a tight token cap. +_SUMMARY_MAX_OUTPUT_TOKENS: int = 600 + +# Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain +# consistency read. With repeated questions a low spread means the assistant answered +# the same question the same way each time. +_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 +_CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 + +# Internal metric key -> plain behaviour phrase, so the brief never leaks the +# "Adherence to X" labels into the model's context. +_DIMENSION_PLAIN_NAME: dict[str, str] = { + JudgeMetricEnum.GROUND_TRUTH.value: "Accuracy against the expected answers", + JudgeMetricEnum.KNOWLEDGE_BASE.value: "Grounding in the source material", + JudgeMetricEnum.PROMPT.value: "Tone and instruction-following", +} + +_SUMMARY_SYSTEM_PROMPT: str = ( + "You write a short, warm, plain-language note about how an AI assistant did on an " + "evaluation, the way a colleague would summarize at a glance. Write 1 to 3 " + "sentences — no bullet points, no headings. Use no scores or numbers at all; the " + "ONLY number you may state is how many times each question was asked. " + "You are given an overall standing, how each area did (as a band word), and how " + "consistent the answers were. Translate everything into plain behaviour words and " + "NEVER use the internal area labels, raw scores, weights, deltas, or the word " + "'verdict'. Map the areas like this: accuracy against the expected answers -> " + "'accurate' or 'matched the expected answers'; grounding in the source material -> " + "'grounded', 'backed by the source material', or 'not made up'; tone and " + "instruction-following -> 'on-tone' or 'followed the instructions (language, " + "style)'. Lead with how it did overall in that plain language, then note the " + "strongest and any weaker area. When each question was asked more than once and the " + "answers stayed stable, say so (e.g. 'answers stayed consistent'); if they varied, " + "say the answers varied. Do not invent facts beyond what you are given. " + "Style anchor — match this tone and length, do not copy the wording: " + '"Consistently grounded and on-tone across the set. Each question was asked 5 ' + 'times and answers stayed consistent."' +) + + +def _consistency_read(std: float | None) -> str: + """Qualitative stability of a metric's per-row scores, derived from its std.""" + if std is None: + return "consistency unknown" + if std <= _CONSISTENCY_STABLE_AT_OR_BELOW: + return "answers stayed consistent" + if std <= _CONSISTENCY_MIXED_AT_OR_BELOW: + return "answers were mostly consistent, with some variation" + return "answers varied" + + +def _format_overall_for_prompt( + *, + overall: OverallSummary, + run_name: str, + summary_scores: list[dict[str, Any]], + duplication_factor: int, +) -> str: + """Compact qualitative brief for the summary model — bands, not raw scores. + + Hands over the overall band, a per-area band + consistency read (from each + metric's std), and the repetition factor. No numbers cross except the repeat + count; the instructions do the plain-language translation. + """ + std_by_name = { + score["name"]: score.get("std") + for score in summary_scores + if isinstance(score, dict) and "name" in score + } + + lines = [ + f"Run: {run_name}", + f"Overall standing: {overall['verdict']}.", + "How each area did:", + ] + for dim in overall["breakdown"]: + plain_name = _DIMENSION_PLAIN_NAME.get(dim["key"], dim["name"]) + consistency = _consistency_read(std_by_name.get(dim["name"])) + lines.append(f"- {plain_name}: {dim['verdict']}; {consistency}") + + if duplication_factor > 1: + lines.append(f"Each question was asked {duplication_factor} times.") + else: + lines.append("Each question was asked once (no repetition to speak of).") + + return "\n".join(lines) + + +def generate_run_ai_summary( + *, + session: Session, + openai_client: OpenAI, + model: str, + overall: OverallSummary, + run_name: str, + summary_scores: list[dict[str, Any]], + duplication_factor: int, +) -> str | None: + """Best-effort one-shot natural-language note on the run's overall quality. + + Reuses the aggregate's existing OpenAI client (never builds a new one) and the + judge's reasoning-model invocation: `map_kaapi_to_openai_params` suppresses + temperature and sets the reasoning effort, then the body goes to the Responses + API. `summary_scores` supplies each metric's std (consistency) and + `duplication_factor` the repeat count that the brief translates into plain + language. A summary is a nicety, so ANY failure — missing key, API error, empty + output — logs a warning and returns None rather than propagating: the + deterministic overall (score/verdict/breakdown) must still persist. + """ + user_message = _format_overall_for_prompt( + overall=overall, + run_name=run_name, + summary_scores=summary_scores, + duplication_factor=duplication_factor, + ) + try: + base_params, mapper_warnings = map_kaapi_to_openai_params( + session=session, + kaapi_params={ + "model": model, + "effort": settings.EVAL_JUDGE_REASONING_EFFORT, + }, + ) + if mapper_warnings: + logger.warning( + f"[generate_run_ai_summary] Mapper warnings: {mapper_warnings}" + ) + params = { + **base_params, + "instructions": _SUMMARY_SYSTEM_PROMPT, + "input": user_message, + "max_output_tokens": _SUMMARY_MAX_OUTPUT_TOKENS, + } + response = openai_client.responses.create(**params) + except openai.OpenAIError as exc: + status = getattr(exc, "status_code", None) + # 5xx is provider-side (alert-worthy); 4xx/None is caller/Kaapi-side noise. + log = logger.error if (status and status >= 500) else logger.warning + tag = "[OPENAI]" if status else "[KAAPI]" + request_id = getattr(exc, "request_id", None) + log( + f"[generate_run_ai_summary] {tag} Summary completion failed " + f"(code: {status or type(exc).__name__}) | model={model} | " + f"run_name={run_name} | request_id={request_id} | {exc}", + exc_info=True, + ) + return None + # Deliberately broad: a summary failure (mapper/config error, unexpected shape) + # must never fail the run, so it degrades to a None result. + except Exception as exc: + logger.warning( + f"[generate_run_ai_summary] Summary call failed; leaving ai_summary " + f"empty | model={model} | run_name={run_name} | error={exc}", + exc_info=True, + ) + return None + + summary = extract_response_text(response).strip() + if not summary: + logger.warning( + f"[generate_run_ai_summary] Empty summary returned | model={model} | " + f"run_name={run_name}" + ) + return None + + logger.info( + f"[generate_run_ai_summary] Generated run summary | model={model} | " + f"run_name={run_name} | chars={len(summary)}" + ) + return summary diff --git a/backend/app/services/evaluations/evaluation.py b/backend/app/services/evaluations/evaluation.py index c01bfbeb9..5f2f90bb2 100644 --- a/backend/app/services/evaluations/evaluation.py +++ b/backend/app/services/evaluations/evaluation.py @@ -453,6 +453,8 @@ def get_evaluation_with_scores( if not get_trace_info: return eval_run, None + run_overall = (eval_run.score or {}).get("overall") + # Caching strategy: trace scores are fetched from Langfuse once, then cached # (traces in S3, summary in the DB). Normal reads serve from that cache, which is # much faster than hitting Langfuse. resync_score=true bypasses the cache. @@ -485,6 +487,8 @@ def get_evaluation_with_scores( unscoreable=eval_run.unscoreable, ) eval_run.score = _attach_category_metrics(cached_score) + if run_overall is not None: + eval_run.score["overall"] = run_overall logger.info( f"[get_evaluation_with_scores] Served traces from cache | " f"evaluation_id={evaluation_id} | traces_count={len(cached_traces)}" @@ -563,6 +567,9 @@ def get_evaluation_with_scores( # `_attach_category_metrics` mutates in place and is idempotent. _attach_category_metrics(merged_score) + if run_overall is not None: + merged_score["overall"] = run_overall + logger.info( f"[get_evaluation_with_scores] Merged traces step-forward | " f"evaluation_id={evaluation_id} | cached={len(cached_traces)} | " diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 636d8c635..fb26aea20 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -34,6 +34,7 @@ JOB_TYPE_EVALUATION_FAST_CHUNK, PROMPT_TEMPLATE_LABEL, _format_top_kb_matches, + _resolve_duplication_factor, _responses_call_for_item, run_fast_evaluation, run_response_chunk, @@ -192,6 +193,17 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)): ) +# Default plain-text stub for the best-effort run-level AI summary. It rides a +# separate `responses.create` call (the judge is patched at _create_judge_response, +# so it never reaches this mock). A bare MagicMock output would poison the score +# JSONB, so every judged run's summary boundary is stubbed with a real string. +DEFAULT_RUN_SUMMARY = "Overall the run performed reasonably; strongest on ground truth." + + +def _summary_response(text: str): + return SimpleNamespace(output_text=text, output=[]) + + @pytest.fixture def _s3_store() -> Iterator[dict[str, list[dict[str, Any]]]]: store: dict[str, list[dict[str, Any]]] = {} @@ -240,7 +252,12 @@ def _seed_chunk( def _persist_score_into(db: Session): def _fake_save_score(*, eval_run_id, score, **_): run = db.get(EvaluationRun, eval_run_id) - run.score = {"summary_scores": score["summary_scores"]} + # Mirror save_score's S3 path: traces go to S3, the DB score keeps the + # summary plus the run-level overall (when present). + db_score: dict[str, Any] = {"summary_scores": score["summary_scores"]} + if score.get("overall") is not None: + db_score["overall"] = score["overall"] + run.score = db_score run.score_trace_url = f"s3://bucket/traces_{eval_run_id}.json" db.add(run) db.commit() @@ -256,6 +273,7 @@ def _run_pipeline( eval_run: EvaluationRun, judge_side_effect, mock_cost: bool = False, + summary_side_effect=None, ) -> tuple[EvaluationRun, MagicMock]: """Run `run_fast_evaluation` for a judged run with all externals stubbed. @@ -263,9 +281,19 @@ def _run_pipeline( key by item_id. The judge completion is driven by `judge_side_effect(params)`. Returns the run plus the OpenAI mock so callers can assert the embedding path was (v1) or was not (v2 judge) exercised. + + The run-level AI summary is a separate `responses.create` call; by default it + returns `DEFAULT_RUN_SUMMARY`. Pass `summary_side_effect` (e.g. an exception) to + drive the best-effort failure path. """ fake_openai = MagicMock() fake_openai.embeddings.create.return_value = _fake_embedding_response() + if summary_side_effect is not None: + fake_openai.responses.create.side_effect = summary_side_effect + else: + fake_openai.responses.create.return_value = _summary_response( + DEFAULT_RUN_SUMMARY + ) def _judge(_client, params): return judge_side_effect(params) @@ -835,6 +863,147 @@ def _judge(params): assert COSINE_SCORE_NAME in summary_names +class TestRunOverallSummary: + """The run-level weighted overall rides the v2 judge run's score.""" + + def _seed_all_three_metrics_run( + self, *, db: Session, user_api_key: TestAuthContext, store + ) -> EvaluationRun: + # Instructions enable the prompt metric; retrieved chunks enable KB; so a + # single judged row scores all three metrics. + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting chunk", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=store) + return eval_run + + def _all_three_judge(self, _params): + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "prompt": {"score": 0.4, "reasoning": "p"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + + def test_judged_run_overall_matches_metric_averages( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, eval_run=eval_run, judge_side_effect=self._all_three_judge + ) + + assert result.status == "completed" + overall = result.score["overall"] + # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.66. + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + # The successful summary boundary flows into ai_summary. + assert overall["ai_summary"] == DEFAULT_RUN_SUMMARY + + # Each breakdown dimension mirrors its run-level summary average. + summary_avgs = { + s["name"]: s["avg"] for s in result.score["summary_scores"] if "avg" in s + } + breakdown_by_name = {dim["name"]: dim for dim in overall["breakdown"]} + assert set(breakdown_by_name) == set(summary_avgs) + for name, avg in summary_avgs.items(): + assert breakdown_by_name[name]["score"] == round(avg, 2) + + def test_summary_failure_leaves_overall_intact_with_null_ai_summary( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=self._all_three_judge, + summary_side_effect=RuntimeError("summary provider down"), + ) + + # The run still completes and the deterministic overall survives whole — + # only the best-effort ai_summary is lost. + assert result.status == "completed" + overall = result.score["overall"] + assert overall["ai_summary"] is None + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + assert len(overall["breakdown"]) == 3 + + def test_overall_survives_into_the_persisted_db_score( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """Re-fetch the run: save_score's S3 path must keep the overall in the DB + score column, not clobber it with the summary alone.""" + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, eval_run=eval_run, judge_side_effect=self._all_three_judge + ) + + db.expire_all() + persisted = db.get(EvaluationRun, result.id).score["overall"] + assert persisted["overall_score"] == 0.66 + assert persisted["verdict"] == "Good" + assert {dim["key"] for dim in persisted["breakdown"]} == { + "ground_truth", + "prompt", + "knowledge_base", + } + + def test_v1_run_has_no_overall_in_score( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + assert result.status == "completed" + assert "overall" not in result.score + + +class TestResolveDuplicationFactor: + """`_resolve_duplication_factor` feeds only the best-effort summary, so a missing + dataset degrades to 1 (no repetition) rather than failing the run.""" + + def test_falls_back_to_one_when_dataset_missing(self) -> None: + with patch("app.crud.evaluations.fast.get_dataset_by_id", return_value=None): + assert ( + _resolve_duplication_factor(session=MagicMock(), eval_run=MagicMock()) + == 1 + ) + + def _responses_item(item_id: str = "item-1") -> dict[str, Any]: return { "id": item_id, diff --git a/backend/app/tests/crud/evaluations/test_overall_summary.py b/backend/app/tests/crud/evaluations/test_overall_summary.py new file mode 100644 index 000000000..a846f89de --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_overall_summary.py @@ -0,0 +1,127 @@ +"""`compute_overall_summary` — the deterministic run-level weighted overall. + +Weights track the real registry (`{key: spec.weight}`) rather than hard-coded +numbers, so a registry reweighting reshapes these expectations instead of silently +passing. Expected overall scores / deltas are hand-computed from the SRD split +(GT 0.5, KB 0.3, prompt 0.2), independent of the implementation's arithmetic. +""" + +import pytest + +from app.crud.evaluations.judge import METRIC_REGISTRY +from app.crud.evaluations.score import ( + GROUND_TRUTH_SCORE_NAME, + KNOWLEDGE_BASE_SCORE_NAME, + PROMPT_SCORE_NAME, + compute_overall_summary, + verdict_from_score, +) + +METRIC_WEIGHTS = {key.value: spec.weight for key, spec in METRIC_REGISTRY.items()} +METRIC_NAMES = {key.value: spec.score_name for key, spec in METRIC_REGISTRY.items()} + + +def _summary(metric_avgs: dict[str, float]): + return compute_overall_summary( + metric_avgs=metric_avgs, + metric_weights=METRIC_WEIGHTS, + metric_names=METRIC_NAMES, + ) + + +class TestAllThreeMetrics: + def test_weighted_overall_and_per_dimension_breakdown(self) -> None: + # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.40 + 0.18 + 0.08 = 0.66. + result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6, "prompt": 0.4}) + assert result is not None + assert result["overall_score"] == 0.66 + assert result["verdict"] == "Good" + assert result["ai_summary"] is None + + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert by_key["ground_truth"]["name"] == GROUND_TRUTH_SCORE_NAME + assert by_key["knowledge_base"]["name"] == KNOWLEDGE_BASE_SCORE_NAME + assert by_key["prompt"]["name"] == PROMPT_SCORE_NAME + + # All three present → base weights already sum to 1, so no renormalization. + assert by_key["ground_truth"]["weight"] == 0.5 + assert by_key["knowledge_base"]["weight"] == 0.3 + assert by_key["prompt"]["weight"] == 0.2 + assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 + + assert by_key["ground_truth"]["score"] == 0.8 + assert by_key["knowledge_base"]["score"] == 0.6 + assert by_key["prompt"]["score"] == 0.4 + + # delta = dimension score - overall (0.66), sign shows pull vs the run. + assert by_key["ground_truth"]["delta"] == 0.14 + assert by_key["knowledge_base"]["delta"] == -0.06 + assert by_key["prompt"]["delta"] == -0.26 + + assert by_key["ground_truth"]["verdict"] == "Good" + assert by_key["knowledge_base"]["verdict"] == "Good" + assert by_key["prompt"]["verdict"] == "Needs Refinement" + + def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None: + result = _summary({"ground_truth": 0.55, "knowledge_base": 0.25, "prompt": 0.9}) + assert result is not None + for dim in result["breakdown"]: + assert dim["verdict"] == verdict_from_score(dim["score"]).value + + +class TestRenormalizationWhenAMetricIsAbsent: + def test_ground_truth_and_knowledge_base_only(self) -> None: + result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6}) + assert result is not None + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert set(by_key) == {"ground_truth", "knowledge_base"} + + # 0.5 and 0.3 renormalize over 0.8 → 0.625 and 0.375. The stored weights are + # rounded to 2 dp; 0.3/0.8 floats to 0.3749… so it rounds DOWN to 0.37, and + # the displayed pair sums to 0.99 (the un-rounded renorm that drives the + # overall still sums to 1.0). + assert by_key["ground_truth"]["weight"] == 0.62 + assert by_key["knowledge_base"]["weight"] == 0.37 + # 0.625*0.8 + 0.375*0.6 = 0.5 + 0.225 = 0.725 → 0.72. + assert result["overall_score"] == 0.72 + + def test_ground_truth_and_prompt_only(self) -> None: + result = _summary({"ground_truth": 0.8, "prompt": 0.4}) + assert result is not None + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert set(by_key) == {"ground_truth", "prompt"} + + # 0.5 and 0.2 renormalize over 0.7 → 0.714… and 0.285… → 0.71 and 0.29. + assert by_key["ground_truth"]["weight"] == 0.71 + assert by_key["prompt"]["weight"] == 0.29 + assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 + + def test_absent_metric_never_drags_the_overall_down(self) -> None: + # A missing metric is dropped, not scored 0: two perfect metrics stay at 1.0. + result = _summary({"ground_truth": 1.0, "knowledge_base": 1.0}) + assert result is not None + assert result["overall_score"] == 1.0 + + +class TestNothingScored: + def test_empty_metric_avgs_returns_none(self) -> None: + assert _summary({}) is None + + +class TestBadgeBoundaries: + def test_overall_exactly_on_needs_refinement_lower_bound(self) -> None: + # Equal dimension averages → the weighted overall equals that value exactly. + result = _summary({"ground_truth": 0.3, "knowledge_base": 0.3, "prompt": 0.3}) + assert result is not None + assert result["overall_score"] == 0.3 + assert result["verdict"] == "Needs Refinement" + for dim in result["breakdown"]: + assert dim["verdict"] == "Needs Refinement" + + def test_overall_exactly_on_good_lower_bound(self) -> None: + result = _summary({"ground_truth": 0.6, "knowledge_base": 0.6, "prompt": 0.6}) + assert result is not None + assert result["overall_score"] == 0.6 + assert result["verdict"] == "Good" + for dim in result["breakdown"]: + assert dim["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py new file mode 100644 index 000000000..412be8d2e --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -0,0 +1,179 @@ +"""`generate_run_ai_summary` — the best-effort natural-language note on a run. + +The summary reuses the judge's reasoning-model invocation: params built via +`map_kaapi_to_openai_params` (mocked here to skip its DB lookup) and one +`responses.create` call (the external boundary, also mocked). The Responses `input` +is a qualitative brief (band words + plain area names + consistency phrases + a +repetition line) — never raw scores or the internal "Adherence to X" labels. Every +failure mode (OpenAI error, generic error, empty output) must resolve to None +WITHOUT raising, leaving the deterministic overall to persist. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import openai +import pytest + +from app.crud.evaluations.score import OverallSummary +from app.crud.evaluations.summary import _consistency_read, generate_run_ai_summary + +_MODEL = "gpt-5-mini" + + +def _overall() -> OverallSummary: + return { + "overall_score": 0.66, + "verdict": "Good", + "ai_summary": None, + "breakdown": [ + { + "name": "Adherence to Ground Truth", + "key": "ground_truth", + "score": 0.8, + "weight": 0.5, + "delta": 0.14, + "verdict": "Good", + }, + { + "name": "Adherence to Knowledge Base", + "key": "knowledge_base", + "score": 0.6, + "weight": 0.3, + "delta": -0.06, + "verdict": "Good", + }, + { + "name": "Adherence to Prompt", + "key": "prompt", + "score": 0.4, + "weight": 0.2, + "delta": -0.26, + "verdict": "Needs Refinement", + }, + ], + } + + +def _summary_scores() -> list[dict]: + # std per dimension drives the consistency read; names match the overall's dims. + return [ + {"name": "Adherence to Ground Truth", "avg": 0.8, "std": 0.05}, + {"name": "Adherence to Knowledge Base", "avg": 0.6, "std": 0.15}, + {"name": "Adherence to Prompt", "avg": 0.4, "std": 0.34}, + ] + + +def _responses_result(output_text: str): + return SimpleNamespace(output_text=output_text, output=[]) + + +def _call( + client: MagicMock, + *, + duplication_factor: int = 5, + summary_scores: list[dict] | None = None, +) -> str | None: + # The mapper does a real DB lookup (is_reasoning_model); patch it so the unit + # test stays about the summary logic, not model resolution. + with patch( + "app.crud.evaluations.summary.map_kaapi_to_openai_params", + return_value=({"model": _MODEL, "effort": "medium"}, []), + ): + return generate_run_ai_summary( + session=MagicMock(), + openai_client=client, + model=_MODEL, + overall=_overall(), + run_name="run-x", + summary_scores=summary_scores + if summary_scores is not None + else _summary_scores(), + duplication_factor=duplication_factor, + ) + + +class TestHappyPath: + def test_returns_the_responses_output_text_stripped(self) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result( + " Consistently grounded and on-tone across the set. " + ) + assert _call(client) == "Consistently grounded and on-tone across the set." + + +class TestQualitativeBrief: + def _input_for(self, *, duplication_factor: int) -> str: + client = MagicMock() + client.responses.create.return_value = _responses_result("A note.") + _call(client, duplication_factor=duplication_factor) + return client.responses.create.call_args.kwargs["input"] + + def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( + self, + ) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result("A note.") + _call(client, duplication_factor=5) + + params = client.responses.create.call_args.kwargs + brief = params["input"] + assert params["max_output_tokens"] == 600 + assert "temperature" not in params + + assert "Accuracy against the expected answers" in brief + assert "Grounding in the source material" in brief + assert "Tone and instruction-following" in brief + + assert "answers stayed consistent" in brief # std 0.05 + assert "answers were mostly consistent, with some variation" in brief # 0.15 + assert "answers varied" in brief # 0.34 + + def test_brief_leaks_no_raw_scores_or_internal_labels(self) -> None: + brief = self._input_for(duplication_factor=5) + assert "Adherence to" not in brief + assert "verdict" not in brief + # Raw score / weight / delta values must not cross into the brief. + for leaked in ("0.66", "0.8", "0.5", "0.14", "-0.26"): + assert leaked not in brief + + def test_repetition_line_states_the_repeat_count_when_gt_one(self) -> None: + brief = self._input_for(duplication_factor=5) + assert "Each question was asked 5 times." in brief + + def test_repetition_line_softens_when_asked_once(self) -> None: + brief = self._input_for(duplication_factor=1) + assert "asked once (no repetition to speak of)" in brief + assert "Each question was asked 1 times" not in brief + + +class TestConsistencyRead: + @pytest.mark.parametrize( + ("std", "expected"), + [ + (0.08, "answers stayed consistent"), + (0.15, "answers were mostly consistent, with some variation"), + (0.34, "answers varied"), + (None, "consistency unknown"), + ], + ) + def test_bands_and_boundaries(self, std, expected) -> None: + assert _consistency_read(std) == expected + + +class TestFailureIsNonFatal: + def test_openai_error_returns_none_without_raising(self) -> None: + client = MagicMock() + client.responses.create.side_effect = openai.OpenAIError("provider down") + assert _call(client) is None + + def test_generic_error_returns_none_without_raising(self) -> None: + client = MagicMock() + client.responses.create.side_effect = RuntimeError("unexpected shape") + assert _call(client) is None + + @pytest.mark.parametrize("output_text", ["", " \n\t"]) + def test_empty_or_whitespace_output_returns_none(self, output_text: str) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result(output_text) + assert _call(client) is None diff --git a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py index 722318ac2..6d37b1f18 100644 --- a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py +++ b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py @@ -15,6 +15,25 @@ ) +# A v2 judge run's deterministic run-level overall — not trace-derived, so the +# trace-merge reconstructions must preserve it rather than rebuild it away. +_OVERALL_BLOCK = { + "overall_score": 0.66, + "verdict": "Good", + "ai_summary": "The run performed well overall.", + "breakdown": [ + { + "name": "Adherence to Ground Truth", + "key": "ground_truth", + "score": 0.8, + "weight": 0.5, + "delta": 0.14, + "verdict": "Good", + } + ], +} + + class TestGetEvaluationWithScoresS3: """Test get_evaluation_with_scores() S3 retrieval.""" @@ -164,6 +183,102 @@ def test_resync_merges_cache_with_langfuse( saved_ids = {t["trace_id"] for t in saved_score["traces"]} assert saved_ids == {"old", "new"} + @patch("app.services.evaluations.evaluation.get_evaluation_run_by_id") + @patch("app.services.evaluations.evaluation.load_json_from_object_store") + @patch("app.services.evaluations.evaluation.get_cloud_storage") + def test_overall_block_survives_cached_trace_serve( + self, + mock_get_storage: MagicMock, + mock_load: MagicMock, + mock_get_eval: MagicMock, + eval_run_factory: Callable[..., MagicMock], + ) -> None: + """v2 judge run: the run-level `overall` block must survive the cached-serve + trace reconstruction (the path the frontend hits), not get dropped when the + score is rebuilt from summary_scores + traces.""" + eval_run = eval_run_factory( + id=200, + status="completed", + score={ + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "overall": _OVERALL_BLOCK, + }, + score_trace_url="s3://bucket/traces.json", + dataset_name="test_dataset", + run_name="test_run", + ) + mock_get_eval.return_value = eval_run + mock_get_storage.return_value = MagicMock() + mock_load.return_value = [{"trace_id": "t1"}] + + result, error = get_evaluation_with_scores( + session=MagicMock(), + evaluation_id=200, + organization_id=1, + project_id=1, + get_trace_info=True, + resync_score=False, + ) + + assert error is None + assert result.score["traces"] == [{"trace_id": "t1"}] + overall = result.score["overall"] + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + assert overall["breakdown"] == _OVERALL_BLOCK["breakdown"] + + @patch("app.services.evaluations.evaluation.save_score") + @patch("app.services.evaluations.evaluation.fetch_trace_scores_from_langfuse") + @patch("app.services.evaluations.evaluation.get_langfuse_client") + @patch("app.services.evaluations.evaluation.get_evaluation_run_by_id") + @patch("app.services.evaluations.evaluation.load_json_from_object_store") + @patch("app.services.evaluations.evaluation.get_cloud_storage") + def test_overall_block_survives_resync_merge_and_persists( + self, + mock_get_storage: MagicMock, + mock_load: MagicMock, + mock_get_eval: MagicMock, + mock_get_langfuse: MagicMock, + mock_fetch_langfuse: MagicMock, + mock_save_score: MagicMock, + eval_run_factory: Callable[..., MagicMock], + ) -> None: + """Resync path: the merged score handed to save_score (which re-persists to + DB) must carry the run-level `overall` block, not just the merged traces.""" + eval_run = eval_run_factory( + id=201, + status="completed", + score={ + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "overall": _OVERALL_BLOCK, + }, + score_trace_url="s3://bucket/traces.json", + dataset_name="test_dataset", + run_name="test_run", + ) + mock_get_eval.return_value = eval_run + mock_get_storage.return_value = MagicMock() + mock_load.return_value = [{"trace_id": "old", "scores": []}] + mock_get_langfuse.return_value = MagicMock() + mock_fetch_langfuse.return_value = { + "summary_scores": [], + "traces": [{"trace_id": "new", "scores": []}], + } + mock_save_score.return_value = eval_run + + get_evaluation_with_scores( + session=MagicMock(), + evaluation_id=201, + organization_id=1, + project_id=1, + get_trace_info=True, + resync_score=True, + ) + + saved_score = mock_save_score.call_args.kwargs["score"] + assert {t["trace_id"] for t in saved_score["traces"]} == {"old", "new"} + assert saved_score["overall"] == _OVERALL_BLOCK + @patch("app.services.evaluations.evaluation.save_score") @patch("app.services.evaluations.evaluation.fetch_trace_scores_from_langfuse") @patch("app.services.evaluations.evaluation.get_langfuse_client") From 6900d8acf9c296fef297c1c2365f490867e3a46f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:47:01 +0530 Subject: [PATCH 05/15] fix(*): update the ai_summary code --- backend/app/crud/evaluations/summary.py | 23 ++++--------------- .../crud/evaluations/test_run_ai_summary.py | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index faef41e03..2d69caf0e 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -8,7 +8,6 @@ from openai import OpenAI from sqlmodel import Session -from app.core.config import settings from app.crud.evaluations.judge import JudgeMetricEnum from app.crud.evaluations.response_parsing import extract_response_text from app.crud.evaluations.score import OverallSummary @@ -19,7 +18,9 @@ # Reasoning tokens count against this cap, so it needs headroom beyond the visible # note — too low and reasoning consumes it all, yielding empty output. Length is # bounded by the "1 to 3 sentences" instruction, not by a tight token cap. -_SUMMARY_MAX_OUTPUT_TOKENS: int = 600 +_SUMMARY_MAX_OUTPUT_TOKENS: int = 2000 + +_SUMMARY_REASONING_EFFORT: str = "minimal" # Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain # consistency read. With repeated questions a low spread means the assistant answered @@ -115,17 +116,7 @@ def generate_run_ai_summary( summary_scores: list[dict[str, Any]], duplication_factor: int, ) -> str | None: - """Best-effort one-shot natural-language note on the run's overall quality. - - Reuses the aggregate's existing OpenAI client (never builds a new one) and the - judge's reasoning-model invocation: `map_kaapi_to_openai_params` suppresses - temperature and sets the reasoning effort, then the body goes to the Responses - API. `summary_scores` supplies each metric's std (consistency) and - `duplication_factor` the repeat count that the brief translates into plain - language. A summary is a nicety, so ANY failure — missing key, API error, empty - output — logs a warning and returns None rather than propagating: the - deterministic overall (score/verdict/breakdown) must still persist. - """ + """Best-effort one-shot natural-language note on the run's overall quality.""" user_message = _format_overall_for_prompt( overall=overall, run_name=run_name, @@ -137,7 +128,7 @@ def generate_run_ai_summary( session=session, kaapi_params={ "model": model, - "effort": settings.EVAL_JUDGE_REASONING_EFFORT, + "effort": _SUMMARY_REASONING_EFFORT, }, ) if mapper_warnings: @@ -182,8 +173,4 @@ def generate_run_ai_summary( ) return None - logger.info( - f"[generate_run_ai_summary] Generated run summary | model={model} | " - f"run_name={run_name} | chars={len(summary)}" - ) return summary diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 412be8d2e..f9dc15bba 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -118,7 +118,7 @@ def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( params = client.responses.create.call_args.kwargs brief = params["input"] - assert params["max_output_tokens"] == 600 + assert params["max_output_tokens"] == 2000 assert "temperature" not in params assert "Accuracy against the expected answers" in brief From 5b1074b34dd9f80b8698d6b87e3a13c0bc665d0e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:28:43 +0530 Subject: [PATCH 06/15] fix(evals): clenaups --- backend/app/crud/evaluations/summary.py | 11 ++++------- .../tests/crud/evaluations/test_run_ai_summary.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 2d69caf0e..dc3fc3af5 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -15,16 +15,10 @@ logger = logging.getLogger(__name__) -# Reasoning tokens count against this cap, so it needs headroom beyond the visible -# note — too low and reasoning consumes it all, yielding empty output. Length is -# bounded by the "1 to 3 sentences" instruction, not by a tight token cap. _SUMMARY_MAX_OUTPUT_TOKENS: int = 2000 _SUMMARY_REASONING_EFFORT: str = "minimal" -# Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain -# consistency read. With repeated questions a low spread means the assistant answered -# the same question the same way each time. _CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 _CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 @@ -142,6 +136,10 @@ def generate_run_ai_summary( "max_output_tokens": _SUMMARY_MAX_OUTPUT_TOKENS, } response = openai_client.responses.create(**params) + # Parse inside the try so a malformed/unexpected Responses payload degrades + # to None like any other failure — the call site has no guard, so an escape + # here would fail the whole run against the best-effort contract. + summary = extract_response_text(response).strip() except openai.OpenAIError as exc: status = getattr(exc, "status_code", None) # 5xx is provider-side (alert-worthy); 4xx/None is caller/Kaapi-side noise. @@ -165,7 +163,6 @@ def generate_run_ai_summary( ) return None - summary = extract_response_text(response).strip() if not summary: logger.warning( f"[generate_run_ai_summary] Empty summary returned | model={model} | " diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index f9dc15bba..813c68368 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -177,3 +177,16 @@ def test_empty_or_whitespace_output_returns_none(self, output_text: str) -> None client = MagicMock() client.responses.create.return_value = _responses_result(output_text) assert _call(client) is None + + def test_malformed_payload_parse_error_returns_none_without_raising(self) -> None: + # extract_response_text runs inside the try, so a raise on an unexpected + # Responses payload must degrade to None, never escape to fail the run. + client = MagicMock() + client.responses.create.return_value = ( + SimpleNamespace() + ) # no output_text/output + with patch( + "app.crud.evaluations.summary.extract_response_text", + side_effect=ValueError("unexpected Responses payload"), + ): + assert _call(client) is None From 28e3b7209143b214692638d357514a42527ec14c Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:42:39 +0530 Subject: [PATCH 07/15] fix(evals): clenaups --- backend/app/crud/evaluations/summary.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index dc3fc3af5..f1a53f6c9 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -22,8 +22,7 @@ _CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 _CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 -# Internal metric key -> plain behaviour phrase, so the brief never leaks the -# "Adherence to X" labels into the model's context. +# Internal metric key -> plain behaviour phrase, so the brief never leaks the labels into the model's context. _DIMENSION_PLAIN_NAME: dict[str, str] = { JudgeMetricEnum.GROUND_TRUTH.value: "Accuracy against the expected answers", JudgeMetricEnum.KNOWLEDGE_BASE.value: "Grounding in the source material", From 0ab017086f61b5e9b702947c540af44ce1f39288 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Mon, 3 Aug 2026 11:27:24 +0530 Subject: [PATCH 08/15] updated wiki --- docs/wiki/modules/evaluations.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index c0119546d..61d496ba7 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -23,11 +23,12 @@ All paths relative to `backend/app/`. Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. +`score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it). ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py` - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries), `score.py`, `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Responses call, qualitative brief only, no raw scores in the prompt), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async From b5b359a97e954bd5e2d9ebf4d7875290efae5f1d Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:22:34 +0530 Subject: [PATCH 09/15] feat(*): update the judge prompt and test cases --- .../docs/evaluation/create_evaluation_v2.md | 2 +- .../api/docs/evaluation/improve_prompt_v2.md | 2 +- backend/app/crud/evaluations/judge.py | 9 +- backend/app/crud/evaluations/score.py | 111 +++++++++++----- backend/app/crud/evaluations/summary.py | 4 +- .../evaluations/prompt_improvement.py | 2 +- .../api/routes/test_improve_prompt_v2.py | 7 +- .../tests/crud/evaluations/test_fast_judge.py | 122 +++++++++--------- .../app/tests/crud/evaluations/test_judge.py | 63 +++++---- .../crud/evaluations/test_overall_summary.py | 48 +++---- .../crud/evaluations/test_run_ai_summary.py | 43 +++--- .../app/tests/crud/evaluations/test_score.py | 18 +-- .../evaluations/test_evaluation_service_s3.py | 16 +-- 13 files changed, 258 insertions(+), 189 deletions(-) diff --git a/backend/app/api/docs/evaluation/create_evaluation_v2.md b/backend/app/api/docs/evaluation/create_evaluation_v2.md index b09463a7e..6a015b34c 100644 --- a/backend/app/api/docs/evaluation/create_evaluation_v2.md +++ b/backend/app/api/docs/evaluation/create_evaluation_v2.md @@ -3,7 +3,7 @@ body with Kaapi's native LLM-as-Judge built in — v1 is left unchanged. v2 runs are **always fast** and always judged (there is no `run_mode`; batch is deferred to a later phase). Every scoreable row is automatically judged (no opt-in -flag) by one combined LLM-judge call scoring each applicable metric in [0, 1] with +flag) by one combined LLM-judge call scoring each applicable metric on an integer 0–5 scale with reasoning: **Adherence to Ground Truth** (answer conveys the same correct information as the golden answer), **Adherence to Prompt** (answer obeys the assistant's configured instructions; applies only when the run resolves a config diff --git a/backend/app/api/docs/evaluation/improve_prompt_v2.md b/backend/app/api/docs/evaluation/improve_prompt_v2.md index 060820f6c..e75bc3592 100644 --- a/backend/app/api/docs/evaluation/improve_prompt_v2.md +++ b/backend/app/api/docs/evaluation/improve_prompt_v2.md @@ -2,7 +2,7 @@ Enqueue a v2 prompt-recommendation job for the configuration evaluated by a judg Unlike v1 (which reads cosine similarity + correctness), this consumes the native three-metric judge results — **Adherence to Ground Truth**, **Adherence to Prompt**, -and **Adherence to Knowledge Base** — each carrying a 0–1 score and the judge's +and **Adherence to Knowledge Base** — each carrying an integer 0–5 score and the judge's reasoning. The worker reads both the score and the reasoning per metric, focuses the rewrite on rows where Adherence to Prompt / Ground Truth are low, and changes only `completion.params.instructions`; model, knowledge base, and all other settings are diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index c228f447b..164440927 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -151,7 +151,7 @@ class JudgeMetricSpec: @dataclass class MetricScore: - """One metric's outcome for a row: score in [0, 1] and its reasoning.""" + """One metric's outcome for a row: integer score in 0–5 and its reasoning.""" score: float reasoning: str @@ -246,8 +246,11 @@ def _parse_metric_score(key: JudgeMetricEnum, raw: Any) -> MetricScore: raise ValueError( f"metric '{key.value}' score is not a number: {raw.get('score')!r}" ) from exc - if not 0.0 <= score <= 1.0: - raise ValueError(f"metric '{key.value}' score out of [0, 1]: {score}") + if score != int(score) or not 0 <= score <= 5: + raise ValueError( + f"metric '{key.value}' score must be an integer 0-5: {raw.get('score')!r}" + ) + score = int(score) reasoning = str(raw.get("reasoning") or "").strip() if not reasoning: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 43a8af5b3..14d52873b 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -12,21 +12,22 @@ class VerdictEnum(str, Enum): - """Qualitative band derived from a 0–1 judge-metric score.""" + """Qualitative band derived from a 0–5 judge-metric score.""" NEEDS_IMPROVEMENT = "Needs Improvement" NEEDS_REFINEMENT = "Needs Refinement" GOOD = "Good" -VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 -VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 2.0 +VERDICT_GOOD_AT_OR_ABOVE: float = 4.0 def verdict_from_score(score: float) -> VerdictEnum: - """Map a 0–1 judge-metric score to its verdict band. + """Map a 0–5 judge-metric score to its verdict band. - Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + Boundaries: below 2 → Needs Improvement, 2 to <4 → Needs Refinement, + 4 and above → Good. """ if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: return VerdictEnum.NEEDS_IMPROVEMENT @@ -64,11 +65,24 @@ def verdict_from_score(score: float) -> VerdictEnum: JUDGE_SYSTEM_PREAMBLE: str = ( "You are a strict, impartial evaluator. You score an assistant's answer on the " - "independent metrics listed below in a single pass. Each metric is a float in " - "[0.0, 1.0] with one or two sentences of reasoning. The metrics are independent " - "— judge each only against its own inputs and rules; do not let one metric's " - "verdict bleed into another. Score EVERY metric listed below. Never omit a metric " - "from the output, even if some input blocks are irrelevant to it." + "independent metrics listed below in a single pass. Each metric is an integer " + "score from 0 to 5, where 0 is the worst case (clearly wrong / ungrounded / a hard " + "instruction violation, or no answer at all) and 5 is the best case (fully correct " + "/ fully grounded / fully compliant), with one or two sentences of reasoning. " + "Scores MUST be integers — 0, 1, 2, 3, 4, or 5 — never fractions, decimals, or " + "percentages. The metrics are independent — judge each only against its own inputs " + "and rules; do not let one metric's verdict bleed into another. Score EVERY metric " + "listed below. Never omit a metric from the output, even if some input blocks are " + "irrelevant to it.\n\n" + 'Write every "reasoning" value in English, regardless of what language the ' + "question, generated answer, golden answer, or configured instructions are in. " + "This is a requirement on the reasoning TEXT only — it never changes what you are " + "judging. In particular, Metric 3 (Adherence to Prompt) may require you to judge " + "whether the answer itself is in Hindi, Tamil, or any other language the configured " + "instructions specify — keep judging that exactly as instructed, and simply write " + 'your explanation of that judgment in English (e.g. "The answer is in Hindi as ' + 'required" is correct; do not switch the reasoning itself into Hindi). Never mix ' + "languages within a single reasoning string." ) GROUND_TRUTH_JUDGE_PROMPT: str = ( @@ -82,7 +96,22 @@ def verdict_from_score(score: float) -> VerdictEnum: "and that would be wrong, is a factual error.\n" "- Do NOT reward or penalize style, tone, length, or language.\n" "- Do NOT use any outside knowledge; the golden answer is the source of truth.\n" - "- Do NOT answer the question yourself.\n" + "- Do NOT answer the question yourself.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Fully correct and complete. Conveys everything material in the golden answer " + "(a paraphrase, reordering, or additional correct detail is still a 5).\n" + "- 4: Correct and materially complete, but omits one minor, non-essential " + "supporting detail.\n" + "- 3: Partially correct. The core of the answer is right, but at least one material " + "fact is missing, incomplete, or slightly off.\n" + "- 2: Mixed or significantly incomplete. Gets some of the answer right but muddles " + "or omits more than one material fact, or is wrong on a meaningful component while " + "looking plausible on the surface.\n" + "- 1: Mostly incorrect. Contradicts the golden answer on a key point; at most small " + "correct fragments remain.\n" + "- 0: Completely wrong or contradicts the golden answer outright, OR the row has no " + "answer / an errored, empty, or non-responsive output.\n" "Reasoning: name what was correct or what was missing/contradicted.\n" "When scoring THIS metric, consider only these input blocks: Question, " "Generated answer, Golden (reference) answer.\n" @@ -98,8 +127,8 @@ def verdict_from_score(score: float) -> VerdictEnum: "documents, so treat any rule about which source or knowledge base to use (e.g. " "'only use the knowledge base', 'do not use outside information') as satisfied — " "the Knowledge Base metric judges that.\n\n" - "Start from 1.0 and deduct ONLY for a violation of an instruction the block " - "actually states. Never invent a requirement the instructions do not set; a " + 'Start from "no violations" and deduct ONLY for a violation of an instruction the ' + "block actually states. Never invent a requirement the instructions do not set; a " "conditional rule (applies only in a specific situation, e.g. " "'ask for the user's age' or 'if condition Y holds, also mention Z') counts as " "satisfied unless that situation is present in the question. Deduct across " @@ -116,20 +145,25 @@ def verdict_from_score(score: float) -> VerdictEnum: "out-of-scope or disallowed ones as instructed.\n" "3. Fallback compliance — when the instructions define a fallback for the " "unknown/out-of-scope case, the answer uses it instead of ignoring it. Only " - "penalize here for CONTRADICTING an explicit instruction (e.g. skipping the " - "configured fallback, answering a clearly disallowed topic); do not infer " + "penalize here for CONTRADICTING an explicit instruction; do not infer " "fabrication from missing grounding.\n" "4. Format compliance — follows any explicit format rules " "(word limit, structure, opening/closing pattern).\n\n" - "Scoring guide:\n" - "- 1.0: No violation of any stated instruction.\n" - "- 0.7–0.9: One soft miss on a stated rule (e.g. slightly off tone, minor format " - "deviation).\n" - "- 0.4–0.69: One clear violation of an explicit rule.\n" - "- 0.0–0.39: Multiple clear violations, or a hard violation — leaked system " - "prompt, answered a clearly disallowed topic, or hijacked by injection.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: No violation of any stated instruction, across all applicable dimensions.\n" + "- 4: One soft, minor miss on a stated rule (e.g. slightly off tone, minor format " + "deviation) — otherwise compliant.\n" + "- 3: One clear violation of a single explicit rule.\n" + "- 2: Multiple clear violations, or one moderately serious violation spanning more " + "than one dimension.\n" + "- 1: A severe violation of a core instruction (e.g. ignoring a configured fallback " + "on a disallowed ask, a partial injection hijack), but not a full hard violation.\n" + "- 0: A hard violation — leaked system prompt, fully answered a clearly disallowed " + "topic, fully hijacked by injection — OR the row has no answer / an errored, empty, " + "or non-responsive output.\n" "Reasoning: name the specific violated instruction and how the answer violated it. " - "If no stated instruction was violated, say so and score high.\n" + "If no stated instruction was violated, say so and score 5.\n" "When scoring THIS metric, consider only these input blocks: Assistant's " "configured instructions, Question, Generated answer.\n" "Do not consider: Golden (reference) answer, Retrieved knowledge-base chunks." @@ -145,18 +179,30 @@ def verdict_from_score(score: float) -> VerdictEnum: "on the same topic as a chunk, or that requires an inferential leap the chunks do " "not spell out is UNSUPPORTED, not supported.\n" "- Identify the answer's load-bearing (material) claims — the ones that carry its " - "substance. If ANY material claim is unsupported, cap the score at 0.3 regardless " - "of how many minor claims are supported. Otherwise score = supported claims / " - "total factual claims.\n" + "substance.\n" "- Text that makes no factual claim (a greeting, a pleasantry, or a plain refusal " - "to answer) is EXCLUDED from the claim count — do not let it inflate the score.\n" + "to answer) is EXCLUDED from the claim count.\n" "- Judge groundedness ONLY, not correctness, completeness, or " "instruction-following. A claim faithful to the chunks is grounded even if the " "chunks are themselves wrong.\n" "- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed " - "source of support.\n" + "source of support.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Every factual claim is explicitly supported by the retrieved chunks. Fully " + "grounded.\n" + "- 4: All load-bearing claims are grounded; only a minor, non-material claim lacks " + "explicit support.\n" + "- 3: One non-critical inferential leap beyond the chunks, but no load-bearing " + "claim is fabricated.\n" + "- 2: At least one load-bearing/material claim is unsupported or invented, even " + "though other claims are grounded.\n" + "- 1: Most claims are unsupported or invented; only incidental/minor claims are " + "grounded.\n" + "- 0: The answer is fabricated wholesale — no claim is grounded in the retrieved " + "chunks — OR the row has no answer / an errored, empty, or non-responsive output.\n" "Reasoning: quote the exact chunk span supporting the main claim. When the score " - "is below 1.0, name the specific unsupported or invented claim.\n" + "is below 5, name the specific unsupported or invented claim.\n" "When scoring THIS metric, consider only these input blocks: Generated answer, " "Retrieved knowledge-base chunks.\n" "Do not consider: Assistant's configured instructions, Question, Golden " @@ -165,9 +211,10 @@ def verdict_from_score(score: float) -> VerdictEnum: JUDGE_OUTPUT_INSTRUCTION: str = ( "Respond with ONLY a single JSON object mapping each metric key to its result, of " - 'the form {{"": {{"score": , "reasoning": ' - '""}}}}. Include exactly these metric keys: {metric_keys}. ' - "Output nothing else." + 'the form {{"": {{"score": , "reasoning": ' + '""}}}}. Scores MUST be integers 0-5. Every ' + '"reasoning" string MUST be written in English. Include exactly these metric keys: ' + "{metric_keys}. Output nothing else." ) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index f1a53f6c9..08fea05fa 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -19,8 +19,8 @@ _SUMMARY_REASONING_EFFORT: str = "minimal" -_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 -_CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 +_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.5 +_CONSISTENCY_MIXED_AT_OR_BELOW: float = 1.0 # Internal metric key -> plain behaviour phrase, so the brief never leaks the labels into the model's context. _DIMENSION_PLAIN_NAME: dict[str, str] = { diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index d7065fdd5..532864841 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -671,7 +671,7 @@ def _draft_improved_prompt( "metric objects; each has `name`, `value`, and `comment`, where:\n" f"- `name` is one of `{GROUND_TRUTH_SCORE_NAME}`, `{PROMPT_SCORE_NAME}`, " f"or `{KNOWLEDGE_BASE_SCORE_NAME}`.\n" - "- `value` is the metric's score from 0 (worst) to 1 (best). Metrics that " + "- `value` is the metric's score, an integer from 0 (worst) to 5 (best). Metrics that " 'could not be scored appear with `value` = "N/A" and `unscoreable` = true; ' "ignore those.\n" "- `comment` is the judge's reasoning for that score — read it, not just " diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index 5daf93ce3..096f4d202 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -76,13 +76,13 @@ "scores": [ { "name": GROUND_TRUTH_SCORE_NAME, - "value": 0.4, + "value": 2, "data_type": "NUMERIC", "comment": _GT_COMMENT, }, { "name": PROMPT_SCORE_NAME, - "value": 0.3, + "value": 1, "data_type": "NUMERIC", "comment": _PROMPT_COMMENT, }, @@ -578,6 +578,9 @@ def test_v2_message_carries_metric_reasoning_and_ignores_na( assert _GT_COMMENT in message assert _PROMPT_COMMENT in message + # The v2 brief describes the judge score on the 0–5 integer scale. + assert "an integer from 0 (worst) to 5 (best)" in message + # The unscoreable KB metric must be presented as ignore-worthy, not a real # score: the prompt tells the model to skip value="N/A" / unscoreable metrics. assert '"N/A"' in message diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index fb26aea20..f250039b5 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -2,7 +2,7 @@ Drives the ground-truth + adherence-to-prompt slices of the three-metric SRD through the real fast pipeline with a judged run (`is_judge_run=True`, -`langfuse=None` as v2 dispatches): FR-2 (trace scores in [0,1] + reasoning), FR-9 +`langfuse=None` as v2 dispatches): FR-2 (integer 0–5 trace scores + reasoning), FR-9 (zero-config uses the fallback model + built-in prompts), FR-14 (run-level summary scores + per-row scores on the trace records), FR-15 (per-row isolation), FR-16 (judge cost stage), FR-18 (v1 never judges). @@ -169,8 +169,8 @@ def _judge_response(score: float, reasoning: str, *, usage=(12, 6, 18)): def _both_metrics_response( *, - ground_truth: tuple[float, str] = (0.8, "conveys the same facts"), - prompt: tuple[float, str] = (0.6, "answered in the wrong language"), + ground_truth: tuple[float, str] = (4, "conveys the same facts"), + prompt: tuple[float, str] = (3, "answered in the wrong language"), usage=(12, 6, 18), ): gt_score, gt_reason = ground_truth @@ -383,7 +383,7 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.8, "conveys the same facts"), + judge_side_effect=lambda _p: _judge_response(4, "conveys the same facts"), ) assert result.status == "completed" @@ -396,8 +396,8 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( gt = _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME) assert _score_named(traces[ref], COSINE_SCORE_NAME) is None assert gt is not None - assert 0.0 <= gt["value"] <= 1.0 - assert gt["value"] == pytest.approx(0.8, abs=0.01) + assert 0 <= gt["value"] <= 5 + assert gt["value"] == 4 assert gt["comment"] == "conveys the same facts" summary_names = {s["name"] for s in result.score["summary_scores"]} @@ -408,12 +408,12 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( for s in result.score["summary_scores"] if s["name"] == GROUND_TRUTH_SCORE_NAME ) - assert gt_summary["avg"] == pytest.approx(0.8, abs=0.01) + assert gt_summary["avg"] == pytest.approx(4, abs=0.01) assert gt_summary["total_pairs"] == 2 assert _metric_values(result, GROUND_TRUTH_SCORE_NAME) == { - "item-1": 0.8, - "item-2": 0.8, + "item-1": 4, + "item-2": 4, } run = db.get(EvaluationRun, result.id) @@ -437,7 +437,7 @@ def test_zero_config_judges_with_fallback_model_and_builtin_prompt( def _capture(params): captured.update(params) - return _judge_response(0.6, "partially correct") + return _judge_response(3, "partially correct") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_capture) @@ -470,7 +470,7 @@ def test_per_row_isolation_leaves_failed_row_unscoreable( def _judge(params): if "Q-bad" in params["input"]: return _raw_judge_response("totally not json") - return _judge_response(0.9, "correct") + return _judge_response(5, "correct") result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=_judge @@ -514,9 +514,7 @@ def test_judge_cost_stage_tracked( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response( - 0.7, "close", usage=(12, 6, 18) - ), + judge_side_effect=lambda _p: _judge_response(3, "close", usage=(12, 6, 18)), mock_cost=True, ) @@ -573,7 +571,7 @@ def _judge(params): for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) assert prompt_score is not None - assert prompt_score["value"] == pytest.approx(0.6, abs=0.01) + assert prompt_score["value"] == 3 assert prompt_score["comment"] == "answered in the wrong language" assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME) is not None @@ -637,7 +635,7 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _both_metrics_response( - ground_truth=(0.9, "same facts"), prompt=(0.25, "answered in English") + ground_truth=(5, "same facts"), prompt=(1, "answered in English") ), ) @@ -646,15 +644,15 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) assert prompt_score == { "name": PROMPT_SCORE_NAME, - "value": 0.25, + "value": 1, "data_type": "NUMERIC", "comment": "answered in English", "verdict": "Needs Improvement", } - assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 + assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 5 prompt_summary = _summary_named(result, PROMPT_SCORE_NAME) - assert prompt_summary["avg"] == 0.25 + assert prompt_summary["avg"] == 1 assert prompt_summary["total_pairs"] == 2 def test_prompt_template_is_appended_to_the_config_prompt_block( @@ -737,7 +735,7 @@ def _assert_only_ground_truth_scored(self, result: EvaluationRun) -> None: trace = _trace_by_ref(result)["item-1"] assert _score_named(trace, PROMPT_SCORE_NAME) is None - assert _score_named(trace, GROUND_TRUTH_SCORE_NAME)["value"] == 0.8 + assert _score_named(trace, GROUND_TRUTH_SCORE_NAME)["value"] == 4 def test_config_without_instructions_drops_the_prompt_metric( self, db: Session, user_api_key: TestAuthContext, _s3_store @@ -759,8 +757,10 @@ def _judge(params): result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) self._assert_only_ground_truth_scored(result) - # The dropped metric leaves no trace in the judge request either. - assert "Adherence to Prompt" not in captured["instructions"] + # The dropped metric's rubric fragment (uniquely marked by its score key) + # leaves no trace in the judge request. The shared preamble still names the + # metric in passing, so match the fragment marker, not the bare label. + assert '(score key "prompt")' not in captured["instructions"] assert "Assistant's configured instructions" not in captured["input"] def test_empty_instructions_drop_the_prompt_metric( @@ -837,7 +837,7 @@ def test_v1_run_produces_no_judge_metrics( def _judge(params): judge_calls.append(params) - return _judge_response(0.9, "should never run") + return _judge_response(5, "should never run") result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=_judge @@ -888,9 +888,9 @@ def _all_three_judge(self, _params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "prompt": {"score": 0.4, "reasoning": "p"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "prompt": {"score": 2, "reasoning": "p"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) @@ -907,9 +907,9 @@ def test_judged_run_overall_matches_metric_averages( assert result.status == "completed" overall = result.score["overall"] - # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.66. - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + # 4*0.5 + 3*0.3 + 2*0.2 = 2.0 + 0.9 + 0.4 = 3.3. + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" # The successful summary boundary flows into ai_summary. assert overall["ai_summary"] == DEFAULT_RUN_SUMMARY @@ -940,8 +940,8 @@ def test_summary_failure_leaves_overall_intact_with_null_ai_summary( assert result.status == "completed" overall = result.score["overall"] assert overall["ai_summary"] is None - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" assert len(overall["breakdown"]) == 3 def test_overall_survives_into_the_persisted_db_score( @@ -958,8 +958,8 @@ def test_overall_survives_into_the_persisted_db_score( db.expire_all() persisted = db.get(EvaluationRun, result.id).score["overall"] - assert persisted["overall_score"] == 0.66 - assert persisted["verdict"] == "Good" + assert persisted["overall_score"] == 3.3 + assert persisted["verdict"] == "Needs Refinement" assert {dim["key"] for dim in persisted["breakdown"]} == { "ground_truth", "prompt", @@ -985,7 +985,7 @@ def test_v1_run_has_no_overall_in_score( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + judge_side_effect=lambda _p: _judge_response(5, "never runs"), ) assert result.status == "completed" @@ -1097,12 +1097,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) @@ -1112,7 +1112,7 @@ def _judge(params): traces = _trace_by_ref(result) kb_chunked = _score_named(traces["item-chunked"], KNOWLEDGE_BASE_SCORE_NAME) assert kb_chunked is not None - assert kb_chunked["value"] == 0.6 + assert kb_chunked["value"] == 3 # The judged-but-chunkless row surfaces a human N/A placeholder that stays out # of the summary avg (it is not a numeric 0). @@ -1140,18 +1140,18 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.7, "reasoning": "grounded"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "grounded"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) assert kb["data_type"] == "NUMERIC" - assert kb["value"] == 0.7 + assert kb["value"] == 3 # No relevance gate: every retrieved chunk names a match, low scores included. assert ( kb["comment"] @@ -1175,18 +1175,18 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.9, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "partial"}, + "ground_truth": {"score": 5, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "partial"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) assert kb["data_type"] == "NUMERIC" - assert kb["value"] == 0.6 + assert kb["value"] == 3 assert kb["comment"] == "partial | Top matches: a.pdf (55.0%), b.pdf (30.0%)" def test_kb_na_placeholder_stays_out_of_summary_avg( @@ -1208,12 +1208,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "grounded"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "grounded"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) @@ -1222,8 +1222,8 @@ def _judge(params): for s in result.score["summary_scores"] if s["name"] == KNOWLEDGE_BASE_SCORE_NAME ) - # Only item-scored (0.6) is a real KB score; the N/A row never enters the avg. - assert kb_summary["avg"] == 0.6 + # Only item-scored (3) is a real KB score; the N/A row never enters the avg. + assert kb_summary["avg"] == 3 assert kb_summary["total_pairs"] == 1 def test_non_kb_metric_none_is_skipped_not_placeholdered( @@ -1243,7 +1243,7 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _raw_judge_response( - json.dumps({"knowledge_base": {"score": 0.7, "reasoning": "grounded"}}) + json.dumps({"knowledge_base": {"score": 3, "reasoning": "grounded"}}) ), ) @@ -1272,17 +1272,17 @@ def test_each_scored_judge_metric_carries_its_verdict_band( ] _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) - # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, - # 0.75 → Good, so the three metrics land in three different bands. + # One score per band: 1 → Needs Improvement, 3 → Needs Refinement, + # 5 → Good, so the three metrics land in three different bands. result, _ = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.2, "reasoning": "gt"}, - "prompt": {"score": 0.45, "reasoning": "p"}, - "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + "ground_truth": {"score": 1, "reasoning": "gt"}, + "prompt": {"score": 3, "reasoning": "p"}, + "knowledge_base": {"score": 5, "reasoning": "kb"}, } ) ), @@ -1319,7 +1319,7 @@ def test_cosine_score_carries_no_verdict( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + judge_side_effect=lambda _p: _judge_response(5, "never runs"), ) cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) @@ -1338,12 +1338,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) - return _judge_response(0.8, "gt only") + return _judge_response(4, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) diff --git a/backend/app/tests/crud/evaluations/test_judge.py b/backend/app/tests/crud/evaluations/test_judge.py index 4503acac9..cb3721e4e 100644 --- a/backend/app/tests/crud/evaluations/test_judge.py +++ b/backend/app/tests/crud/evaluations/test_judge.py @@ -1,7 +1,7 @@ """Unit tests for the native LLM-as-judge scoring primitives (`crud/evaluations/judge.py`). Covers the ground-truth and adherence-to-prompt judge slices of the three-metric -SRD: FR-2 (score in [0,1] + reasoning), FR-3 (ground truth judged against the +SRD: FR-2 (integer 0–5 score + reasoning), FR-3 (ground truth judged against the golden answer), FR-9 (zero-config default prompt), FR-15 (malformed → raises so the row isolates), plus the per-row gating that drops a metric whose inputs the row cannot supply. The single external boundary — the OpenAI judge completion — is @@ -66,24 +66,36 @@ class TestParseJudgeOutput: def test_parses_well_formed_ground_truth(self) -> None: specs = _ALL_METRICS result = _parse_judge_output( - json.dumps( - {"ground_truth": {"score": 0.75, "reasoning": "close paraphrase"}} - ), + json.dumps({"ground_truth": {"score": 4, "reasoning": "close paraphrase"}}), specs, ) assert set(result) == {JudgeMetricEnum.GROUND_TRUTH} score = result[JudgeMetricEnum.GROUND_TRUTH] - assert score.score == 0.75 + assert score.score == 4 assert score.reasoning == "close paraphrase" def test_extracts_json_from_prose_wrapper(self) -> None: specs = _ALL_METRICS wrapped = ( - 'Here is my grade:\n{"ground_truth": {"score": 0.4, "reasoning": ' + 'Here is my grade:\n{"ground_truth": {"score": 2, "reasoning": ' '"missing a key fact"}}\nThanks.' ) result = _parse_judge_output(wrapped, specs) - assert result[JudgeMetricEnum.GROUND_TRUTH].score == 0.4 + assert result[JudgeMetricEnum.GROUND_TRUTH].score == 2 + + def test_integer_score_accepted_fraction_rejected(self) -> None: + # 0–5 is a stepped integer scale: an in-range integer parses, a fraction that + # was legal on the old 0–1 scale is now rejected. + result = _parse_judge_output( + json.dumps({"ground_truth": {"score": 4, "reasoning": "ok"}}), + _ALL_METRICS, + ) + assert result[JudgeMetricEnum.GROUND_TRUTH].score == 4 + with pytest.raises(ValueError, match="integer 0-5"): + _parse_judge_output( + json.dumps({"ground_truth": {"score": 0.5, "reasoning": "x"}}), + _ALL_METRICS, + ) def test_drops_only_the_missing_metric_when_others_present(self) -> None: # Grade against two specs (ground_truth + a synthetic sibling); a reply that @@ -93,7 +105,7 @@ def test_drops_only_the_missing_metric_when_others_present(self) -> None: sibling = replace(gt_spec, key=sibling_key) result = _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.9, "reasoning": "correct"}}), + json.dumps({"ground_truth": {"score": 5, "reasoning": "correct"}}), [gt_spec, sibling], ) assert list(result) == [JudgeMetricEnum.GROUND_TRUTH] @@ -102,9 +114,9 @@ def test_parses_both_metrics_when_both_requested(self) -> None: result = _parse_judge_output( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "same facts"}, + "ground_truth": {"score": 4, "reasoning": "same facts"}, "knowledge_base": { - "score": 0.6, + "score": 3, "reasoning": "one unsupported claim", }, } @@ -115,13 +127,13 @@ def test_parses_both_metrics_when_both_requested(self) -> None: JudgeMetricEnum.GROUND_TRUTH, JudgeMetricEnum.KNOWLEDGE_BASE, } - assert result[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.6 + assert result[JudgeMetricEnum.KNOWLEDGE_BASE].score == 3 def test_missing_knowledge_base_leaves_only_ground_truth(self) -> None: # Both metrics requested but the reply omits knowledge_base: that metric is # silently unscoreable for the row — no raise, ground_truth still parsed. result = _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.9, "reasoning": "correct"}}), + json.dumps({"ground_truth": {"score": 5, "reasoning": "correct"}}), [_GROUND_TRUTH_SPEC, _KNOWLEDGE_BASE_SPEC], ) assert set(result) == {JudgeMetricEnum.GROUND_TRUTH} @@ -129,7 +141,7 @@ def test_missing_knowledge_base_leaves_only_ground_truth(self) -> None: def test_raises_when_no_enabled_metric_scored(self) -> None: specs = _ALL_METRICS with pytest.raises(ValueError, match="scored no enabled metric"): - _parse_judge_output(json.dumps({"unrelated": {"score": 0.5}}), specs) + _parse_judge_output(json.dumps({"unrelated": {"score": 3}}), specs) def test_raises_on_empty_response(self) -> None: with pytest.raises(ValueError, match="empty judge response"): @@ -139,17 +151,18 @@ def test_raises_on_non_json(self) -> None: with pytest.raises(ValueError, match="no JSON object"): _parse_judge_output("the answer is basically fine", _ALL_METRICS) - def test_raises_on_score_out_of_range(self) -> None: - with pytest.raises(ValueError, match="out of .0, 1."): + @pytest.mark.parametrize("bad_score", [6, -1]) + def test_raises_on_score_out_of_range(self, bad_score: int) -> None: + with pytest.raises(ValueError, match="integer 0-5"): _parse_judge_output( - json.dumps({"ground_truth": {"score": 1.4, "reasoning": "x"}}), + json.dumps({"ground_truth": {"score": bad_score, "reasoning": "x"}}), _ALL_METRICS, ) def test_raises_on_empty_reasoning(self) -> None: with pytest.raises(ValueError, match="empty 'reasoning'"): _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.8, "reasoning": " "}}), + json.dumps({"ground_truth": {"score": 4, "reasoning": " "}}), _ALL_METRICS, ) @@ -319,7 +332,7 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: with patch( "app.crud.evaluations.judge._create_judge_response", return_value=_judge_response( - {"ground_truth": {"score": 0.9, "reasoning": "same meaning"}} + {"ground_truth": {"score": 5, "reasoning": "same meaning"}} ), ): result = judge_row( @@ -330,7 +343,7 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: ) assert result.metrics[JudgeMetricEnum.GROUND_TRUTH] == MetricScore( - score=0.9, reasoning="same meaning" + score=5, reasoning="same meaning" ) assert result.usage == { "input_tokens": 15, @@ -343,7 +356,7 @@ def test_chunkless_row_judges_ground_truth_only(self, db: Session) -> None: def _capture(_client, params): captured.update(params) - return _judge_response({"ground_truth": {"score": 0.9, "reasoning": "ok"}}) + return _judge_response({"ground_truth": {"score": 5, "reasoning": "ok"}}) with patch( "app.crud.evaluations.judge._create_judge_response", side_effect=_capture @@ -367,8 +380,8 @@ def test_chunk_present_row_judges_both_metrics(self, db: Session) -> None: "app.crud.evaluations.judge._create_judge_response", return_value=_judge_response( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.7, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ), ): @@ -386,7 +399,7 @@ def test_chunk_present_row_judges_both_metrics(self, db: Session) -> None: JudgeMetricEnum.GROUND_TRUTH, JudgeMetricEnum.KNOWLEDGE_BASE, } - assert result.metrics[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.7 + assert result.metrics[JudgeMetricEnum.KNOWLEDGE_BASE].score == 3 def test_chunkless_prompt_is_byte_identical_to_ground_truth_only( self, db: Session @@ -396,7 +409,7 @@ def test_chunkless_prompt_is_byte_identical_to_ground_truth_only( def _capture(_client, params): captured.update(params) - return _judge_response({"ground_truth": {"score": 0.5, "reasoning": "x"}}) + return _judge_response({"ground_truth": {"score": 3, "reasoning": "x"}}) with patch( "app.crud.evaluations.judge._create_judge_response", side_effect=_capture @@ -421,7 +434,7 @@ def test_judge_call_receives_question_answer_and_golden(self, db: Session) -> No def _capture(_client, params): captured.update(params) return _judge_response( - {"ground_truth": {"score": 0.5, "reasoning": "partial"}} + {"ground_truth": {"score": 3, "reasoning": "partial"}} ) with patch( diff --git a/backend/app/tests/crud/evaluations/test_overall_summary.py b/backend/app/tests/crud/evaluations/test_overall_summary.py index a846f89de..257136840 100644 --- a/backend/app/tests/crud/evaluations/test_overall_summary.py +++ b/backend/app/tests/crud/evaluations/test_overall_summary.py @@ -31,11 +31,11 @@ def _summary(metric_avgs: dict[str, float]): class TestAllThreeMetrics: def test_weighted_overall_and_per_dimension_breakdown(self) -> None: - # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.40 + 0.18 + 0.08 = 0.66. - result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6, "prompt": 0.4}) + # 4*0.5 + 3*0.3 + 2*0.2 = 2.0 + 0.9 + 0.4 = 3.3. + result = _summary({"ground_truth": 4, "knowledge_base": 3, "prompt": 2}) assert result is not None - assert result["overall_score"] == 0.66 - assert result["verdict"] == "Good" + assert result["overall_score"] == 3.3 + assert result["verdict"] == "Needs Refinement" assert result["ai_summary"] is None by_key = {dim["key"]: dim for dim in result["breakdown"]} @@ -49,21 +49,21 @@ def test_weighted_overall_and_per_dimension_breakdown(self) -> None: assert by_key["prompt"]["weight"] == 0.2 assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 - assert by_key["ground_truth"]["score"] == 0.8 - assert by_key["knowledge_base"]["score"] == 0.6 - assert by_key["prompt"]["score"] == 0.4 + assert by_key["ground_truth"]["score"] == 4 + assert by_key["knowledge_base"]["score"] == 3 + assert by_key["prompt"]["score"] == 2 - # delta = dimension score - overall (0.66), sign shows pull vs the run. - assert by_key["ground_truth"]["delta"] == 0.14 - assert by_key["knowledge_base"]["delta"] == -0.06 - assert by_key["prompt"]["delta"] == -0.26 + # delta = dimension score - overall (3.3), sign shows pull vs the run. + assert by_key["ground_truth"]["delta"] == 0.7 + assert by_key["knowledge_base"]["delta"] == -0.3 + assert by_key["prompt"]["delta"] == -1.3 assert by_key["ground_truth"]["verdict"] == "Good" - assert by_key["knowledge_base"]["verdict"] == "Good" + assert by_key["knowledge_base"]["verdict"] == "Needs Refinement" assert by_key["prompt"]["verdict"] == "Needs Refinement" def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None: - result = _summary({"ground_truth": 0.55, "knowledge_base": 0.25, "prompt": 0.9}) + result = _summary({"ground_truth": 2.75, "knowledge_base": 1.25, "prompt": 4.5}) assert result is not None for dim in result["breakdown"]: assert dim["verdict"] == verdict_from_score(dim["score"]).value @@ -71,7 +71,7 @@ def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None class TestRenormalizationWhenAMetricIsAbsent: def test_ground_truth_and_knowledge_base_only(self) -> None: - result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6}) + result = _summary({"ground_truth": 4, "knowledge_base": 3}) assert result is not None by_key = {dim["key"]: dim for dim in result["breakdown"]} assert set(by_key) == {"ground_truth", "knowledge_base"} @@ -82,11 +82,11 @@ def test_ground_truth_and_knowledge_base_only(self) -> None: # overall still sums to 1.0). assert by_key["ground_truth"]["weight"] == 0.62 assert by_key["knowledge_base"]["weight"] == 0.37 - # 0.625*0.8 + 0.375*0.6 = 0.5 + 0.225 = 0.725 → 0.72. - assert result["overall_score"] == 0.72 + # 0.625*4 + 0.375*3 = 2.5 + 1.125 = 3.625 → 3.62. + assert result["overall_score"] == 3.62 def test_ground_truth_and_prompt_only(self) -> None: - result = _summary({"ground_truth": 0.8, "prompt": 0.4}) + result = _summary({"ground_truth": 4, "prompt": 2}) assert result is not None by_key = {dim["key"]: dim for dim in result["breakdown"]} assert set(by_key) == {"ground_truth", "prompt"} @@ -97,10 +97,10 @@ def test_ground_truth_and_prompt_only(self) -> None: assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 def test_absent_metric_never_drags_the_overall_down(self) -> None: - # A missing metric is dropped, not scored 0: two perfect metrics stay at 1.0. - result = _summary({"ground_truth": 1.0, "knowledge_base": 1.0}) + # A missing metric is dropped, not scored 0: two perfect metrics stay at 5.0. + result = _summary({"ground_truth": 5, "knowledge_base": 5}) assert result is not None - assert result["overall_score"] == 1.0 + assert result["overall_score"] == 5.0 class TestNothingScored: @@ -111,17 +111,17 @@ def test_empty_metric_avgs_returns_none(self) -> None: class TestBadgeBoundaries: def test_overall_exactly_on_needs_refinement_lower_bound(self) -> None: # Equal dimension averages → the weighted overall equals that value exactly. - result = _summary({"ground_truth": 0.3, "knowledge_base": 0.3, "prompt": 0.3}) + result = _summary({"ground_truth": 2, "knowledge_base": 2, "prompt": 2}) assert result is not None - assert result["overall_score"] == 0.3 + assert result["overall_score"] == 2.0 assert result["verdict"] == "Needs Refinement" for dim in result["breakdown"]: assert dim["verdict"] == "Needs Refinement" def test_overall_exactly_on_good_lower_bound(self) -> None: - result = _summary({"ground_truth": 0.6, "knowledge_base": 0.6, "prompt": 0.6}) + result = _summary({"ground_truth": 4, "knowledge_base": 4, "prompt": 4}) assert result is not None - assert result["overall_score"] == 0.6 + assert result["overall_score"] == 4.0 assert result["verdict"] == "Good" for dim in result["breakdown"]: assert dim["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 813c68368..68317b87b 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -23,32 +23,32 @@ def _overall() -> OverallSummary: return { - "overall_score": 0.66, - "verdict": "Good", + "overall_score": 3.3, + "verdict": "Needs Refinement", "ai_summary": None, "breakdown": [ { "name": "Adherence to Ground Truth", "key": "ground_truth", - "score": 0.8, + "score": 4, "weight": 0.5, - "delta": 0.14, + "delta": 0.7, "verdict": "Good", }, { "name": "Adherence to Knowledge Base", "key": "knowledge_base", - "score": 0.6, + "score": 3, "weight": 0.3, - "delta": -0.06, - "verdict": "Good", + "delta": -0.3, + "verdict": "Needs Refinement", }, { "name": "Adherence to Prompt", "key": "prompt", - "score": 0.4, + "score": 2, "weight": 0.2, - "delta": -0.26, + "delta": -1.3, "verdict": "Needs Refinement", }, ], @@ -56,11 +56,12 @@ def _overall() -> OverallSummary: def _summary_scores() -> list[dict]: - # std per dimension drives the consistency read; names match the overall's dims. + # std per dimension drives the consistency read (0–5 spread; cutoffs 0.5 / 1.0); + # names match the overall's dims. return [ - {"name": "Adherence to Ground Truth", "avg": 0.8, "std": 0.05}, - {"name": "Adherence to Knowledge Base", "avg": 0.6, "std": 0.15}, - {"name": "Adherence to Prompt", "avg": 0.4, "std": 0.34}, + {"name": "Adherence to Ground Truth", "avg": 4.0, "std": 0.3}, + {"name": "Adherence to Knowledge Base", "avg": 3.0, "std": 0.8}, + {"name": "Adherence to Prompt", "avg": 2.0, "std": 1.5}, ] @@ -125,16 +126,16 @@ def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( assert "Grounding in the source material" in brief assert "Tone and instruction-following" in brief - assert "answers stayed consistent" in brief # std 0.05 - assert "answers were mostly consistent, with some variation" in brief # 0.15 - assert "answers varied" in brief # 0.34 + assert "answers stayed consistent" in brief # std 0.3 + assert "answers were mostly consistent, with some variation" in brief # 0.8 + assert "answers varied" in brief # 1.5 def test_brief_leaks_no_raw_scores_or_internal_labels(self) -> None: brief = self._input_for(duplication_factor=5) assert "Adherence to" not in brief assert "verdict" not in brief # Raw score / weight / delta values must not cross into the brief. - for leaked in ("0.66", "0.8", "0.5", "0.14", "-0.26"): + for leaked in ("3.3", "0.7", "-1.3", "0.5", "0.3"): assert leaked not in brief def test_repetition_line_states_the_repeat_count_when_gt_one(self) -> None: @@ -151,9 +152,11 @@ class TestConsistencyRead: @pytest.mark.parametrize( ("std", "expected"), [ - (0.08, "answers stayed consistent"), - (0.15, "answers were mostly consistent, with some variation"), - (0.34, "answers varied"), + (0.3, "answers stayed consistent"), + (0.5, "answers stayed consistent"), + (0.8, "answers were mostly consistent, with some variation"), + (1.0, "answers were mostly consistent, with some variation"), + (1.5, "answers varied"), (None, "consistency unknown"), ], ) diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py index f1d2aa03e..a5e46041e 100644 --- a/backend/app/tests/crud/evaluations/test_score.py +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -1,8 +1,8 @@ """`verdict_from_score` — the per-metric verdict band on v2 judge runs. -Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs -Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the -display string is part of the contract. +Bands are upper-bound exclusive on the 0–5 integer scale: [0, 2) Needs Improvement, +[2, 4) Needs Refinement, [4, 5] Good. The enum serializes by value into the trace +JSON, so the display string is part of the contract. """ import pytest @@ -14,12 +14,12 @@ class TestVerdictFromScore: @pytest.mark.parametrize( ("score", "expected"), [ - (0.0, VerdictEnum.NEEDS_IMPROVEMENT), - (0.29, VerdictEnum.NEEDS_IMPROVEMENT), - (0.3, VerdictEnum.NEEDS_REFINEMENT), - (0.59, VerdictEnum.NEEDS_REFINEMENT), - (0.6, VerdictEnum.GOOD), - (1.0, VerdictEnum.GOOD), + (0, VerdictEnum.NEEDS_IMPROVEMENT), + (1, VerdictEnum.NEEDS_IMPROVEMENT), + (2, VerdictEnum.NEEDS_REFINEMENT), + (3, VerdictEnum.NEEDS_REFINEMENT), + (4, VerdictEnum.GOOD), + (5, VerdictEnum.GOOD), ], ) def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: diff --git a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py index 6d37b1f18..e0b755c77 100644 --- a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py +++ b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py @@ -18,16 +18,16 @@ # A v2 judge run's deterministic run-level overall — not trace-derived, so the # trace-merge reconstructions must preserve it rather than rebuild it away. _OVERALL_BLOCK = { - "overall_score": 0.66, - "verdict": "Good", + "overall_score": 3.3, + "verdict": "Needs Refinement", "ai_summary": "The run performed well overall.", "breakdown": [ { "name": "Adherence to Ground Truth", "key": "ground_truth", - "score": 0.8, + "score": 4, "weight": 0.5, - "delta": 0.14, + "delta": 0.7, "verdict": "Good", } ], @@ -200,7 +200,7 @@ def test_overall_block_survives_cached_trace_serve( id=200, status="completed", score={ - "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 4}], "overall": _OVERALL_BLOCK, }, score_trace_url="s3://bucket/traces.json", @@ -223,8 +223,8 @@ def test_overall_block_survives_cached_trace_serve( assert error is None assert result.score["traces"] == [{"trace_id": "t1"}] overall = result.score["overall"] - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" assert overall["breakdown"] == _OVERALL_BLOCK["breakdown"] @patch("app.services.evaluations.evaluation.save_score") @@ -249,7 +249,7 @@ def test_overall_block_survives_resync_merge_and_persists( id=201, status="completed", score={ - "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 4}], "overall": _OVERALL_BLOCK, }, score_trace_url="s3://bucket/traces.json", From c24ce2b4772b8bbd418f65975fa2215c1c152f1f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:45:44 +0530 Subject: [PATCH 10/15] Merge branch 'main' of https://github.com/ProjectTech4DevAI/kaapi-backend into fix/judge-metrics-prompt-update --- backend/app/crud/evaluations/fast.py | 56 +++++----------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 22da3c23b..0cd8e88ca 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -758,51 +758,6 @@ def _stage2_embeddings( return eval_run, embedding_results -def _resolve_config_prompt( - *, session: Session, eval_run: EvaluationRun, log_prefix: str -) -> str | None: - """The evaluated bot's own configured prompt, or None if unresolvable. - - The prompt template is appended when the config carries one, since it is equally - part of what the bot was told to do. Returns None when the config carries no - instructions, so the caller drops the prompt metric rather than grading against "". - """ - if not eval_run.config_id or not eval_run.config_version: - return None - - config, error = resolve_evaluation_config( - session=session, - config_id=eval_run.config_id, - config_version=eval_run.config_version, - project_id=eval_run.project_id, - ) - if error or config is None: - return None - - # Native/proxy params aren't TextLLMParams-shaped; a mismatch just means there are - # no instructions to grade against, not a run failure. - try: - params = TextLLMParams.model_validate(config.completion.params) - except ValidationError as exc: - logger.info( - f"[_resolve_config_prompt] {log_prefix} Completion params are not " - f"text params; prompt metric unscoreable | error={exc}" - ) - return None - - sections: list[str] = [] - if params.instructions: - sections.append(params.instructions.strip()) - if config.prompt_template and config.prompt_template.template: - sections.append( - f"{PROMPT_TEMPLATE_LABEL}\n{config.prompt_template.template.strip()}" - ) - - if not sections: - return None - return "\n\n".join(sections) - - def _judge_rows( *, session: Session, @@ -1149,8 +1104,15 @@ def _stage3_score_and_trace( if overall is not None: # Falls back to 1 (no repetition) if the dataset can't be resolved, so # the summary still generates. - duplication_factor = _resolve_duplication_factor( - session=session, eval_run=eval_run + dataset = get_dataset_by_id( + session=session, + dataset_id=eval_run.dataset_id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + metadata = dataset.dataset_metadata if dataset else None + duplication_factor = max( + 1, int((metadata or {}).get(DATASET_META_DUPLICATION_FACTOR, 1)) ) overall["ai_summary"] = generate_run_ai_summary( session=session, From d4239282906aa7737c23cb39285923d07616be0d Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:46:47 +0530 Subject: [PATCH 11/15] fix(*): cleanups --- backend/app/crud/evaluations/fast.py | 63 ++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 0cd8e88ca..b1e4e361a 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -758,6 +758,51 @@ def _stage2_embeddings( return eval_run, embedding_results +def _resolve_config_prompt( + *, session: Session, eval_run: EvaluationRun, log_prefix: str +) -> str | None: + """The evaluated bot's own configured prompt, or None if unresolvable. + + The prompt template is appended when the config carries one, since it is equally + part of what the bot was told to do. Returns None when the config carries no + instructions, so the caller drops the prompt metric rather than grading against "". + """ + if not eval_run.config_id or not eval_run.config_version: + return None + + config, error = resolve_evaluation_config( + session=session, + config_id=eval_run.config_id, + config_version=eval_run.config_version, + project_id=eval_run.project_id, + ) + if error or config is None: + return None + + # Native/proxy params aren't TextLLMParams-shaped; a mismatch just means there are + # no instructions to grade against, not a run failure. + try: + params = TextLLMParams.model_validate(config.completion.params) + except ValidationError as exc: + logger.info( + f"[_resolve_config_prompt] {log_prefix} Completion params are not " + f"text params; prompt metric unscoreable | error={exc}" + ) + return None + + sections: list[str] = [] + if params.instructions: + sections.append(params.instructions.strip()) + if config.prompt_template and config.prompt_template.template: + sections.append( + f"{PROMPT_TEMPLATE_LABEL}\n{config.prompt_template.template.strip()}" + ) + + if not sections: + return None + return "\n\n".join(sections) + + def _judge_rows( *, session: Session, @@ -862,24 +907,6 @@ def _attach_metric_scores( ) -def _resolve_duplication_factor(*, session: Session, eval_run: EvaluationRun) -> int: - """How many times each question was asked, from the run's dataset metadata. - - Feeds only the best-effort AI summary, so it never fails the run: an - unresolvable dataset or missing metadata falls back to 1 (no repetition). - """ - dataset = get_dataset_by_id( - session=session, - dataset_id=eval_run.dataset_id, - organization_id=eval_run.organization_id, - project_id=eval_run.project_id, - ) - if dataset is None: - return 1 - metadata = dataset.dataset_metadata or {} - return max(1, int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1))) - - def _stage3_score_and_trace( *, session: Session, From ff54e8f77a876ef15e0ceef8dd78c10b56030a24 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:48:10 +0530 Subject: [PATCH 12/15] fix(*): few updates on evals flow --- backend/app/crud/evaluations/fast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index b1e4e361a..4876ddc90 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -1129,8 +1129,8 @@ def _stage3_score_and_trace( metric_names={spec.key.value: spec.score_name for spec in metrics}, ) if overall is not None: - # Falls back to 1 (no repetition) if the dataset can't be resolved, so - # the summary still generates. + # Falls back to 1 (no repetition) if the dataset/metadata can't be + # resolved, so the summary still generates. dataset = get_dataset_by_id( session=session, dataset_id=eval_run.dataset_id, From 215448b6bd57d4a0fbff9a33a049e263ab16e76d Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:59:10 +0530 Subject: [PATCH 13/15] fix(*): update the the system preamble --- backend/app/crud/evaluations/score.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 14d52873b..2a40e2da0 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -73,16 +73,7 @@ def verdict_from_score(score: float) -> VerdictEnum: "percentages. The metrics are independent — judge each only against its own inputs " "and rules; do not let one metric's verdict bleed into another. Score EVERY metric " "listed below. Never omit a metric from the output, even if some input blocks are " - "irrelevant to it.\n\n" - 'Write every "reasoning" value in English, regardless of what language the ' - "question, generated answer, golden answer, or configured instructions are in. " - "This is a requirement on the reasoning TEXT only — it never changes what you are " - "judging. In particular, Metric 3 (Adherence to Prompt) may require you to judge " - "whether the answer itself is in Hindi, Tamil, or any other language the configured " - "instructions specify — keep judging that exactly as instructed, and simply write " - 'your explanation of that judgment in English (e.g. "The answer is in Hindi as ' - 'required" is correct; do not switch the reasoning itself into Hindi). Never mix ' - "languages within a single reasoning string." + "irrelevant to it." ) GROUND_TRUTH_JUDGE_PROMPT: str = ( From 7c91096d0d7c94ee6f3873eb8b487bc9a2a40f28 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:59:23 +0530 Subject: [PATCH 14/15] fix(*): update the wiki --- docs/wiki/modules/evaluations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 61d496ba7..3bc78e38b 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Judge metrics score on an **integer 0–5 stepped scale** (the LLM returns integers 0–5; `crud/evaluations/judge.py::_parse_metric_score` enforces it) with English-only reasoning; scores are stored raw (0–5), so the API structure is unchanged but the value range is 0–5 (cosine on the v1 path stays 0–1). Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, 0–5 cutoffs 2/4 → 0–1 Needs Improvement, 2–3 Needs Refinement, 4–5 Good), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. `score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it). ## Services / CRUD From 28a164aede4181a902ef9f3831ac80cbdbfad69e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:06:18 +0530 Subject: [PATCH 15/15] fix(*): update the test cases --- .../app/tests/crud/evaluations/test_fast_judge.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index f250039b5..2f164a0d4 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -34,7 +34,6 @@ JOB_TYPE_EVALUATION_FAST_CHUNK, PROMPT_TEMPLATE_LABEL, _format_top_kb_matches, - _resolve_duplication_factor, _responses_call_for_item, run_fast_evaluation, run_response_chunk, @@ -992,18 +991,6 @@ def test_v1_run_has_no_overall_in_score( assert "overall" not in result.score -class TestResolveDuplicationFactor: - """`_resolve_duplication_factor` feeds only the best-effort summary, so a missing - dataset degrades to 1 (no repetition) rather than failing the run.""" - - def test_falls_back_to_one_when_dataset_missing(self) -> None: - with patch("app.crud.evaluations.fast.get_dataset_by_id", return_value=None): - assert ( - _resolve_duplication_factor(session=MagicMock(), eval_run=MagicMock()) - == 1 - ) - - def _responses_item(item_id: str = "item-1") -> dict[str, Any]: return { "id": item_id,