From 46a3037dc2a8fa8d10a2beb5a05f74f46a3c0cb4 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:21:37 +0530 Subject: [PATCH 1/9] feat(evals): added the verdict corresponding to judge metric in each row --- backend/app/crud/evaluations/fast.py | 5 +- backend/app/crud/evaluations/score.py | 29 +++++ .../tests/crud/evaluations/test_fast_judge.py | 105 ++++++++++++++++++ .../app/tests/crud/evaluations/test_score.py | 42 +++++++ docs/wiki/modules/evaluations.md | 2 +- 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 backend/app/tests/crud/evaluations/test_score.py diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index bbfe16912..de39f4a20 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -83,6 +83,7 @@ EvaluationScore, TraceData, TraceScore, + verdict_from_score, ) from app.crud.job import ( create_batch_job, @@ -1171,12 +1172,14 @@ def _stage3_score_and_trace( comment = metric_score.reasoning if is_kb: comment = f"{comment} | Top matches: {top_matches}" + rounded_score = round(metric_score.score, 2) trace_scores.append( { "name": spec.score_name, - "value": round(metric_score.score, 2), + "value": rounded_score, "data_type": "NUMERIC", "comment": comment, + "verdict": verdict_from_score(rounded_score), } ) elif is_kb: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 93dbc70a2..df012b29e 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -5,10 +5,36 @@ used throughout the evaluation system. """ +from enum import Enum from typing import NotRequired, TypedDict DEFAULT_CATEGORY: str = "Other" + +class VerdictEnum(str, Enum): + """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + + NEEDS_IMPROVEMENT = "Needs Improvement" + NEEDS_REFINEMENT = "Needs Refinement" + GOOD = "Good" + + +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 +VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 + + +def verdict_from_score(score: float) -> VerdictEnum: + """Map a 0–1 judge-metric score to its verdict band. + + Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + """ + if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: + return VerdictEnum.NEEDS_IMPROVEMENT + if score < VERDICT_GOOD_AT_OR_ABOVE: + return VerdictEnum.NEEDS_REFINEMENT + return VerdictEnum.GOOD + + # Canonical name/comment for the cosine-similarity score, centralized to avoid # import cycles. COSINE_SCORE_NAME: str = "Cosine Similarity" @@ -152,6 +178,9 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] + # Verdict band; present only on numeric judge-metric scores (v2 runs), never + # on cosine or unscoreable/"N/A" entries. + verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 81ee818b2..ff9b20aac 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -43,6 +43,7 @@ JUDGE_FAILED_REASON, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, + verdict_from_score, ) from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob @@ -615,11 +616,13 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) + # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, "data_type": "NUMERIC", "comment": "answered in English", + "verdict": "Needs Improvement", } assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 @@ -1082,6 +1085,108 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None +class TestVerdictBandOnTraceScores: + """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + the cosine (v1) score and the KB N/A placeholder never do.""" + + def test_each_scored_judge_metric_carries_its_verdict_band( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, + # 0.75 → Good, so the three metrics land in three different bands. + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.2, "reasoning": "gt"}, + "prompt": {"score": 0.45, "reasoning": "p"}, + "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + } + ) + ), + ) + + trace = _trace_by_ref(result)["item-1"] + gt = _score_named(trace, GROUND_TRUTH_SCORE_NAME) + prompt = _score_named(trace, PROMPT_SCORE_NAME) + kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) + + # Expected bands are the #1092 spec, not a re-run of the mapper. + assert gt["verdict"] == "Needs Improvement" + assert prompt["verdict"] == "Needs Refinement" + assert kb["verdict"] == "Good" + + for score in (gt, prompt, kb): + assert score["verdict"] == verdict_from_score(score["value"]) + + def test_cosine_score_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) + assert cosine is not None + assert "verdict" not in cosine + + def test_kb_na_placeholder_carries_no_verdict( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + plain = _resp_result("item-plain", "Q1", "golden-1") # no retrieved_chunks + _seed_chunk(db=db, eval_run=eval_run, results=[plain], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + return _judge_response(0.8, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named( + _trace_by_ref(result)["item-plain"], KNOWLEDGE_BASE_SCORE_NAME + ) + assert kb["value"] == "N/A" + assert "verdict" not in kb + + class TestFormatTopKbMatches: """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py new file mode 100644 index 000000000..dcf140c24 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -0,0 +1,42 @@ +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). + +Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs +Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the +display string is part of the contract. +""" + +import pytest + +from app.crud.evaluations.score import VerdictEnum, verdict_from_score + + +class TestVerdictFromScore: + @pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, VerdictEnum.NEEDS_IMPROVEMENT), + (0.29, VerdictEnum.NEEDS_IMPROVEMENT), + (0.3, VerdictEnum.NEEDS_REFINEMENT), + (0.59, VerdictEnum.NEEDS_REFINEMENT), + (0.6, VerdictEnum.GOOD), + (1.0, VerdictEnum.GOOD), + ], + ) + def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: + assert verdict_from_score(score) is expected + + @pytest.mark.parametrize( + ("member", "display"), + [ + (VerdictEnum.NEEDS_IMPROVEMENT, "Needs Improvement"), + (VerdictEnum.NEEDS_REFINEMENT, "Needs Refinement"), + (VerdictEnum.GOOD, "Good"), + ], + ) + def test_display_string_is_the_serialized_value( + self, member: VerdictEnum, display: str + ) -> None: + assert member.value == display + + def test_returns_verdict_enum_instance(self) -> None: + assert isinstance(verdict_from_score(0.5), VerdictEnum) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index cd0efcd3d..c0119546d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py` From 6cd310be49486551a523e879d38c4f8f06d3de9f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:23:35 +0530 Subject: [PATCH 2/9] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 +- backend/app/tests/crud/evaluations/test_fast_judge.py | 4 +--- backend/app/tests/crud/evaluations/test_score.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index df012b29e..dd0ec1ff7 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -12,7 +12,7 @@ class VerdictEnum(str, Enum): - """Qualitative band derived from a 0–1 judge-metric score (issue #1092).""" + """Qualitative band derived from a 0–1 judge-metric score.""" NEEDS_IMPROVEMENT = "Needs Improvement" NEEDS_REFINEMENT = "Needs Refinement" diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index ff9b20aac..636d8c635 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -616,7 +616,6 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( traces = _trace_by_ref(result) for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) - # 0.25 falls in the [0, 0.3) band → "Needs Improvement" (issue #1092). assert prompt_score == { "name": PROMPT_SCORE_NAME, "value": 0.25, @@ -1086,7 +1085,7 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( class TestVerdictBandOnTraceScores: - """issue #1092: each scored judge metric on a v2 trace carries a verdict band; + """each scored judge metric on a v2 trace carries a verdict band; the cosine (v1) score and the KB N/A placeholder never do.""" def test_each_scored_judge_metric_carries_its_verdict_band( @@ -1125,7 +1124,6 @@ def test_each_scored_judge_metric_carries_its_verdict_band( prompt = _score_named(trace, PROMPT_SCORE_NAME) kb = _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) - # Expected bands are the #1092 spec, not a re-run of the mapper. assert gt["verdict"] == "Needs Improvement" assert prompt["verdict"] == "Needs Refinement" assert kb["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py index dcf140c24..f1d2aa03e 100644 --- a/backend/app/tests/crud/evaluations/test_score.py +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -1,4 +1,4 @@ -"""`verdict_from_score` — the per-metric verdict band on v2 judge runs (issue #1092). +"""`verdict_from_score` — the per-metric verdict band on v2 judge runs. Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the From 5ff0202d460904e6ea20e94854758d0d1fba377b Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:35:33 +0530 Subject: [PATCH 3/9] fix(*): cleanups --- backend/app/crud/evaluations/score.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index dd0ec1ff7..2ea27f567 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -178,8 +178,6 @@ class TraceScore(TypedDict): value: float | str data_type: str comment: NotRequired[str] - # Verdict band; present only on numeric judge-metric scores (v2 runs), never - # on cosine or unscoreable/"N/A" entries. verdict: NotRequired[str] # True for placeholder scores on unscoreable items; excluded from summary stats. unscoreable: NotRequired[bool] From 194c8f58e2be1b9cc17b030d9960b481a913eca8 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:33:43 +0530 Subject: [PATCH 4/9] feat(*): implement the evals human readable summary generation --- backend/app/core/config.py | 2 + backend/app/crud/evaluations/core.py | 2 + backend/app/crud/evaluations/fast.py | 62 +++++- backend/app/crud/evaluations/judge.py | 4 + backend/app/crud/evaluations/score.py | 74 +++++++ backend/app/crud/evaluations/summary.py | 189 ++++++++++++++++++ .../app/services/evaluations/evaluation.py | 7 + .../tests/crud/evaluations/test_fast_judge.py | 171 +++++++++++++++- .../crud/evaluations/test_overall_summary.py | 127 ++++++++++++ .../crud/evaluations/test_run_ai_summary.py | 179 +++++++++++++++++ .../evaluations/test_evaluation_service_s3.py | 115 +++++++++++ 11 files changed, 930 insertions(+), 2 deletions(-) create mode 100644 backend/app/crud/evaluations/summary.py create mode 100644 backend/app/tests/crud/evaluations/test_overall_summary.py create mode 100644 backend/app/tests/crud/evaluations/test_run_ai_summary.py diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8ebc37b53..337f2bd4f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -216,6 +216,8 @@ def AWS_S3_BUCKET(self) -> str: # One of: none | minimal | low | medium | high | xhigh. EVAL_JUDGE_REASONING_EFFORT: str = "medium" + EVAL_SUMMARY_MODEL: str = "gpt-5-mini" + # Judge runs alone in the single aggregate task (no response calls competing), # so it uses a larger pool than the response stage to finish the max dataset # (EVAL_FAST_MAX_UNIQUE_ROWS x duplication) well under CELERY_TASK_SOFT_TIME_LIMIT. diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index ed15f720e..9a2a0e5e9 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -485,6 +485,8 @@ def save_score( # TODO: Evaluate whether this behaviour is needed or completely discard the storing data in db if score_trace_url: db_score = {"summary_scores": summary_score} + if score.get("overall") is not None: + db_score["overall"] = score["overall"] else: # fallback to store data in db if failed to store in s3 db_score = score diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index de39f4a20..22da3c23b 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -49,6 +49,10 @@ update_evaluation_run, ) from app.crud.evaluations.cost import attach_cost +from app.crud.evaluations.dataset import ( + DATASET_META_DUPLICATION_FACTOR, + get_dataset_by_id, +) from app.crud.evaluations.embeddings import ( EMBEDDING_MODEL, calculate_cosine_similarity, @@ -81,10 +85,13 @@ UNSCOREABLE_EMPTY_GROUND_TRUTH, UNSCOREABLE_EMPTY_OUTPUT, EvaluationScore, + OverallSummary, TraceData, TraceScore, + compute_overall_summary, verdict_from_score, ) +from app.crud.evaluations.summary import generate_run_ai_summary from app.crud.job import ( create_batch_job, delete_batch_job, @@ -900,6 +907,24 @@ def _attach_metric_scores( ) +def _resolve_duplication_factor(*, session: Session, eval_run: EvaluationRun) -> int: + """How many times each question was asked, from the run's dataset metadata. + + Feeds only the best-effort AI summary, so it never fails the run: an + unresolvable dataset or missing metadata falls back to 1 (no repetition). + """ + dataset = get_dataset_by_id( + session=session, + dataset_id=eval_run.dataset_id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + if dataset is None: + return 1 + metadata = dataset.dataset_metadata or {} + return max(1, int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1))) + + def _stage3_score_and_trace( *, session: Session, @@ -953,6 +978,7 @@ def _stage3_score_and_trace( unscoreable: dict[str, str] = {} # {ref: reason} write_items: list[dict[str, Any]] = [] summary_scores: list[dict[str, Any]] = [] + overall: OverallSummary | None = None if is_judge_run: # v2: no cosine. A row is judgeable only with a non-empty generated AND @@ -1109,6 +1135,33 @@ def _stage3_score_and_trace( summary_scores=summary_scores, ) + avg_by_name = {s["name"]: s["avg"] for s in summary_scores if "avg" in s} + metric_avgs = { + spec.key.value: avg_by_name[spec.score_name] + for spec in metrics + if spec.score_name in avg_by_name + } + overall = compute_overall_summary( + metric_avgs=metric_avgs, + metric_weights={spec.key.value: spec.weight for spec in metrics}, + metric_names={spec.key.value: spec.score_name for spec in metrics}, + ) + if overall is not None: + # Falls back to 1 (no repetition) if the dataset can't be resolved, so + # the summary still generates. + duplication_factor = _resolve_duplication_factor( + session=session, eval_run=eval_run + ) + overall["ai_summary"] = generate_run_ai_summary( + session=session, + openai_client=openai_client, + model=settings.EVAL_SUMMARY_MODEL, + overall=overall, + run_name=eval_run.run_name, + summary_scores=summary_scores, + duplication_factor=duplication_factor, + ) + # One combined call grades every metric, so its tokens can't be split per # metric — they land in a single "judge" cost stage. if judge_results and judge_model: @@ -1230,6 +1283,8 @@ def _stage3_score_and_trace( "summary_scores": summary_scores, "traces": traces, } + if overall is not None: + score["overall"] = overall return eval_run, score, write_items @@ -1312,7 +1367,12 @@ def run_fast_evaluation( eval_run=eval_run, update=EvaluationRunUpdate( status="completed", - score={"summary_scores": score["summary_scores"]}, + # Persist the overall alongside the summary so GET run status shows the + # run-level score/verdict/breakdown without loading the S3 trace unit. + score={ + "summary_scores": score["summary_scores"], + "overall": score.get("overall"), + }, cost=eval_run.cost, ), ) diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index 30db8e8eb..c228f447b 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -82,6 +82,7 @@ class JudgeMetricSpec: score_name: str prompt_fragment: str required_inputs: tuple[JudgeInputEnum, ...] + weight: float # All metrics are graded by one combined call, so they share a single judge model @@ -97,6 +98,7 @@ class JudgeMetricSpec: JudgeInputEnum.GENERATED_ANSWER, JudgeInputEnum.GOLDEN_ANSWER, ), + weight=0.5, ), JudgeMetricEnum.PROMPT: JudgeMetricSpec( key=JudgeMetricEnum.PROMPT, @@ -107,6 +109,7 @@ class JudgeMetricSpec: JudgeInputEnum.QUESTION, JudgeInputEnum.GENERATED_ANSWER, ), + weight=0.2, ), JudgeMetricEnum.KNOWLEDGE_BASE: JudgeMetricSpec( key=JudgeMetricEnum.KNOWLEDGE_BASE, @@ -117,6 +120,7 @@ class JudgeMetricSpec: JudgeInputEnum.GENERATED_ANSWER, JudgeInputEnum.RETRIEVED_CHUNKS, ), + weight=0.3, ), } diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 2ea27f567..43a8af5b3 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -235,8 +235,82 @@ class CategoricalSummaryScore(TypedDict): SummaryScore = NumericSummaryScore | CategoricalSummaryScore +class OverallDimension(TypedDict): + """One judge metric's contribution to the run-level overall score.""" + + name: str + key: str + score: float + weight: float + delta: float + verdict: str + + +class OverallSummary(TypedDict): + """Run-level weighted quality view for a v2 judge run. + + `ai_summary` is filled by a best-effort LLM step after the deterministic + fields; it stays None when no summary was generated. + """ + + overall_score: float + verdict: str + ai_summary: str | None + breakdown: list[OverallDimension] + + +def compute_overall_summary( + *, + metric_avgs: dict[str, float], + metric_weights: dict[str, float], + metric_names: dict[str, str], +) -> OverallSummary | None: + """Weighted run-level overall score + per-dimension breakdown. No LLM. + + All three dicts are keyed by metric key value. Only metrics present in + `metric_avgs` (i.e. that actually scored ≥1 row) count; a metric with no + scoreable rows is dropped and the remaining base weights are renormalized to + sum to 1, so a missing metric never drags the overall down. Returns None when + nothing scored. `ai_summary` is None here — the LLM step fills it later. + """ + scored_keys = [key for key in metric_avgs if key in metric_weights] + weight_total = sum(metric_weights[key] for key in scored_keys) + if not scored_keys or weight_total <= 0: + return None + + renorm_weights = {key: metric_weights[key] / weight_total for key in scored_keys} + # Round the overall once, then reuse it everywhere so the badge and the number + # (and every delta) are computed from the same value and can never disagree. + overall_score = round( + sum(renorm_weights[key] * metric_avgs[key] for key in scored_keys), 2 + ) + verdict = verdict_from_score(overall_score).value + + breakdown: list[OverallDimension] = [] + for key in scored_keys: + avg = round(metric_avgs[key], 2) + breakdown.append( + { + "name": metric_names.get(key, key), + "key": key, + "score": avg, + "weight": round(renorm_weights[key], 2), + "delta": round(avg - overall_score, 2), + "verdict": verdict_from_score(avg).value, + } + ) + + return { + "overall_score": overall_score, + "verdict": verdict, + "ai_summary": None, + "breakdown": breakdown, + } + + class EvaluationScore(TypedDict): """Complete evaluation score data with traces and summary statistics.""" summary_scores: list[SummaryScore] traces: list[TraceData] + overall: NotRequired[OverallSummary] diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py new file mode 100644 index 000000000..faef41e03 --- /dev/null +++ b/backend/app/crud/evaluations/summary.py @@ -0,0 +1,189 @@ +"""Human-readable AI summary of a v2 judge run's overall quality. +""" + +import logging +from typing import Any + +import openai +from openai import OpenAI +from sqlmodel import Session + +from app.core.config import settings +from app.crud.evaluations.judge import JudgeMetricEnum +from app.crud.evaluations.response_parsing import extract_response_text +from app.crud.evaluations.score import OverallSummary +from app.services.llm.mappers import map_kaapi_to_openai_params + +logger = logging.getLogger(__name__) + +# Reasoning tokens count against this cap, so it needs headroom beyond the visible +# note — too low and reasoning consumes it all, yielding empty output. Length is +# bounded by the "1 to 3 sentences" instruction, not by a tight token cap. +_SUMMARY_MAX_OUTPUT_TOKENS: int = 600 + +# Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain +# consistency read. With repeated questions a low spread means the assistant answered +# the same question the same way each time. +_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 +_CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 + +# Internal metric key -> plain behaviour phrase, so the brief never leaks the +# "Adherence to X" labels into the model's context. +_DIMENSION_PLAIN_NAME: dict[str, str] = { + JudgeMetricEnum.GROUND_TRUTH.value: "Accuracy against the expected answers", + JudgeMetricEnum.KNOWLEDGE_BASE.value: "Grounding in the source material", + JudgeMetricEnum.PROMPT.value: "Tone and instruction-following", +} + +_SUMMARY_SYSTEM_PROMPT: str = ( + "You write a short, warm, plain-language note about how an AI assistant did on an " + "evaluation, the way a colleague would summarize at a glance. Write 1 to 3 " + "sentences — no bullet points, no headings. Use no scores or numbers at all; the " + "ONLY number you may state is how many times each question was asked. " + "You are given an overall standing, how each area did (as a band word), and how " + "consistent the answers were. Translate everything into plain behaviour words and " + "NEVER use the internal area labels, raw scores, weights, deltas, or the word " + "'verdict'. Map the areas like this: accuracy against the expected answers -> " + "'accurate' or 'matched the expected answers'; grounding in the source material -> " + "'grounded', 'backed by the source material', or 'not made up'; tone and " + "instruction-following -> 'on-tone' or 'followed the instructions (language, " + "style)'. Lead with how it did overall in that plain language, then note the " + "strongest and any weaker area. When each question was asked more than once and the " + "answers stayed stable, say so (e.g. 'answers stayed consistent'); if they varied, " + "say the answers varied. Do not invent facts beyond what you are given. " + "Style anchor — match this tone and length, do not copy the wording: " + '"Consistently grounded and on-tone across the set. Each question was asked 5 ' + 'times and answers stayed consistent."' +) + + +def _consistency_read(std: float | None) -> str: + """Qualitative stability of a metric's per-row scores, derived from its std.""" + if std is None: + return "consistency unknown" + if std <= _CONSISTENCY_STABLE_AT_OR_BELOW: + return "answers stayed consistent" + if std <= _CONSISTENCY_MIXED_AT_OR_BELOW: + return "answers were mostly consistent, with some variation" + return "answers varied" + + +def _format_overall_for_prompt( + *, + overall: OverallSummary, + run_name: str, + summary_scores: list[dict[str, Any]], + duplication_factor: int, +) -> str: + """Compact qualitative brief for the summary model — bands, not raw scores. + + Hands over the overall band, a per-area band + consistency read (from each + metric's std), and the repetition factor. No numbers cross except the repeat + count; the instructions do the plain-language translation. + """ + std_by_name = { + score["name"]: score.get("std") + for score in summary_scores + if isinstance(score, dict) and "name" in score + } + + lines = [ + f"Run: {run_name}", + f"Overall standing: {overall['verdict']}.", + "How each area did:", + ] + for dim in overall["breakdown"]: + plain_name = _DIMENSION_PLAIN_NAME.get(dim["key"], dim["name"]) + consistency = _consistency_read(std_by_name.get(dim["name"])) + lines.append(f"- {plain_name}: {dim['verdict']}; {consistency}") + + if duplication_factor > 1: + lines.append(f"Each question was asked {duplication_factor} times.") + else: + lines.append("Each question was asked once (no repetition to speak of).") + + return "\n".join(lines) + + +def generate_run_ai_summary( + *, + session: Session, + openai_client: OpenAI, + model: str, + overall: OverallSummary, + run_name: str, + summary_scores: list[dict[str, Any]], + duplication_factor: int, +) -> str | None: + """Best-effort one-shot natural-language note on the run's overall quality. + + Reuses the aggregate's existing OpenAI client (never builds a new one) and the + judge's reasoning-model invocation: `map_kaapi_to_openai_params` suppresses + temperature and sets the reasoning effort, then the body goes to the Responses + API. `summary_scores` supplies each metric's std (consistency) and + `duplication_factor` the repeat count that the brief translates into plain + language. A summary is a nicety, so ANY failure — missing key, API error, empty + output — logs a warning and returns None rather than propagating: the + deterministic overall (score/verdict/breakdown) must still persist. + """ + user_message = _format_overall_for_prompt( + overall=overall, + run_name=run_name, + summary_scores=summary_scores, + duplication_factor=duplication_factor, + ) + try: + base_params, mapper_warnings = map_kaapi_to_openai_params( + session=session, + kaapi_params={ + "model": model, + "effort": settings.EVAL_JUDGE_REASONING_EFFORT, + }, + ) + if mapper_warnings: + logger.warning( + f"[generate_run_ai_summary] Mapper warnings: {mapper_warnings}" + ) + params = { + **base_params, + "instructions": _SUMMARY_SYSTEM_PROMPT, + "input": user_message, + "max_output_tokens": _SUMMARY_MAX_OUTPUT_TOKENS, + } + response = openai_client.responses.create(**params) + except openai.OpenAIError as exc: + status = getattr(exc, "status_code", None) + # 5xx is provider-side (alert-worthy); 4xx/None is caller/Kaapi-side noise. + log = logger.error if (status and status >= 500) else logger.warning + tag = "[OPENAI]" if status else "[KAAPI]" + request_id = getattr(exc, "request_id", None) + log( + f"[generate_run_ai_summary] {tag} Summary completion failed " + f"(code: {status or type(exc).__name__}) | model={model} | " + f"run_name={run_name} | request_id={request_id} | {exc}", + exc_info=True, + ) + return None + # Deliberately broad: a summary failure (mapper/config error, unexpected shape) + # must never fail the run, so it degrades to a None result. + except Exception as exc: + logger.warning( + f"[generate_run_ai_summary] Summary call failed; leaving ai_summary " + f"empty | model={model} | run_name={run_name} | error={exc}", + exc_info=True, + ) + return None + + summary = extract_response_text(response).strip() + if not summary: + logger.warning( + f"[generate_run_ai_summary] Empty summary returned | model={model} | " + f"run_name={run_name}" + ) + return None + + logger.info( + f"[generate_run_ai_summary] Generated run summary | model={model} | " + f"run_name={run_name} | chars={len(summary)}" + ) + return summary diff --git a/backend/app/services/evaluations/evaluation.py b/backend/app/services/evaluations/evaluation.py index c01bfbeb9..5f2f90bb2 100644 --- a/backend/app/services/evaluations/evaluation.py +++ b/backend/app/services/evaluations/evaluation.py @@ -453,6 +453,8 @@ def get_evaluation_with_scores( if not get_trace_info: return eval_run, None + run_overall = (eval_run.score or {}).get("overall") + # Caching strategy: trace scores are fetched from Langfuse once, then cached # (traces in S3, summary in the DB). Normal reads serve from that cache, which is # much faster than hitting Langfuse. resync_score=true bypasses the cache. @@ -485,6 +487,8 @@ def get_evaluation_with_scores( unscoreable=eval_run.unscoreable, ) eval_run.score = _attach_category_metrics(cached_score) + if run_overall is not None: + eval_run.score["overall"] = run_overall logger.info( f"[get_evaluation_with_scores] Served traces from cache | " f"evaluation_id={evaluation_id} | traces_count={len(cached_traces)}" @@ -563,6 +567,9 @@ def get_evaluation_with_scores( # `_attach_category_metrics` mutates in place and is idempotent. _attach_category_metrics(merged_score) + if run_overall is not None: + merged_score["overall"] = run_overall + logger.info( f"[get_evaluation_with_scores] Merged traces step-forward | " f"evaluation_id={evaluation_id} | cached={len(cached_traces)} | " diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 636d8c635..fb26aea20 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -34,6 +34,7 @@ JOB_TYPE_EVALUATION_FAST_CHUNK, PROMPT_TEMPLATE_LABEL, _format_top_kb_matches, + _resolve_duplication_factor, _responses_call_for_item, run_fast_evaluation, run_response_chunk, @@ -192,6 +193,17 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)): ) +# Default plain-text stub for the best-effort run-level AI summary. It rides a +# separate `responses.create` call (the judge is patched at _create_judge_response, +# so it never reaches this mock). A bare MagicMock output would poison the score +# JSONB, so every judged run's summary boundary is stubbed with a real string. +DEFAULT_RUN_SUMMARY = "Overall the run performed reasonably; strongest on ground truth." + + +def _summary_response(text: str): + return SimpleNamespace(output_text=text, output=[]) + + @pytest.fixture def _s3_store() -> Iterator[dict[str, list[dict[str, Any]]]]: store: dict[str, list[dict[str, Any]]] = {} @@ -240,7 +252,12 @@ def _seed_chunk( def _persist_score_into(db: Session): def _fake_save_score(*, eval_run_id, score, **_): run = db.get(EvaluationRun, eval_run_id) - run.score = {"summary_scores": score["summary_scores"]} + # Mirror save_score's S3 path: traces go to S3, the DB score keeps the + # summary plus the run-level overall (when present). + db_score: dict[str, Any] = {"summary_scores": score["summary_scores"]} + if score.get("overall") is not None: + db_score["overall"] = score["overall"] + run.score = db_score run.score_trace_url = f"s3://bucket/traces_{eval_run_id}.json" db.add(run) db.commit() @@ -256,6 +273,7 @@ def _run_pipeline( eval_run: EvaluationRun, judge_side_effect, mock_cost: bool = False, + summary_side_effect=None, ) -> tuple[EvaluationRun, MagicMock]: """Run `run_fast_evaluation` for a judged run with all externals stubbed. @@ -263,9 +281,19 @@ def _run_pipeline( key by item_id. The judge completion is driven by `judge_side_effect(params)`. Returns the run plus the OpenAI mock so callers can assert the embedding path was (v1) or was not (v2 judge) exercised. + + The run-level AI summary is a separate `responses.create` call; by default it + returns `DEFAULT_RUN_SUMMARY`. Pass `summary_side_effect` (e.g. an exception) to + drive the best-effort failure path. """ fake_openai = MagicMock() fake_openai.embeddings.create.return_value = _fake_embedding_response() + if summary_side_effect is not None: + fake_openai.responses.create.side_effect = summary_side_effect + else: + fake_openai.responses.create.return_value = _summary_response( + DEFAULT_RUN_SUMMARY + ) def _judge(_client, params): return judge_side_effect(params) @@ -835,6 +863,147 @@ def _judge(params): assert COSINE_SCORE_NAME in summary_names +class TestRunOverallSummary: + """The run-level weighted overall rides the v2 judge run's score.""" + + def _seed_all_three_metrics_run( + self, *, db: Session, user_api_key: TestAuthContext, store + ) -> EvaluationRun: + # Instructions enable the prompt metric; retrieved chunks enable KB; so a + # single judged row scores all three metrics. + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=True, + instructions=BOT_INSTRUCTIONS, + ) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting chunk", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=store) + return eval_run + + def _all_three_judge(self, _params): + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "prompt": {"score": 0.4, "reasoning": "p"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + + def test_judged_run_overall_matches_metric_averages( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, eval_run=eval_run, judge_side_effect=self._all_three_judge + ) + + assert result.status == "completed" + overall = result.score["overall"] + # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.66. + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + # The successful summary boundary flows into ai_summary. + assert overall["ai_summary"] == DEFAULT_RUN_SUMMARY + + # Each breakdown dimension mirrors its run-level summary average. + summary_avgs = { + s["name"]: s["avg"] for s in result.score["summary_scores"] if "avg" in s + } + breakdown_by_name = {dim["name"]: dim for dim in overall["breakdown"]} + assert set(breakdown_by_name) == set(summary_avgs) + for name, avg in summary_avgs.items(): + assert breakdown_by_name[name]["score"] == round(avg, 2) + + def test_summary_failure_leaves_overall_intact_with_null_ai_summary( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=self._all_three_judge, + summary_side_effect=RuntimeError("summary provider down"), + ) + + # The run still completes and the deterministic overall survives whole — + # only the best-effort ai_summary is lost. + assert result.status == "completed" + overall = result.score["overall"] + assert overall["ai_summary"] is None + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + assert len(overall["breakdown"]) == 3 + + def test_overall_survives_into_the_persisted_db_score( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """Re-fetch the run: save_score's S3 path must keep the overall in the DB + score column, not clobber it with the summary alone.""" + eval_run = self._seed_all_three_metrics_run( + db=db, user_api_key=user_api_key, store=_s3_store + ) + result, _ = _run_pipeline( + db=db, eval_run=eval_run, judge_side_effect=self._all_three_judge + ) + + db.expire_all() + persisted = db.get(EvaluationRun, result.id).score["overall"] + assert persisted["overall_score"] == 0.66 + assert persisted["verdict"] == "Good" + assert {dim["key"] for dim in persisted["breakdown"]} == { + "ground_truth", + "prompt", + "knowledge_base", + } + + def test_v1_run_has_no_overall_in_score( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + eval_run = _make_run( + db=db, + user_api_key=user_api_key, + is_judge_run=False, + instructions=BOT_INSTRUCTIONS, + ) + _seed_chunk( + db=db, + eval_run=eval_run, + results=[_resp_result("item-1", "Q1", "golden-1")], + store=_s3_store, + ) + + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + ) + + assert result.status == "completed" + assert "overall" not in result.score + + +class TestResolveDuplicationFactor: + """`_resolve_duplication_factor` feeds only the best-effort summary, so a missing + dataset degrades to 1 (no repetition) rather than failing the run.""" + + def test_falls_back_to_one_when_dataset_missing(self) -> None: + with patch("app.crud.evaluations.fast.get_dataset_by_id", return_value=None): + assert ( + _resolve_duplication_factor(session=MagicMock(), eval_run=MagicMock()) + == 1 + ) + + def _responses_item(item_id: str = "item-1") -> dict[str, Any]: return { "id": item_id, diff --git a/backend/app/tests/crud/evaluations/test_overall_summary.py b/backend/app/tests/crud/evaluations/test_overall_summary.py new file mode 100644 index 000000000..a846f89de --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_overall_summary.py @@ -0,0 +1,127 @@ +"""`compute_overall_summary` — the deterministic run-level weighted overall. + +Weights track the real registry (`{key: spec.weight}`) rather than hard-coded +numbers, so a registry reweighting reshapes these expectations instead of silently +passing. Expected overall scores / deltas are hand-computed from the SRD split +(GT 0.5, KB 0.3, prompt 0.2), independent of the implementation's arithmetic. +""" + +import pytest + +from app.crud.evaluations.judge import METRIC_REGISTRY +from app.crud.evaluations.score import ( + GROUND_TRUTH_SCORE_NAME, + KNOWLEDGE_BASE_SCORE_NAME, + PROMPT_SCORE_NAME, + compute_overall_summary, + verdict_from_score, +) + +METRIC_WEIGHTS = {key.value: spec.weight for key, spec in METRIC_REGISTRY.items()} +METRIC_NAMES = {key.value: spec.score_name for key, spec in METRIC_REGISTRY.items()} + + +def _summary(metric_avgs: dict[str, float]): + return compute_overall_summary( + metric_avgs=metric_avgs, + metric_weights=METRIC_WEIGHTS, + metric_names=METRIC_NAMES, + ) + + +class TestAllThreeMetrics: + def test_weighted_overall_and_per_dimension_breakdown(self) -> None: + # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.40 + 0.18 + 0.08 = 0.66. + result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6, "prompt": 0.4}) + assert result is not None + assert result["overall_score"] == 0.66 + assert result["verdict"] == "Good" + assert result["ai_summary"] is None + + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert by_key["ground_truth"]["name"] == GROUND_TRUTH_SCORE_NAME + assert by_key["knowledge_base"]["name"] == KNOWLEDGE_BASE_SCORE_NAME + assert by_key["prompt"]["name"] == PROMPT_SCORE_NAME + + # All three present → base weights already sum to 1, so no renormalization. + assert by_key["ground_truth"]["weight"] == 0.5 + assert by_key["knowledge_base"]["weight"] == 0.3 + assert by_key["prompt"]["weight"] == 0.2 + assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 + + assert by_key["ground_truth"]["score"] == 0.8 + assert by_key["knowledge_base"]["score"] == 0.6 + assert by_key["prompt"]["score"] == 0.4 + + # delta = dimension score - overall (0.66), sign shows pull vs the run. + assert by_key["ground_truth"]["delta"] == 0.14 + assert by_key["knowledge_base"]["delta"] == -0.06 + assert by_key["prompt"]["delta"] == -0.26 + + assert by_key["ground_truth"]["verdict"] == "Good" + assert by_key["knowledge_base"]["verdict"] == "Good" + assert by_key["prompt"]["verdict"] == "Needs Refinement" + + def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None: + result = _summary({"ground_truth": 0.55, "knowledge_base": 0.25, "prompt": 0.9}) + assert result is not None + for dim in result["breakdown"]: + assert dim["verdict"] == verdict_from_score(dim["score"]).value + + +class TestRenormalizationWhenAMetricIsAbsent: + def test_ground_truth_and_knowledge_base_only(self) -> None: + result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6}) + assert result is not None + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert set(by_key) == {"ground_truth", "knowledge_base"} + + # 0.5 and 0.3 renormalize over 0.8 → 0.625 and 0.375. The stored weights are + # rounded to 2 dp; 0.3/0.8 floats to 0.3749… so it rounds DOWN to 0.37, and + # the displayed pair sums to 0.99 (the un-rounded renorm that drives the + # overall still sums to 1.0). + assert by_key["ground_truth"]["weight"] == 0.62 + assert by_key["knowledge_base"]["weight"] == 0.37 + # 0.625*0.8 + 0.375*0.6 = 0.5 + 0.225 = 0.725 → 0.72. + assert result["overall_score"] == 0.72 + + def test_ground_truth_and_prompt_only(self) -> None: + result = _summary({"ground_truth": 0.8, "prompt": 0.4}) + assert result is not None + by_key = {dim["key"]: dim for dim in result["breakdown"]} + assert set(by_key) == {"ground_truth", "prompt"} + + # 0.5 and 0.2 renormalize over 0.7 → 0.714… and 0.285… → 0.71 and 0.29. + assert by_key["ground_truth"]["weight"] == 0.71 + assert by_key["prompt"]["weight"] == 0.29 + assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 + + def test_absent_metric_never_drags_the_overall_down(self) -> None: + # A missing metric is dropped, not scored 0: two perfect metrics stay at 1.0. + result = _summary({"ground_truth": 1.0, "knowledge_base": 1.0}) + assert result is not None + assert result["overall_score"] == 1.0 + + +class TestNothingScored: + def test_empty_metric_avgs_returns_none(self) -> None: + assert _summary({}) is None + + +class TestBadgeBoundaries: + def test_overall_exactly_on_needs_refinement_lower_bound(self) -> None: + # Equal dimension averages → the weighted overall equals that value exactly. + result = _summary({"ground_truth": 0.3, "knowledge_base": 0.3, "prompt": 0.3}) + assert result is not None + assert result["overall_score"] == 0.3 + assert result["verdict"] == "Needs Refinement" + for dim in result["breakdown"]: + assert dim["verdict"] == "Needs Refinement" + + def test_overall_exactly_on_good_lower_bound(self) -> None: + result = _summary({"ground_truth": 0.6, "knowledge_base": 0.6, "prompt": 0.6}) + assert result is not None + assert result["overall_score"] == 0.6 + assert result["verdict"] == "Good" + for dim in result["breakdown"]: + assert dim["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py new file mode 100644 index 000000000..412be8d2e --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -0,0 +1,179 @@ +"""`generate_run_ai_summary` — the best-effort natural-language note on a run. + +The summary reuses the judge's reasoning-model invocation: params built via +`map_kaapi_to_openai_params` (mocked here to skip its DB lookup) and one +`responses.create` call (the external boundary, also mocked). The Responses `input` +is a qualitative brief (band words + plain area names + consistency phrases + a +repetition line) — never raw scores or the internal "Adherence to X" labels. Every +failure mode (OpenAI error, generic error, empty output) must resolve to None +WITHOUT raising, leaving the deterministic overall to persist. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import openai +import pytest + +from app.crud.evaluations.score import OverallSummary +from app.crud.evaluations.summary import _consistency_read, generate_run_ai_summary + +_MODEL = "gpt-5-mini" + + +def _overall() -> OverallSummary: + return { + "overall_score": 0.66, + "verdict": "Good", + "ai_summary": None, + "breakdown": [ + { + "name": "Adherence to Ground Truth", + "key": "ground_truth", + "score": 0.8, + "weight": 0.5, + "delta": 0.14, + "verdict": "Good", + }, + { + "name": "Adherence to Knowledge Base", + "key": "knowledge_base", + "score": 0.6, + "weight": 0.3, + "delta": -0.06, + "verdict": "Good", + }, + { + "name": "Adherence to Prompt", + "key": "prompt", + "score": 0.4, + "weight": 0.2, + "delta": -0.26, + "verdict": "Needs Refinement", + }, + ], + } + + +def _summary_scores() -> list[dict]: + # std per dimension drives the consistency read; names match the overall's dims. + return [ + {"name": "Adherence to Ground Truth", "avg": 0.8, "std": 0.05}, + {"name": "Adherence to Knowledge Base", "avg": 0.6, "std": 0.15}, + {"name": "Adherence to Prompt", "avg": 0.4, "std": 0.34}, + ] + + +def _responses_result(output_text: str): + return SimpleNamespace(output_text=output_text, output=[]) + + +def _call( + client: MagicMock, + *, + duplication_factor: int = 5, + summary_scores: list[dict] | None = None, +) -> str | None: + # The mapper does a real DB lookup (is_reasoning_model); patch it so the unit + # test stays about the summary logic, not model resolution. + with patch( + "app.crud.evaluations.summary.map_kaapi_to_openai_params", + return_value=({"model": _MODEL, "effort": "medium"}, []), + ): + return generate_run_ai_summary( + session=MagicMock(), + openai_client=client, + model=_MODEL, + overall=_overall(), + run_name="run-x", + summary_scores=summary_scores + if summary_scores is not None + else _summary_scores(), + duplication_factor=duplication_factor, + ) + + +class TestHappyPath: + def test_returns_the_responses_output_text_stripped(self) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result( + " Consistently grounded and on-tone across the set. " + ) + assert _call(client) == "Consistently grounded and on-tone across the set." + + +class TestQualitativeBrief: + def _input_for(self, *, duplication_factor: int) -> str: + client = MagicMock() + client.responses.create.return_value = _responses_result("A note.") + _call(client, duplication_factor=duplication_factor) + return client.responses.create.call_args.kwargs["input"] + + def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( + self, + ) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result("A note.") + _call(client, duplication_factor=5) + + params = client.responses.create.call_args.kwargs + brief = params["input"] + assert params["max_output_tokens"] == 600 + assert "temperature" not in params + + assert "Accuracy against the expected answers" in brief + assert "Grounding in the source material" in brief + assert "Tone and instruction-following" in brief + + assert "answers stayed consistent" in brief # std 0.05 + assert "answers were mostly consistent, with some variation" in brief # 0.15 + assert "answers varied" in brief # 0.34 + + def test_brief_leaks_no_raw_scores_or_internal_labels(self) -> None: + brief = self._input_for(duplication_factor=5) + assert "Adherence to" not in brief + assert "verdict" not in brief + # Raw score / weight / delta values must not cross into the brief. + for leaked in ("0.66", "0.8", "0.5", "0.14", "-0.26"): + assert leaked not in brief + + def test_repetition_line_states_the_repeat_count_when_gt_one(self) -> None: + brief = self._input_for(duplication_factor=5) + assert "Each question was asked 5 times." in brief + + def test_repetition_line_softens_when_asked_once(self) -> None: + brief = self._input_for(duplication_factor=1) + assert "asked once (no repetition to speak of)" in brief + assert "Each question was asked 1 times" not in brief + + +class TestConsistencyRead: + @pytest.mark.parametrize( + ("std", "expected"), + [ + (0.08, "answers stayed consistent"), + (0.15, "answers were mostly consistent, with some variation"), + (0.34, "answers varied"), + (None, "consistency unknown"), + ], + ) + def test_bands_and_boundaries(self, std, expected) -> None: + assert _consistency_read(std) == expected + + +class TestFailureIsNonFatal: + def test_openai_error_returns_none_without_raising(self) -> None: + client = MagicMock() + client.responses.create.side_effect = openai.OpenAIError("provider down") + assert _call(client) is None + + def test_generic_error_returns_none_without_raising(self) -> None: + client = MagicMock() + client.responses.create.side_effect = RuntimeError("unexpected shape") + assert _call(client) is None + + @pytest.mark.parametrize("output_text", ["", " \n\t"]) + def test_empty_or_whitespace_output_returns_none(self, output_text: str) -> None: + client = MagicMock() + client.responses.create.return_value = _responses_result(output_text) + assert _call(client) is None diff --git a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py index 722318ac2..6d37b1f18 100644 --- a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py +++ b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py @@ -15,6 +15,25 @@ ) +# A v2 judge run's deterministic run-level overall — not trace-derived, so the +# trace-merge reconstructions must preserve it rather than rebuild it away. +_OVERALL_BLOCK = { + "overall_score": 0.66, + "verdict": "Good", + "ai_summary": "The run performed well overall.", + "breakdown": [ + { + "name": "Adherence to Ground Truth", + "key": "ground_truth", + "score": 0.8, + "weight": 0.5, + "delta": 0.14, + "verdict": "Good", + } + ], +} + + class TestGetEvaluationWithScoresS3: """Test get_evaluation_with_scores() S3 retrieval.""" @@ -164,6 +183,102 @@ def test_resync_merges_cache_with_langfuse( saved_ids = {t["trace_id"] for t in saved_score["traces"]} assert saved_ids == {"old", "new"} + @patch("app.services.evaluations.evaluation.get_evaluation_run_by_id") + @patch("app.services.evaluations.evaluation.load_json_from_object_store") + @patch("app.services.evaluations.evaluation.get_cloud_storage") + def test_overall_block_survives_cached_trace_serve( + self, + mock_get_storage: MagicMock, + mock_load: MagicMock, + mock_get_eval: MagicMock, + eval_run_factory: Callable[..., MagicMock], + ) -> None: + """v2 judge run: the run-level `overall` block must survive the cached-serve + trace reconstruction (the path the frontend hits), not get dropped when the + score is rebuilt from summary_scores + traces.""" + eval_run = eval_run_factory( + id=200, + status="completed", + score={ + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "overall": _OVERALL_BLOCK, + }, + score_trace_url="s3://bucket/traces.json", + dataset_name="test_dataset", + run_name="test_run", + ) + mock_get_eval.return_value = eval_run + mock_get_storage.return_value = MagicMock() + mock_load.return_value = [{"trace_id": "t1"}] + + result, error = get_evaluation_with_scores( + session=MagicMock(), + evaluation_id=200, + organization_id=1, + project_id=1, + get_trace_info=True, + resync_score=False, + ) + + assert error is None + assert result.score["traces"] == [{"trace_id": "t1"}] + overall = result.score["overall"] + assert overall["overall_score"] == 0.66 + assert overall["verdict"] == "Good" + assert overall["breakdown"] == _OVERALL_BLOCK["breakdown"] + + @patch("app.services.evaluations.evaluation.save_score") + @patch("app.services.evaluations.evaluation.fetch_trace_scores_from_langfuse") + @patch("app.services.evaluations.evaluation.get_langfuse_client") + @patch("app.services.evaluations.evaluation.get_evaluation_run_by_id") + @patch("app.services.evaluations.evaluation.load_json_from_object_store") + @patch("app.services.evaluations.evaluation.get_cloud_storage") + def test_overall_block_survives_resync_merge_and_persists( + self, + mock_get_storage: MagicMock, + mock_load: MagicMock, + mock_get_eval: MagicMock, + mock_get_langfuse: MagicMock, + mock_fetch_langfuse: MagicMock, + mock_save_score: MagicMock, + eval_run_factory: Callable[..., MagicMock], + ) -> None: + """Resync path: the merged score handed to save_score (which re-persists to + DB) must carry the run-level `overall` block, not just the merged traces.""" + eval_run = eval_run_factory( + id=201, + status="completed", + score={ + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "overall": _OVERALL_BLOCK, + }, + score_trace_url="s3://bucket/traces.json", + dataset_name="test_dataset", + run_name="test_run", + ) + mock_get_eval.return_value = eval_run + mock_get_storage.return_value = MagicMock() + mock_load.return_value = [{"trace_id": "old", "scores": []}] + mock_get_langfuse.return_value = MagicMock() + mock_fetch_langfuse.return_value = { + "summary_scores": [], + "traces": [{"trace_id": "new", "scores": []}], + } + mock_save_score.return_value = eval_run + + get_evaluation_with_scores( + session=MagicMock(), + evaluation_id=201, + organization_id=1, + project_id=1, + get_trace_info=True, + resync_score=True, + ) + + saved_score = mock_save_score.call_args.kwargs["score"] + assert {t["trace_id"] for t in saved_score["traces"]} == {"old", "new"} + assert saved_score["overall"] == _OVERALL_BLOCK + @patch("app.services.evaluations.evaluation.save_score") @patch("app.services.evaluations.evaluation.fetch_trace_scores_from_langfuse") @patch("app.services.evaluations.evaluation.get_langfuse_client") From 6900d8acf9c296fef297c1c2365f490867e3a46f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:47:01 +0530 Subject: [PATCH 5/9] fix(*): update the ai_summary code --- backend/app/crud/evaluations/summary.py | 23 ++++--------------- .../crud/evaluations/test_run_ai_summary.py | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index faef41e03..2d69caf0e 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -8,7 +8,6 @@ from openai import OpenAI from sqlmodel import Session -from app.core.config import settings from app.crud.evaluations.judge import JudgeMetricEnum from app.crud.evaluations.response_parsing import extract_response_text from app.crud.evaluations.score import OverallSummary @@ -19,7 +18,9 @@ # Reasoning tokens count against this cap, so it needs headroom beyond the visible # note — too low and reasoning consumes it all, yielding empty output. Length is # bounded by the "1 to 3 sentences" instruction, not by a tight token cap. -_SUMMARY_MAX_OUTPUT_TOKENS: int = 600 +_SUMMARY_MAX_OUTPUT_TOKENS: int = 2000 + +_SUMMARY_REASONING_EFFORT: str = "minimal" # Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain # consistency read. With repeated questions a low spread means the assistant answered @@ -115,17 +116,7 @@ def generate_run_ai_summary( summary_scores: list[dict[str, Any]], duplication_factor: int, ) -> str | None: - """Best-effort one-shot natural-language note on the run's overall quality. - - Reuses the aggregate's existing OpenAI client (never builds a new one) and the - judge's reasoning-model invocation: `map_kaapi_to_openai_params` suppresses - temperature and sets the reasoning effort, then the body goes to the Responses - API. `summary_scores` supplies each metric's std (consistency) and - `duplication_factor` the repeat count that the brief translates into plain - language. A summary is a nicety, so ANY failure — missing key, API error, empty - output — logs a warning and returns None rather than propagating: the - deterministic overall (score/verdict/breakdown) must still persist. - """ + """Best-effort one-shot natural-language note on the run's overall quality.""" user_message = _format_overall_for_prompt( overall=overall, run_name=run_name, @@ -137,7 +128,7 @@ def generate_run_ai_summary( session=session, kaapi_params={ "model": model, - "effort": settings.EVAL_JUDGE_REASONING_EFFORT, + "effort": _SUMMARY_REASONING_EFFORT, }, ) if mapper_warnings: @@ -182,8 +173,4 @@ def generate_run_ai_summary( ) return None - logger.info( - f"[generate_run_ai_summary] Generated run summary | model={model} | " - f"run_name={run_name} | chars={len(summary)}" - ) return summary diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 412be8d2e..f9dc15bba 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -118,7 +118,7 @@ def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( params = client.responses.create.call_args.kwargs brief = params["input"] - assert params["max_output_tokens"] == 600 + assert params["max_output_tokens"] == 2000 assert "temperature" not in params assert "Accuracy against the expected answers" in brief From 5b1074b34dd9f80b8698d6b87e3a13c0bc665d0e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:28:43 +0530 Subject: [PATCH 6/9] fix(evals): clenaups --- backend/app/crud/evaluations/summary.py | 11 ++++------- .../tests/crud/evaluations/test_run_ai_summary.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 2d69caf0e..dc3fc3af5 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -15,16 +15,10 @@ logger = logging.getLogger(__name__) -# Reasoning tokens count against this cap, so it needs headroom beyond the visible -# note — too low and reasoning consumes it all, yielding empty output. Length is -# bounded by the "1 to 3 sentences" instruction, not by a tight token cap. _SUMMARY_MAX_OUTPUT_TOKENS: int = 2000 _SUMMARY_REASONING_EFFORT: str = "minimal" -# Bands for turning a metric's std (spread of its per-row scores, 0-1) into a plain -# consistency read. With repeated questions a low spread means the assistant answered -# the same question the same way each time. _CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 _CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 @@ -142,6 +136,10 @@ def generate_run_ai_summary( "max_output_tokens": _SUMMARY_MAX_OUTPUT_TOKENS, } response = openai_client.responses.create(**params) + # Parse inside the try so a malformed/unexpected Responses payload degrades + # to None like any other failure — the call site has no guard, so an escape + # here would fail the whole run against the best-effort contract. + summary = extract_response_text(response).strip() except openai.OpenAIError as exc: status = getattr(exc, "status_code", None) # 5xx is provider-side (alert-worthy); 4xx/None is caller/Kaapi-side noise. @@ -165,7 +163,6 @@ def generate_run_ai_summary( ) return None - summary = extract_response_text(response).strip() if not summary: logger.warning( f"[generate_run_ai_summary] Empty summary returned | model={model} | " diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index f9dc15bba..813c68368 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -177,3 +177,16 @@ def test_empty_or_whitespace_output_returns_none(self, output_text: str) -> None client = MagicMock() client.responses.create.return_value = _responses_result(output_text) assert _call(client) is None + + def test_malformed_payload_parse_error_returns_none_without_raising(self) -> None: + # extract_response_text runs inside the try, so a raise on an unexpected + # Responses payload must degrade to None, never escape to fail the run. + client = MagicMock() + client.responses.create.return_value = ( + SimpleNamespace() + ) # no output_text/output + with patch( + "app.crud.evaluations.summary.extract_response_text", + side_effect=ValueError("unexpected Responses payload"), + ): + assert _call(client) is None From 28e3b7209143b214692638d357514a42527ec14c Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:42:39 +0530 Subject: [PATCH 7/9] fix(evals): clenaups --- backend/app/crud/evaluations/summary.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index dc3fc3af5..f1a53f6c9 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -22,8 +22,7 @@ _CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 _CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 -# Internal metric key -> plain behaviour phrase, so the brief never leaks the -# "Adherence to X" labels into the model's context. +# Internal metric key -> plain behaviour phrase, so the brief never leaks the labels into the model's context. _DIMENSION_PLAIN_NAME: dict[str, str] = { JudgeMetricEnum.GROUND_TRUTH.value: "Accuracy against the expected answers", JudgeMetricEnum.KNOWLEDGE_BASE.value: "Grounding in the source material", From 0ab017086f61b5e9b702947c540af44ce1f39288 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Mon, 3 Aug 2026 11:27:24 +0530 Subject: [PATCH 8/9] updated wiki --- docs/wiki/modules/evaluations.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index c0119546d..61d496ba7 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -23,11 +23,12 @@ All paths relative to `backend/app/`. Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. +`score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it). ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py`, `batch_job.py`, `validators.py`, `prompt_improvement.py` - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries), `score.py`, `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Responses call, qualitative brief only, no raw scores in the prompt), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async From e2adfcb439590a95289b5c25d12f92190c39a039 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Mon, 3 Aug 2026 16:04:10 +0530 Subject: [PATCH 9/9] cleanups --- backend/app/crud/evaluations/fast.py | 33 +++++++------------ .../tests/crud/evaluations/test_fast_judge.py | 13 -------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 22da3c23b..4876ddc90 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -907,24 +907,6 @@ def _attach_metric_scores( ) -def _resolve_duplication_factor(*, session: Session, eval_run: EvaluationRun) -> int: - """How many times each question was asked, from the run's dataset metadata. - - Feeds only the best-effort AI summary, so it never fails the run: an - unresolvable dataset or missing metadata falls back to 1 (no repetition). - """ - dataset = get_dataset_by_id( - session=session, - dataset_id=eval_run.dataset_id, - organization_id=eval_run.organization_id, - project_id=eval_run.project_id, - ) - if dataset is None: - return 1 - metadata = dataset.dataset_metadata or {} - return max(1, int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1))) - - def _stage3_score_and_trace( *, session: Session, @@ -1147,10 +1129,17 @@ def _stage3_score_and_trace( metric_names={spec.key.value: spec.score_name for spec in metrics}, ) if overall is not None: - # Falls back to 1 (no repetition) if the dataset can't be resolved, so - # the summary still generates. - duplication_factor = _resolve_duplication_factor( - session=session, eval_run=eval_run + # Falls back to 1 (no repetition) if the dataset/metadata can't be + # resolved, so the summary still generates. + dataset = get_dataset_by_id( + session=session, + dataset_id=eval_run.dataset_id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + metadata = dataset.dataset_metadata if dataset else None + duplication_factor = max( + 1, int((metadata or {}).get(DATASET_META_DUPLICATION_FACTOR, 1)) ) overall["ai_summary"] = generate_run_ai_summary( session=session, diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index fb26aea20..58af113f7 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -34,7 +34,6 @@ JOB_TYPE_EVALUATION_FAST_CHUNK, PROMPT_TEMPLATE_LABEL, _format_top_kb_matches, - _resolve_duplication_factor, _responses_call_for_item, run_fast_evaluation, run_response_chunk, @@ -992,18 +991,6 @@ def test_v1_run_has_no_overall_in_score( assert "overall" not in result.score -class TestResolveDuplicationFactor: - """`_resolve_duplication_factor` feeds only the best-effort summary, so a missing - dataset degrades to 1 (no repetition) rather than failing the run.""" - - def test_falls_back_to_one_when_dataset_missing(self) -> None: - with patch("app.crud.evaluations.fast.get_dataset_by_id", return_value=None): - assert ( - _resolve_duplication_factor(session=MagicMock(), eval_run=MagicMock()) - == 1 - ) - - def _responses_item(item_id: str = "item-1") -> dict[str, Any]: return { "id": item_id,