diff --git a/backend/app/api/docs/evaluation/create_evaluation_v2.md b/backend/app/api/docs/evaluation/create_evaluation_v2.md index b09463a7e..6a015b34c 100644 --- a/backend/app/api/docs/evaluation/create_evaluation_v2.md +++ b/backend/app/api/docs/evaluation/create_evaluation_v2.md @@ -3,7 +3,7 @@ body with Kaapi's native LLM-as-Judge built in — v1 is left unchanged. v2 runs are **always fast** and always judged (there is no `run_mode`; batch is deferred to a later phase). Every scoreable row is automatically judged (no opt-in -flag) by one combined LLM-judge call scoring each applicable metric in [0, 1] with +flag) by one combined LLM-judge call scoring each applicable metric on an integer 0–5 scale with reasoning: **Adherence to Ground Truth** (answer conveys the same correct information as the golden answer), **Adherence to Prompt** (answer obeys the assistant's configured instructions; applies only when the run resolves a config diff --git a/backend/app/api/docs/evaluation/improve_prompt_v2.md b/backend/app/api/docs/evaluation/improve_prompt_v2.md index 060820f6c..e75bc3592 100644 --- a/backend/app/api/docs/evaluation/improve_prompt_v2.md +++ b/backend/app/api/docs/evaluation/improve_prompt_v2.md @@ -2,7 +2,7 @@ Enqueue a v2 prompt-recommendation job for the configuration evaluated by a judg Unlike v1 (which reads cosine similarity + correctness), this consumes the native three-metric judge results — **Adherence to Ground Truth**, **Adherence to Prompt**, -and **Adherence to Knowledge Base** — each carrying a 0–1 score and the judge's +and **Adherence to Knowledge Base** — each carrying an integer 0–5 score and the judge's reasoning. The worker reads both the score and the reasoning per metric, focuses the rewrite on rows where Adherence to Prompt / Ground Truth are low, and changes only `completion.params.instructions`; model, knowledge base, and all other settings are diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index c228f447b..164440927 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -151,7 +151,7 @@ class JudgeMetricSpec: @dataclass class MetricScore: - """One metric's outcome for a row: score in [0, 1] and its reasoning.""" + """One metric's outcome for a row: integer score in 0–5 and its reasoning.""" score: float reasoning: str @@ -246,8 +246,11 @@ def _parse_metric_score(key: JudgeMetricEnum, raw: Any) -> MetricScore: raise ValueError( f"metric '{key.value}' score is not a number: {raw.get('score')!r}" ) from exc - if not 0.0 <= score <= 1.0: - raise ValueError(f"metric '{key.value}' score out of [0, 1]: {score}") + if score != int(score) or not 0 <= score <= 5: + raise ValueError( + f"metric '{key.value}' score must be an integer 0-5: {raw.get('score')!r}" + ) + score = int(score) reasoning = str(raw.get("reasoning") or "").strip() if not reasoning: diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 43a8af5b3..2a40e2da0 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -12,21 +12,22 @@ class VerdictEnum(str, Enum): - """Qualitative band derived from a 0–1 judge-metric score.""" + """Qualitative band derived from a 0–5 judge-metric score.""" NEEDS_IMPROVEMENT = "Needs Improvement" NEEDS_REFINEMENT = "Needs Refinement" GOOD = "Good" -VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3 -VERDICT_GOOD_AT_OR_ABOVE: float = 0.6 +VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 2.0 +VERDICT_GOOD_AT_OR_ABOVE: float = 4.0 def verdict_from_score(score: float) -> VerdictEnum: - """Map a 0–1 judge-metric score to its verdict band. + """Map a 0–5 judge-metric score to its verdict band. - Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good. + Boundaries: below 2 → Needs Improvement, 2 to <4 → Needs Refinement, + 4 and above → Good. """ if score < VERDICT_NEEDS_IMPROVEMENT_BELOW: return VerdictEnum.NEEDS_IMPROVEMENT @@ -64,11 +65,15 @@ def verdict_from_score(score: float) -> VerdictEnum: JUDGE_SYSTEM_PREAMBLE: str = ( "You are a strict, impartial evaluator. You score an assistant's answer on the " - "independent metrics listed below in a single pass. Each metric is a float in " - "[0.0, 1.0] with one or two sentences of reasoning. The metrics are independent " - "— judge each only against its own inputs and rules; do not let one metric's " - "verdict bleed into another. Score EVERY metric listed below. Never omit a metric " - "from the output, even if some input blocks are irrelevant to it." + "independent metrics listed below in a single pass. Each metric is an integer " + "score from 0 to 5, where 0 is the worst case (clearly wrong / ungrounded / a hard " + "instruction violation, or no answer at all) and 5 is the best case (fully correct " + "/ fully grounded / fully compliant), with one or two sentences of reasoning. " + "Scores MUST be integers — 0, 1, 2, 3, 4, or 5 — never fractions, decimals, or " + "percentages. The metrics are independent — judge each only against its own inputs " + "and rules; do not let one metric's verdict bleed into another. Score EVERY metric " + "listed below. Never omit a metric from the output, even if some input blocks are " + "irrelevant to it." ) GROUND_TRUTH_JUDGE_PROMPT: str = ( @@ -82,7 +87,22 @@ def verdict_from_score(score: float) -> VerdictEnum: "and that would be wrong, is a factual error.\n" "- Do NOT reward or penalize style, tone, length, or language.\n" "- Do NOT use any outside knowledge; the golden answer is the source of truth.\n" - "- Do NOT answer the question yourself.\n" + "- Do NOT answer the question yourself.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Fully correct and complete. Conveys everything material in the golden answer " + "(a paraphrase, reordering, or additional correct detail is still a 5).\n" + "- 4: Correct and materially complete, but omits one minor, non-essential " + "supporting detail.\n" + "- 3: Partially correct. The core of the answer is right, but at least one material " + "fact is missing, incomplete, or slightly off.\n" + "- 2: Mixed or significantly incomplete. Gets some of the answer right but muddles " + "or omits more than one material fact, or is wrong on a meaningful component while " + "looking plausible on the surface.\n" + "- 1: Mostly incorrect. Contradicts the golden answer on a key point; at most small " + "correct fragments remain.\n" + "- 0: Completely wrong or contradicts the golden answer outright, OR the row has no " + "answer / an errored, empty, or non-responsive output.\n" "Reasoning: name what was correct or what was missing/contradicted.\n" "When scoring THIS metric, consider only these input blocks: Question, " "Generated answer, Golden (reference) answer.\n" @@ -98,8 +118,8 @@ def verdict_from_score(score: float) -> VerdictEnum: "documents, so treat any rule about which source or knowledge base to use (e.g. " "'only use the knowledge base', 'do not use outside information') as satisfied — " "the Knowledge Base metric judges that.\n\n" - "Start from 1.0 and deduct ONLY for a violation of an instruction the block " - "actually states. Never invent a requirement the instructions do not set; a " + 'Start from "no violations" and deduct ONLY for a violation of an instruction the ' + "block actually states. Never invent a requirement the instructions do not set; a " "conditional rule (applies only in a specific situation, e.g. " "'ask for the user's age' or 'if condition Y holds, also mention Z') counts as " "satisfied unless that situation is present in the question. Deduct across " @@ -116,20 +136,25 @@ def verdict_from_score(score: float) -> VerdictEnum: "out-of-scope or disallowed ones as instructed.\n" "3. Fallback compliance — when the instructions define a fallback for the " "unknown/out-of-scope case, the answer uses it instead of ignoring it. Only " - "penalize here for CONTRADICTING an explicit instruction (e.g. skipping the " - "configured fallback, answering a clearly disallowed topic); do not infer " + "penalize here for CONTRADICTING an explicit instruction; do not infer " "fabrication from missing grounding.\n" "4. Format compliance — follows any explicit format rules " "(word limit, structure, opening/closing pattern).\n\n" - "Scoring guide:\n" - "- 1.0: No violation of any stated instruction.\n" - "- 0.7–0.9: One soft miss on a stated rule (e.g. slightly off tone, minor format " - "deviation).\n" - "- 0.4–0.69: One clear violation of an explicit rule.\n" - "- 0.0–0.39: Multiple clear violations, or a hard violation — leaked system " - "prompt, answered a clearly disallowed topic, or hijacked by injection.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: No violation of any stated instruction, across all applicable dimensions.\n" + "- 4: One soft, minor miss on a stated rule (e.g. slightly off tone, minor format " + "deviation) — otherwise compliant.\n" + "- 3: One clear violation of a single explicit rule.\n" + "- 2: Multiple clear violations, or one moderately serious violation spanning more " + "than one dimension.\n" + "- 1: A severe violation of a core instruction (e.g. ignoring a configured fallback " + "on a disallowed ask, a partial injection hijack), but not a full hard violation.\n" + "- 0: A hard violation — leaked system prompt, fully answered a clearly disallowed " + "topic, fully hijacked by injection — OR the row has no answer / an errored, empty, " + "or non-responsive output.\n" "Reasoning: name the specific violated instruction and how the answer violated it. " - "If no stated instruction was violated, say so and score high.\n" + "If no stated instruction was violated, say so and score 5.\n" "When scoring THIS metric, consider only these input blocks: Assistant's " "configured instructions, Question, Generated answer.\n" "Do not consider: Golden (reference) answer, Retrieved knowledge-base chunks." @@ -145,18 +170,30 @@ def verdict_from_score(score: float) -> VerdictEnum: "on the same topic as a chunk, or that requires an inferential leap the chunks do " "not spell out is UNSUPPORTED, not supported.\n" "- Identify the answer's load-bearing (material) claims — the ones that carry its " - "substance. If ANY material claim is unsupported, cap the score at 0.3 regardless " - "of how many minor claims are supported. Otherwise score = supported claims / " - "total factual claims.\n" + "substance.\n" "- Text that makes no factual claim (a greeting, a pleasantry, or a plain refusal " - "to answer) is EXCLUDED from the claim count — do not let it inflate the score.\n" + "to answer) is EXCLUDED from the claim count.\n" "- Judge groundedness ONLY, not correctness, completeness, or " "instruction-following. A claim faithful to the chunks is grounded even if the " "chunks are themselves wrong.\n" "- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed " - "source of support.\n" + "source of support.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Every factual claim is explicitly supported by the retrieved chunks. Fully " + "grounded.\n" + "- 4: All load-bearing claims are grounded; only a minor, non-material claim lacks " + "explicit support.\n" + "- 3: One non-critical inferential leap beyond the chunks, but no load-bearing " + "claim is fabricated.\n" + "- 2: At least one load-bearing/material claim is unsupported or invented, even " + "though other claims are grounded.\n" + "- 1: Most claims are unsupported or invented; only incidental/minor claims are " + "grounded.\n" + "- 0: The answer is fabricated wholesale — no claim is grounded in the retrieved " + "chunks — OR the row has no answer / an errored, empty, or non-responsive output.\n" "Reasoning: quote the exact chunk span supporting the main claim. When the score " - "is below 1.0, name the specific unsupported or invented claim.\n" + "is below 5, name the specific unsupported or invented claim.\n" "When scoring THIS metric, consider only these input blocks: Generated answer, " "Retrieved knowledge-base chunks.\n" "Do not consider: Assistant's configured instructions, Question, Golden " @@ -165,9 +202,10 @@ def verdict_from_score(score: float) -> VerdictEnum: JUDGE_OUTPUT_INSTRUCTION: str = ( "Respond with ONLY a single JSON object mapping each metric key to its result, of " - 'the form {{"": {{"score": , "reasoning": ' - '""}}}}. Include exactly these metric keys: {metric_keys}. ' - "Output nothing else." + 'the form {{"": {{"score": , "reasoning": ' + '""}}}}. Scores MUST be integers 0-5. Every ' + '"reasoning" string MUST be written in English. Include exactly these metric keys: ' + "{metric_keys}. Output nothing else." ) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index f1a53f6c9..08fea05fa 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -19,8 +19,8 @@ _SUMMARY_REASONING_EFFORT: str = "minimal" -_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1 -_CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2 +_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.5 +_CONSISTENCY_MIXED_AT_OR_BELOW: float = 1.0 # Internal metric key -> plain behaviour phrase, so the brief never leaks the labels into the model's context. _DIMENSION_PLAIN_NAME: dict[str, str] = { diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index d7065fdd5..532864841 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -671,7 +671,7 @@ def _draft_improved_prompt( "metric objects; each has `name`, `value`, and `comment`, where:\n" f"- `name` is one of `{GROUND_TRUTH_SCORE_NAME}`, `{PROMPT_SCORE_NAME}`, " f"or `{KNOWLEDGE_BASE_SCORE_NAME}`.\n" - "- `value` is the metric's score from 0 (worst) to 1 (best). Metrics that " + "- `value` is the metric's score, an integer from 0 (worst) to 5 (best). Metrics that " 'could not be scored appear with `value` = "N/A" and `unscoreable` = true; ' "ignore those.\n" "- `comment` is the judge's reasoning for that score — read it, not just " diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index 5daf93ce3..096f4d202 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -76,13 +76,13 @@ "scores": [ { "name": GROUND_TRUTH_SCORE_NAME, - "value": 0.4, + "value": 2, "data_type": "NUMERIC", "comment": _GT_COMMENT, }, { "name": PROMPT_SCORE_NAME, - "value": 0.3, + "value": 1, "data_type": "NUMERIC", "comment": _PROMPT_COMMENT, }, @@ -578,6 +578,9 @@ def test_v2_message_carries_metric_reasoning_and_ignores_na( assert _GT_COMMENT in message assert _PROMPT_COMMENT in message + # The v2 brief describes the judge score on the 0–5 integer scale. + assert "an integer from 0 (worst) to 5 (best)" in message + # The unscoreable KB metric must be presented as ignore-worthy, not a real # score: the prompt tells the model to skip value="N/A" / unscoreable metrics. assert '"N/A"' in message diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 58af113f7..2f164a0d4 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -2,7 +2,7 @@ Drives the ground-truth + adherence-to-prompt slices of the three-metric SRD through the real fast pipeline with a judged run (`is_judge_run=True`, -`langfuse=None` as v2 dispatches): FR-2 (trace scores in [0,1] + reasoning), FR-9 +`langfuse=None` as v2 dispatches): FR-2 (integer 0–5 trace scores + reasoning), FR-9 (zero-config uses the fallback model + built-in prompts), FR-14 (run-level summary scores + per-row scores on the trace records), FR-15 (per-row isolation), FR-16 (judge cost stage), FR-18 (v1 never judges). @@ -168,8 +168,8 @@ def _judge_response(score: float, reasoning: str, *, usage=(12, 6, 18)): def _both_metrics_response( *, - ground_truth: tuple[float, str] = (0.8, "conveys the same facts"), - prompt: tuple[float, str] = (0.6, "answered in the wrong language"), + ground_truth: tuple[float, str] = (4, "conveys the same facts"), + prompt: tuple[float, str] = (3, "answered in the wrong language"), usage=(12, 6, 18), ): gt_score, gt_reason = ground_truth @@ -382,7 +382,7 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.8, "conveys the same facts"), + judge_side_effect=lambda _p: _judge_response(4, "conveys the same facts"), ) assert result.status == "completed" @@ -395,8 +395,8 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( gt = _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME) assert _score_named(traces[ref], COSINE_SCORE_NAME) is None assert gt is not None - assert 0.0 <= gt["value"] <= 1.0 - assert gt["value"] == pytest.approx(0.8, abs=0.01) + assert 0 <= gt["value"] <= 5 + assert gt["value"] == 4 assert gt["comment"] == "conveys the same facts" summary_names = {s["name"] for s in result.score["summary_scores"]} @@ -407,12 +407,12 @@ def test_ground_truth_score_is_the_only_scorer_no_cosine( for s in result.score["summary_scores"] if s["name"] == GROUND_TRUTH_SCORE_NAME ) - assert gt_summary["avg"] == pytest.approx(0.8, abs=0.01) + assert gt_summary["avg"] == pytest.approx(4, abs=0.01) assert gt_summary["total_pairs"] == 2 assert _metric_values(result, GROUND_TRUTH_SCORE_NAME) == { - "item-1": 0.8, - "item-2": 0.8, + "item-1": 4, + "item-2": 4, } run = db.get(EvaluationRun, result.id) @@ -436,7 +436,7 @@ def test_zero_config_judges_with_fallback_model_and_builtin_prompt( def _capture(params): captured.update(params) - return _judge_response(0.6, "partially correct") + return _judge_response(3, "partially correct") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_capture) @@ -469,7 +469,7 @@ def test_per_row_isolation_leaves_failed_row_unscoreable( def _judge(params): if "Q-bad" in params["input"]: return _raw_judge_response("totally not json") - return _judge_response(0.9, "correct") + return _judge_response(5, "correct") result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=_judge @@ -513,9 +513,7 @@ def test_judge_cost_stage_tracked( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response( - 0.7, "close", usage=(12, 6, 18) - ), + judge_side_effect=lambda _p: _judge_response(3, "close", usage=(12, 6, 18)), mock_cost=True, ) @@ -572,7 +570,7 @@ def _judge(params): for ref in ("item-1", "item-2"): prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) assert prompt_score is not None - assert prompt_score["value"] == pytest.approx(0.6, abs=0.01) + assert prompt_score["value"] == 3 assert prompt_score["comment"] == "answered in the wrong language" assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME) is not None @@ -636,7 +634,7 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _both_metrics_response( - ground_truth=(0.9, "same facts"), prompt=(0.25, "answered in English") + ground_truth=(5, "same facts"), prompt=(1, "answered in English") ), ) @@ -645,15 +643,15 @@ def test_per_row_prompt_score_and_reasoning_land_in_the_trace_scores( prompt_score = _score_named(traces[ref], PROMPT_SCORE_NAME) assert prompt_score == { "name": PROMPT_SCORE_NAME, - "value": 0.25, + "value": 1, "data_type": "NUMERIC", "comment": "answered in English", "verdict": "Needs Improvement", } - assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 0.9 + assert _score_named(traces[ref], GROUND_TRUTH_SCORE_NAME)["value"] == 5 prompt_summary = _summary_named(result, PROMPT_SCORE_NAME) - assert prompt_summary["avg"] == 0.25 + assert prompt_summary["avg"] == 1 assert prompt_summary["total_pairs"] == 2 def test_prompt_template_is_appended_to_the_config_prompt_block( @@ -736,7 +734,7 @@ def _assert_only_ground_truth_scored(self, result: EvaluationRun) -> None: trace = _trace_by_ref(result)["item-1"] assert _score_named(trace, PROMPT_SCORE_NAME) is None - assert _score_named(trace, GROUND_TRUTH_SCORE_NAME)["value"] == 0.8 + assert _score_named(trace, GROUND_TRUTH_SCORE_NAME)["value"] == 4 def test_config_without_instructions_drops_the_prompt_metric( self, db: Session, user_api_key: TestAuthContext, _s3_store @@ -758,8 +756,10 @@ def _judge(params): result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) self._assert_only_ground_truth_scored(result) - # The dropped metric leaves no trace in the judge request either. - assert "Adherence to Prompt" not in captured["instructions"] + # The dropped metric's rubric fragment (uniquely marked by its score key) + # leaves no trace in the judge request. The shared preamble still names the + # metric in passing, so match the fragment marker, not the bare label. + assert '(score key "prompt")' not in captured["instructions"] assert "Assistant's configured instructions" not in captured["input"] def test_empty_instructions_drop_the_prompt_metric( @@ -836,7 +836,7 @@ def test_v1_run_produces_no_judge_metrics( def _judge(params): judge_calls.append(params) - return _judge_response(0.9, "should never run") + return _judge_response(5, "should never run") result, fake_openai = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=_judge @@ -887,9 +887,9 @@ def _all_three_judge(self, _params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "prompt": {"score": 0.4, "reasoning": "p"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "prompt": {"score": 2, "reasoning": "p"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) @@ -906,9 +906,9 @@ def test_judged_run_overall_matches_metric_averages( assert result.status == "completed" overall = result.score["overall"] - # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.66. - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + # 4*0.5 + 3*0.3 + 2*0.2 = 2.0 + 0.9 + 0.4 = 3.3. + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" # The successful summary boundary flows into ai_summary. assert overall["ai_summary"] == DEFAULT_RUN_SUMMARY @@ -939,8 +939,8 @@ def test_summary_failure_leaves_overall_intact_with_null_ai_summary( assert result.status == "completed" overall = result.score["overall"] assert overall["ai_summary"] is None - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" assert len(overall["breakdown"]) == 3 def test_overall_survives_into_the_persisted_db_score( @@ -957,8 +957,8 @@ def test_overall_survives_into_the_persisted_db_score( db.expire_all() persisted = db.get(EvaluationRun, result.id).score["overall"] - assert persisted["overall_score"] == 0.66 - assert persisted["verdict"] == "Good" + assert persisted["overall_score"] == 3.3 + assert persisted["verdict"] == "Needs Refinement" assert {dim["key"] for dim in persisted["breakdown"]} == { "ground_truth", "prompt", @@ -984,7 +984,7 @@ def test_v1_run_has_no_overall_in_score( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + judge_side_effect=lambda _p: _judge_response(5, "never runs"), ) assert result.status == "completed" @@ -1084,12 +1084,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) @@ -1099,7 +1099,7 @@ def _judge(params): traces = _trace_by_ref(result) kb_chunked = _score_named(traces["item-chunked"], KNOWLEDGE_BASE_SCORE_NAME) assert kb_chunked is not None - assert kb_chunked["value"] == 0.6 + assert kb_chunked["value"] == 3 # The judged-but-chunkless row surfaces a human N/A placeholder that stays out # of the summary avg (it is not a numeric 0). @@ -1127,18 +1127,18 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.7, "reasoning": "grounded"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "grounded"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) assert kb["data_type"] == "NUMERIC" - assert kb["value"] == 0.7 + assert kb["value"] == 3 # No relevance gate: every retrieved chunk names a match, low scores included. assert ( kb["comment"] @@ -1162,18 +1162,18 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.9, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "partial"}, + "ground_truth": {"score": 5, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "partial"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) assert kb["data_type"] == "NUMERIC" - assert kb["value"] == 0.6 + assert kb["value"] == 3 assert kb["comment"] == "partial | Top matches: a.pdf (55.0%), b.pdf (30.0%)" def test_kb_na_placeholder_stays_out_of_summary_avg( @@ -1195,12 +1195,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "grounded"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "grounded"}, } ) ) - return _judge_response(0.9, "gt only") + return _judge_response(5, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) @@ -1209,8 +1209,8 @@ def _judge(params): for s in result.score["summary_scores"] if s["name"] == KNOWLEDGE_BASE_SCORE_NAME ) - # Only item-scored (0.6) is a real KB score; the N/A row never enters the avg. - assert kb_summary["avg"] == 0.6 + # Only item-scored (3) is a real KB score; the N/A row never enters the avg. + assert kb_summary["avg"] == 3 assert kb_summary["total_pairs"] == 1 def test_non_kb_metric_none_is_skipped_not_placeholdered( @@ -1230,7 +1230,7 @@ def test_non_kb_metric_none_is_skipped_not_placeholdered( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _raw_judge_response( - json.dumps({"knowledge_base": {"score": 0.7, "reasoning": "grounded"}}) + json.dumps({"knowledge_base": {"score": 3, "reasoning": "grounded"}}) ), ) @@ -1259,17 +1259,17 @@ def test_each_scored_judge_metric_carries_its_verdict_band( ] _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) - # One score per band: 0.2 → Needs Improvement, 0.45 → Needs Refinement, - # 0.75 → Good, so the three metrics land in three different bands. + # One score per band: 1 → Needs Improvement, 3 → Needs Refinement, + # 5 → Good, so the three metrics land in three different bands. result, _ = _run_pipeline( db=db, eval_run=eval_run, judge_side_effect=lambda _p: _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.2, "reasoning": "gt"}, - "prompt": {"score": 0.45, "reasoning": "p"}, - "knowledge_base": {"score": 0.75, "reasoning": "kb"}, + "ground_truth": {"score": 1, "reasoning": "gt"}, + "prompt": {"score": 3, "reasoning": "p"}, + "knowledge_base": {"score": 5, "reasoning": "kb"}, } ) ), @@ -1306,7 +1306,7 @@ def test_cosine_score_carries_no_verdict( result, _ = _run_pipeline( db=db, eval_run=eval_run, - judge_side_effect=lambda _p: _judge_response(0.9, "never runs"), + judge_side_effect=lambda _p: _judge_response(5, "never runs"), ) cosine = _score_named(_trace_by_ref(result)["item-1"], COSINE_SCORE_NAME) @@ -1325,12 +1325,12 @@ def _judge(params): return _raw_judge_response( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ) ) - return _judge_response(0.8, "gt only") + return _judge_response(4, "gt only") result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) diff --git a/backend/app/tests/crud/evaluations/test_judge.py b/backend/app/tests/crud/evaluations/test_judge.py index 4503acac9..cb3721e4e 100644 --- a/backend/app/tests/crud/evaluations/test_judge.py +++ b/backend/app/tests/crud/evaluations/test_judge.py @@ -1,7 +1,7 @@ """Unit tests for the native LLM-as-judge scoring primitives (`crud/evaluations/judge.py`). Covers the ground-truth and adherence-to-prompt judge slices of the three-metric -SRD: FR-2 (score in [0,1] + reasoning), FR-3 (ground truth judged against the +SRD: FR-2 (integer 0–5 score + reasoning), FR-3 (ground truth judged against the golden answer), FR-9 (zero-config default prompt), FR-15 (malformed → raises so the row isolates), plus the per-row gating that drops a metric whose inputs the row cannot supply. The single external boundary — the OpenAI judge completion — is @@ -66,24 +66,36 @@ class TestParseJudgeOutput: def test_parses_well_formed_ground_truth(self) -> None: specs = _ALL_METRICS result = _parse_judge_output( - json.dumps( - {"ground_truth": {"score": 0.75, "reasoning": "close paraphrase"}} - ), + json.dumps({"ground_truth": {"score": 4, "reasoning": "close paraphrase"}}), specs, ) assert set(result) == {JudgeMetricEnum.GROUND_TRUTH} score = result[JudgeMetricEnum.GROUND_TRUTH] - assert score.score == 0.75 + assert score.score == 4 assert score.reasoning == "close paraphrase" def test_extracts_json_from_prose_wrapper(self) -> None: specs = _ALL_METRICS wrapped = ( - 'Here is my grade:\n{"ground_truth": {"score": 0.4, "reasoning": ' + 'Here is my grade:\n{"ground_truth": {"score": 2, "reasoning": ' '"missing a key fact"}}\nThanks.' ) result = _parse_judge_output(wrapped, specs) - assert result[JudgeMetricEnum.GROUND_TRUTH].score == 0.4 + assert result[JudgeMetricEnum.GROUND_TRUTH].score == 2 + + def test_integer_score_accepted_fraction_rejected(self) -> None: + # 0–5 is a stepped integer scale: an in-range integer parses, a fraction that + # was legal on the old 0–1 scale is now rejected. + result = _parse_judge_output( + json.dumps({"ground_truth": {"score": 4, "reasoning": "ok"}}), + _ALL_METRICS, + ) + assert result[JudgeMetricEnum.GROUND_TRUTH].score == 4 + with pytest.raises(ValueError, match="integer 0-5"): + _parse_judge_output( + json.dumps({"ground_truth": {"score": 0.5, "reasoning": "x"}}), + _ALL_METRICS, + ) def test_drops_only_the_missing_metric_when_others_present(self) -> None: # Grade against two specs (ground_truth + a synthetic sibling); a reply that @@ -93,7 +105,7 @@ def test_drops_only_the_missing_metric_when_others_present(self) -> None: sibling = replace(gt_spec, key=sibling_key) result = _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.9, "reasoning": "correct"}}), + json.dumps({"ground_truth": {"score": 5, "reasoning": "correct"}}), [gt_spec, sibling], ) assert list(result) == [JudgeMetricEnum.GROUND_TRUTH] @@ -102,9 +114,9 @@ def test_parses_both_metrics_when_both_requested(self) -> None: result = _parse_judge_output( json.dumps( { - "ground_truth": {"score": 0.8, "reasoning": "same facts"}, + "ground_truth": {"score": 4, "reasoning": "same facts"}, "knowledge_base": { - "score": 0.6, + "score": 3, "reasoning": "one unsupported claim", }, } @@ -115,13 +127,13 @@ def test_parses_both_metrics_when_both_requested(self) -> None: JudgeMetricEnum.GROUND_TRUTH, JudgeMetricEnum.KNOWLEDGE_BASE, } - assert result[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.6 + assert result[JudgeMetricEnum.KNOWLEDGE_BASE].score == 3 def test_missing_knowledge_base_leaves_only_ground_truth(self) -> None: # Both metrics requested but the reply omits knowledge_base: that metric is # silently unscoreable for the row — no raise, ground_truth still parsed. result = _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.9, "reasoning": "correct"}}), + json.dumps({"ground_truth": {"score": 5, "reasoning": "correct"}}), [_GROUND_TRUTH_SPEC, _KNOWLEDGE_BASE_SPEC], ) assert set(result) == {JudgeMetricEnum.GROUND_TRUTH} @@ -129,7 +141,7 @@ def test_missing_knowledge_base_leaves_only_ground_truth(self) -> None: def test_raises_when_no_enabled_metric_scored(self) -> None: specs = _ALL_METRICS with pytest.raises(ValueError, match="scored no enabled metric"): - _parse_judge_output(json.dumps({"unrelated": {"score": 0.5}}), specs) + _parse_judge_output(json.dumps({"unrelated": {"score": 3}}), specs) def test_raises_on_empty_response(self) -> None: with pytest.raises(ValueError, match="empty judge response"): @@ -139,17 +151,18 @@ def test_raises_on_non_json(self) -> None: with pytest.raises(ValueError, match="no JSON object"): _parse_judge_output("the answer is basically fine", _ALL_METRICS) - def test_raises_on_score_out_of_range(self) -> None: - with pytest.raises(ValueError, match="out of .0, 1."): + @pytest.mark.parametrize("bad_score", [6, -1]) + def test_raises_on_score_out_of_range(self, bad_score: int) -> None: + with pytest.raises(ValueError, match="integer 0-5"): _parse_judge_output( - json.dumps({"ground_truth": {"score": 1.4, "reasoning": "x"}}), + json.dumps({"ground_truth": {"score": bad_score, "reasoning": "x"}}), _ALL_METRICS, ) def test_raises_on_empty_reasoning(self) -> None: with pytest.raises(ValueError, match="empty 'reasoning'"): _parse_judge_output( - json.dumps({"ground_truth": {"score": 0.8, "reasoning": " "}}), + json.dumps({"ground_truth": {"score": 4, "reasoning": " "}}), _ALL_METRICS, ) @@ -319,7 +332,7 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: with patch( "app.crud.evaluations.judge._create_judge_response", return_value=_judge_response( - {"ground_truth": {"score": 0.9, "reasoning": "same meaning"}} + {"ground_truth": {"score": 5, "reasoning": "same meaning"}} ), ): result = judge_row( @@ -330,7 +343,7 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: ) assert result.metrics[JudgeMetricEnum.GROUND_TRUTH] == MetricScore( - score=0.9, reasoning="same meaning" + score=5, reasoning="same meaning" ) assert result.usage == { "input_tokens": 15, @@ -343,7 +356,7 @@ def test_chunkless_row_judges_ground_truth_only(self, db: Session) -> None: def _capture(_client, params): captured.update(params) - return _judge_response({"ground_truth": {"score": 0.9, "reasoning": "ok"}}) + return _judge_response({"ground_truth": {"score": 5, "reasoning": "ok"}}) with patch( "app.crud.evaluations.judge._create_judge_response", side_effect=_capture @@ -367,8 +380,8 @@ def test_chunk_present_row_judges_both_metrics(self, db: Session) -> None: "app.crud.evaluations.judge._create_judge_response", return_value=_judge_response( { - "ground_truth": {"score": 0.8, "reasoning": "gt"}, - "knowledge_base": {"score": 0.7, "reasoning": "kb"}, + "ground_truth": {"score": 4, "reasoning": "gt"}, + "knowledge_base": {"score": 3, "reasoning": "kb"}, } ), ): @@ -386,7 +399,7 @@ def test_chunk_present_row_judges_both_metrics(self, db: Session) -> None: JudgeMetricEnum.GROUND_TRUTH, JudgeMetricEnum.KNOWLEDGE_BASE, } - assert result.metrics[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.7 + assert result.metrics[JudgeMetricEnum.KNOWLEDGE_BASE].score == 3 def test_chunkless_prompt_is_byte_identical_to_ground_truth_only( self, db: Session @@ -396,7 +409,7 @@ def test_chunkless_prompt_is_byte_identical_to_ground_truth_only( def _capture(_client, params): captured.update(params) - return _judge_response({"ground_truth": {"score": 0.5, "reasoning": "x"}}) + return _judge_response({"ground_truth": {"score": 3, "reasoning": "x"}}) with patch( "app.crud.evaluations.judge._create_judge_response", side_effect=_capture @@ -421,7 +434,7 @@ def test_judge_call_receives_question_answer_and_golden(self, db: Session) -> No def _capture(_client, params): captured.update(params) return _judge_response( - {"ground_truth": {"score": 0.5, "reasoning": "partial"}} + {"ground_truth": {"score": 3, "reasoning": "partial"}} ) with patch( diff --git a/backend/app/tests/crud/evaluations/test_overall_summary.py b/backend/app/tests/crud/evaluations/test_overall_summary.py index a846f89de..257136840 100644 --- a/backend/app/tests/crud/evaluations/test_overall_summary.py +++ b/backend/app/tests/crud/evaluations/test_overall_summary.py @@ -31,11 +31,11 @@ def _summary(metric_avgs: dict[str, float]): class TestAllThreeMetrics: def test_weighted_overall_and_per_dimension_breakdown(self) -> None: - # 0.8*0.5 + 0.6*0.3 + 0.4*0.2 = 0.40 + 0.18 + 0.08 = 0.66. - result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6, "prompt": 0.4}) + # 4*0.5 + 3*0.3 + 2*0.2 = 2.0 + 0.9 + 0.4 = 3.3. + result = _summary({"ground_truth": 4, "knowledge_base": 3, "prompt": 2}) assert result is not None - assert result["overall_score"] == 0.66 - assert result["verdict"] == "Good" + assert result["overall_score"] == 3.3 + assert result["verdict"] == "Needs Refinement" assert result["ai_summary"] is None by_key = {dim["key"]: dim for dim in result["breakdown"]} @@ -49,21 +49,21 @@ def test_weighted_overall_and_per_dimension_breakdown(self) -> None: assert by_key["prompt"]["weight"] == 0.2 assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 - assert by_key["ground_truth"]["score"] == 0.8 - assert by_key["knowledge_base"]["score"] == 0.6 - assert by_key["prompt"]["score"] == 0.4 + assert by_key["ground_truth"]["score"] == 4 + assert by_key["knowledge_base"]["score"] == 3 + assert by_key["prompt"]["score"] == 2 - # delta = dimension score - overall (0.66), sign shows pull vs the run. - assert by_key["ground_truth"]["delta"] == 0.14 - assert by_key["knowledge_base"]["delta"] == -0.06 - assert by_key["prompt"]["delta"] == -0.26 + # delta = dimension score - overall (3.3), sign shows pull vs the run. + assert by_key["ground_truth"]["delta"] == 0.7 + assert by_key["knowledge_base"]["delta"] == -0.3 + assert by_key["prompt"]["delta"] == -1.3 assert by_key["ground_truth"]["verdict"] == "Good" - assert by_key["knowledge_base"]["verdict"] == "Good" + assert by_key["knowledge_base"]["verdict"] == "Needs Refinement" assert by_key["prompt"]["verdict"] == "Needs Refinement" def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None: - result = _summary({"ground_truth": 0.55, "knowledge_base": 0.25, "prompt": 0.9}) + result = _summary({"ground_truth": 2.75, "knowledge_base": 1.25, "prompt": 4.5}) assert result is not None for dim in result["breakdown"]: assert dim["verdict"] == verdict_from_score(dim["score"]).value @@ -71,7 +71,7 @@ def test_per_dimension_verdict_matches_the_rounded_dimension_score(self) -> None class TestRenormalizationWhenAMetricIsAbsent: def test_ground_truth_and_knowledge_base_only(self) -> None: - result = _summary({"ground_truth": 0.8, "knowledge_base": 0.6}) + result = _summary({"ground_truth": 4, "knowledge_base": 3}) assert result is not None by_key = {dim["key"]: dim for dim in result["breakdown"]} assert set(by_key) == {"ground_truth", "knowledge_base"} @@ -82,11 +82,11 @@ def test_ground_truth_and_knowledge_base_only(self) -> None: # overall still sums to 1.0). assert by_key["ground_truth"]["weight"] == 0.62 assert by_key["knowledge_base"]["weight"] == 0.37 - # 0.625*0.8 + 0.375*0.6 = 0.5 + 0.225 = 0.725 → 0.72. - assert result["overall_score"] == 0.72 + # 0.625*4 + 0.375*3 = 2.5 + 1.125 = 3.625 → 3.62. + assert result["overall_score"] == 3.62 def test_ground_truth_and_prompt_only(self) -> None: - result = _summary({"ground_truth": 0.8, "prompt": 0.4}) + result = _summary({"ground_truth": 4, "prompt": 2}) assert result is not None by_key = {dim["key"]: dim for dim in result["breakdown"]} assert set(by_key) == {"ground_truth", "prompt"} @@ -97,10 +97,10 @@ def test_ground_truth_and_prompt_only(self) -> None: assert sum(dim["weight"] for dim in result["breakdown"]) == 1.0 def test_absent_metric_never_drags_the_overall_down(self) -> None: - # A missing metric is dropped, not scored 0: two perfect metrics stay at 1.0. - result = _summary({"ground_truth": 1.0, "knowledge_base": 1.0}) + # A missing metric is dropped, not scored 0: two perfect metrics stay at 5.0. + result = _summary({"ground_truth": 5, "knowledge_base": 5}) assert result is not None - assert result["overall_score"] == 1.0 + assert result["overall_score"] == 5.0 class TestNothingScored: @@ -111,17 +111,17 @@ def test_empty_metric_avgs_returns_none(self) -> None: class TestBadgeBoundaries: def test_overall_exactly_on_needs_refinement_lower_bound(self) -> None: # Equal dimension averages → the weighted overall equals that value exactly. - result = _summary({"ground_truth": 0.3, "knowledge_base": 0.3, "prompt": 0.3}) + result = _summary({"ground_truth": 2, "knowledge_base": 2, "prompt": 2}) assert result is not None - assert result["overall_score"] == 0.3 + assert result["overall_score"] == 2.0 assert result["verdict"] == "Needs Refinement" for dim in result["breakdown"]: assert dim["verdict"] == "Needs Refinement" def test_overall_exactly_on_good_lower_bound(self) -> None: - result = _summary({"ground_truth": 0.6, "knowledge_base": 0.6, "prompt": 0.6}) + result = _summary({"ground_truth": 4, "knowledge_base": 4, "prompt": 4}) assert result is not None - assert result["overall_score"] == 0.6 + assert result["overall_score"] == 4.0 assert result["verdict"] == "Good" for dim in result["breakdown"]: assert dim["verdict"] == "Good" diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 813c68368..68317b87b 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -23,32 +23,32 @@ def _overall() -> OverallSummary: return { - "overall_score": 0.66, - "verdict": "Good", + "overall_score": 3.3, + "verdict": "Needs Refinement", "ai_summary": None, "breakdown": [ { "name": "Adherence to Ground Truth", "key": "ground_truth", - "score": 0.8, + "score": 4, "weight": 0.5, - "delta": 0.14, + "delta": 0.7, "verdict": "Good", }, { "name": "Adherence to Knowledge Base", "key": "knowledge_base", - "score": 0.6, + "score": 3, "weight": 0.3, - "delta": -0.06, - "verdict": "Good", + "delta": -0.3, + "verdict": "Needs Refinement", }, { "name": "Adherence to Prompt", "key": "prompt", - "score": 0.4, + "score": 2, "weight": 0.2, - "delta": -0.26, + "delta": -1.3, "verdict": "Needs Refinement", }, ], @@ -56,11 +56,12 @@ def _overall() -> OverallSummary: def _summary_scores() -> list[dict]: - # std per dimension drives the consistency read; names match the overall's dims. + # std per dimension drives the consistency read (0–5 spread; cutoffs 0.5 / 1.0); + # names match the overall's dims. return [ - {"name": "Adherence to Ground Truth", "avg": 0.8, "std": 0.05}, - {"name": "Adherence to Knowledge Base", "avg": 0.6, "std": 0.15}, - {"name": "Adherence to Prompt", "avg": 0.4, "std": 0.34}, + {"name": "Adherence to Ground Truth", "avg": 4.0, "std": 0.3}, + {"name": "Adherence to Knowledge Base", "avg": 3.0, "std": 0.8}, + {"name": "Adherence to Prompt", "avg": 2.0, "std": 1.5}, ] @@ -125,16 +126,16 @@ def test_input_carries_plain_area_and_consistency_phrases_and_token_cap( assert "Grounding in the source material" in brief assert "Tone and instruction-following" in brief - assert "answers stayed consistent" in brief # std 0.05 - assert "answers were mostly consistent, with some variation" in brief # 0.15 - assert "answers varied" in brief # 0.34 + assert "answers stayed consistent" in brief # std 0.3 + assert "answers were mostly consistent, with some variation" in brief # 0.8 + assert "answers varied" in brief # 1.5 def test_brief_leaks_no_raw_scores_or_internal_labels(self) -> None: brief = self._input_for(duplication_factor=5) assert "Adherence to" not in brief assert "verdict" not in brief # Raw score / weight / delta values must not cross into the brief. - for leaked in ("0.66", "0.8", "0.5", "0.14", "-0.26"): + for leaked in ("3.3", "0.7", "-1.3", "0.5", "0.3"): assert leaked not in brief def test_repetition_line_states_the_repeat_count_when_gt_one(self) -> None: @@ -151,9 +152,11 @@ class TestConsistencyRead: @pytest.mark.parametrize( ("std", "expected"), [ - (0.08, "answers stayed consistent"), - (0.15, "answers were mostly consistent, with some variation"), - (0.34, "answers varied"), + (0.3, "answers stayed consistent"), + (0.5, "answers stayed consistent"), + (0.8, "answers were mostly consistent, with some variation"), + (1.0, "answers were mostly consistent, with some variation"), + (1.5, "answers varied"), (None, "consistency unknown"), ], ) diff --git a/backend/app/tests/crud/evaluations/test_score.py b/backend/app/tests/crud/evaluations/test_score.py index f1d2aa03e..a5e46041e 100644 --- a/backend/app/tests/crud/evaluations/test_score.py +++ b/backend/app/tests/crud/evaluations/test_score.py @@ -1,8 +1,8 @@ """`verdict_from_score` — the per-metric verdict band on v2 judge runs. -Bands are upper-bound exclusive: [0, 0.3) Needs Improvement, [0.3, 0.6) Needs -Refinement, [0.6, 1] Good. The enum serializes by value into the trace JSON, so the -display string is part of the contract. +Bands are upper-bound exclusive on the 0–5 integer scale: [0, 2) Needs Improvement, +[2, 4) Needs Refinement, [4, 5] Good. The enum serializes by value into the trace +JSON, so the display string is part of the contract. """ import pytest @@ -14,12 +14,12 @@ class TestVerdictFromScore: @pytest.mark.parametrize( ("score", "expected"), [ - (0.0, VerdictEnum.NEEDS_IMPROVEMENT), - (0.29, VerdictEnum.NEEDS_IMPROVEMENT), - (0.3, VerdictEnum.NEEDS_REFINEMENT), - (0.59, VerdictEnum.NEEDS_REFINEMENT), - (0.6, VerdictEnum.GOOD), - (1.0, VerdictEnum.GOOD), + (0, VerdictEnum.NEEDS_IMPROVEMENT), + (1, VerdictEnum.NEEDS_IMPROVEMENT), + (2, VerdictEnum.NEEDS_REFINEMENT), + (3, VerdictEnum.NEEDS_REFINEMENT), + (4, VerdictEnum.GOOD), + (5, VerdictEnum.GOOD), ], ) def test_bands_and_boundaries(self, score: float, expected: VerdictEnum) -> None: diff --git a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py index 6d37b1f18..e0b755c77 100644 --- a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py +++ b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py @@ -18,16 +18,16 @@ # A v2 judge run's deterministic run-level overall — not trace-derived, so the # trace-merge reconstructions must preserve it rather than rebuild it away. _OVERALL_BLOCK = { - "overall_score": 0.66, - "verdict": "Good", + "overall_score": 3.3, + "verdict": "Needs Refinement", "ai_summary": "The run performed well overall.", "breakdown": [ { "name": "Adherence to Ground Truth", "key": "ground_truth", - "score": 0.8, + "score": 4, "weight": 0.5, - "delta": 0.14, + "delta": 0.7, "verdict": "Good", } ], @@ -200,7 +200,7 @@ def test_overall_block_survives_cached_trace_serve( id=200, status="completed", score={ - "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 4}], "overall": _OVERALL_BLOCK, }, score_trace_url="s3://bucket/traces.json", @@ -223,8 +223,8 @@ def test_overall_block_survives_cached_trace_serve( assert error is None assert result.score["traces"] == [{"trace_id": "t1"}] overall = result.score["overall"] - assert overall["overall_score"] == 0.66 - assert overall["verdict"] == "Good" + assert overall["overall_score"] == 3.3 + assert overall["verdict"] == "Needs Refinement" assert overall["breakdown"] == _OVERALL_BLOCK["breakdown"] @patch("app.services.evaluations.evaluation.save_score") @@ -249,7 +249,7 @@ def test_overall_block_survives_resync_merge_and_persists( id=201, status="completed", score={ - "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 0.8}], + "summary_scores": [{"name": "Adherence to Ground Truth", "avg": 4}], "overall": _OVERALL_BLOCK, }, score_trace_url="s3://bucket/traces.json", diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 61d496ba7..3bc78e38b 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -22,7 +22,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, thresholds 0.3/0.6), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. +v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Judge metrics score on an **integer 0–5 stepped scale** (the LLM returns integers 0–5; `crud/evaluations/judge.py::_parse_metric_score` enforces it) with English-only reasoning; scores are stored raw (0–5), so the API structure is unchanged but the value range is 0–5 (cosine on the v1 path stays 0–1). Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, 0–5 cutoffs 2/4 → 0–1 Needs Improvement, 2–3 Needs Refinement, 4–5 Good), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. `score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it). ## Services / CRUD