diff --git a/docs/numeric-evidence-completion.md b/docs/numeric-evidence-completion.md new file mode 100644 index 0000000..b087035 --- /dev/null +++ b/docs/numeric-evidence-completion.md @@ -0,0 +1,96 @@ +# Numeric evidence at structured completion + +The first native reliability campaign found a reserved-case evidence regression: +the candidate reported the correct profit in major currency units, but both cited SQL +queries returned only minor units. The conversion appeared in the prose, not executable +output. The independent evidence evaluator correctly rejected the answer; its score and +the original candidate remain unchanged. + +Previously, runtime finalization re-executed every evidence item and rejected execution +errors, but it did not check the reported number against the output. The revised numeric +completion check collects native query cells and successful Python stdout, and requires +the scalar answer to appear in that output. It accepts scientific notation, grouped +numeric text and the existing display-rounding allowance. Booleans and nonfinite values +do not establish a numeric answer. A number in a note or unprinted program source is not +output. Every cited program still runs and any execution failure rejects finalization. + +The existing Pydantic/ToolStrategy retry returns a specific error to the analyst, asking +it to put conversion or rounding into the executable evidence and output the final +number. The host does not guess a currency conversion or rewrite the answer. The check +uses no question-specific expected value, oracle, grading reference or hidden data. +Independent correctness, report, refusal and evidence scoring remain unchanged. Numeric +presence is not proof of correctness or full data lineage. + +The same reserved batch exposed an answered report with unavailable required metrics: +its current-period mean and change were null even though the method explained that the +comparison could not be calculated. Runtime report finalization now rejects null metric +values and asks the analyst to decline with the missing-information reason. It does not +infer answerability from hidden references or fabricate a complete report. Historical +records remain parseable by the permissive persisted contract. + +The initial implementation's offline verification included a real installed +graph that rejects an omitted conversion and then accepts corrected SQL. Additional +tests cover SQL and Python mismatches, printed versus unprinted values, display rounding +and notation, plus partial-report rejection followed by a proper refusal. Ruff, Vulture, +diff and Gitleaks checks passed. This is implementation +evidence, not live qualification of the revised candidate. + +The consumed acceptance-v1 reserved worlds cannot qualify a revision as unseen. The +evaluator builder therefore supports an explicit `reserved_generation=2` for the +acceptance split only. Generation 1 remains the default and reproduces its original +identities/data. Generation 2 uses distinct world IDs, periods, financial values, +report distributions and price/delivery inputs. Thirteen answerable references are +independently checked by SQL; four cases require refusal. Its 17 worlds are controlled +variants, not new customer domains. Old manifests and database files remain intact. + +The initial prepared follow-up used 63 new responses: 12 candidate original regressions, 17 +candidate development responses, and 34 fresh reserved baseline/candidate responses. +The 29 original/development baseline records retain their actual unchanged source and +data/evaluator identities; they are reused rather than relabeled or regenerated. +Models, scoring rules and tolerances remained fixed. Its initial proposed additional +allocation was $2.15, subject to the cumulative allowance. Later staged execution used +the existing allowance as recorded below; no additional spending is approved here. + +Local freeze: `data/output/reliability-native-refresh-20261114-r2/proposal.json` in the +canonical operator checkout. It retains source/manifest/policy/data/reference hashes and +the proposed caps. It superseded the first zero-call preparation, which preceded the +second observed failure; both remain retained. The following checkpoint supersedes +the preparation-only state. No v0.0.1 release is claimed. + +## Generation-2 result and report-binding revision + +The d0d476e04fc7f378 candidate passed 12/12 original and 17/17 development responses, +then 16/17 fresh reserved candidate responses. The generation-2 baseline arm did not +run. All 46 new records are valid; one mandatory reserved result failed evidence +reproduction even though its reported facts were correct. The status label was in the +SQL unit column while the explicitly named metric's value was numeric. The original +data, score and rejected result remain unchanged. + +Runtime finalization now also checks explicit report metric/value associations in SQL +and structured Python JSON/JSON-lines output. A label in a unit column or a matching +value under another metric does not repair a contradictory binding. A second query +with the correct value cannot erase a contradictory cited result. Single wide rows +are recognized; several wide population rows are not arbitrarily mapped to one global +metric. Unstructured Python output still has independent evidence grading. No runtime +code imports oracle answers or the benchmark scorer. + +The existing finalization feedback asks the analyst to repair its executable output. +Real installed-graph regression checks exercise this repair with unrelated local data. +The current code passes 277 tests with 15 optional skips, plus Ruff/Vulture/secret checks. +Code review also preserved SQLite's 0/1 representation for Boolean report values. +The scalar-output and unavailable-metric guards remain in place. The frozen independent +evaluator and tolerances are unchanged; output consistency is not full data lineage or +an independent correctness proof. + +Generation 2 is consumed regression evidence. Explicit generation 3 supplies 17 distinct +reserved worlds for the next frozen candidate, preserving both earlier generations and +all old manifests. Thirteen numerical references are independently checked by SQL; +four scenarios require refusal. No generation-3 model inference has run. + +The completed revised inference used $1.413655 in conservative estimates. Cumulative +goal usage and retained reservations is $4.92681681225 of the existing $5 authorization. +The next full qualification requires 63 new responses and can reuse 29 unchanged +baseline records. A proposed shared $2.05 cap would keep the total below $7, subject +to owner approval; it supersedes the earlier $6 request because another candidate +revision and fresh reserved generation are now required. No new allowance is implied +by this document, and no release acceptance is claimed. diff --git a/src/backintel/answer_validation.py b/src/backintel/answer_validation.py index 5a5c5eb..36d9e17 100644 --- a/src/backintel/answer_validation.py +++ b/src/backintel/answer_validation.py @@ -9,6 +9,7 @@ import json import math +import re from collections.abc import Sequence from langchain.agents.middleware import AgentMiddleware, ModelRequest @@ -21,6 +22,7 @@ from backintel.stores.multi import DIRECTIVE, MultiStore DOMAIN_READS = frozenset({"list_tables", "describe_table", "run_sql", "run_query", "run_python"}) +NUMERIC_OUTPUT = re.compile(r"(? None: metric = row.get("metric") if not isinstance(metric, str) or not metric.strip() or "value" not in row or "unit" not in row: raise ValueError("Each report row requires a nonempty metric identifier, value, and unit.") + if row["value"] is None: + raise ValueError( + "An answered report cannot contain an unavailable metric value. If a required " + "metric cannot be calculated, return status='cannot_answer' and explain the " + "missing information instead of returning a partial answered report." + ) names.append(" ".join(metric.split()).casefold()) if not names or len(names) != len(set(names)): raise ValueError("Report metric identifiers must be unique: return exactly one row per metric.") @@ -118,13 +126,79 @@ def validate_answer_shape(answer: AnalystAnswer) -> None: raise ValueError("An answered response must cite the exact executable evidence.") +def _number_matches(target: float, cell) -> bool: + if cell is None or isinstance(cell, bool): + return False + try: + value = float(cell.replace(",", "") if isinstance(cell, str) else cell) + except (ValueError, TypeError, OverflowError): + return False + return math.isfinite(value) and math.isclose(value, target, rel_tol=1e-4, abs_tol=0.011) + + +def _report_value_matches(value, cell) -> bool: + if isinstance(value, str): + if not isinstance(cell, str): + return False + target, observed = (" ".join(v.split()).casefold() for v in (value, cell)) + return observed == target or target in observed or (len(observed) >= 3 and observed in target) + if isinstance(value, bool): + # SQLite represents Boolean query results as integer 0/1. + return isinstance(cell, (bool, int, float)) and value == cell + return value is not None and _number_matches(float(value), cell) + + +def _collect_report_bindings(bindings: dict[str, list], columns, rows) -> None: + names = [" ".join(name.split()).casefold() for name in columns] + tall = "metric" in names and "value" in names + # A wide table with several rows can describe several populations; its column + # names alone do not bind any one value to the global report metric. + if not tall and len(rows) != 1: + return + for row in rows: + pairs = ( + [(row[names.index("metric")], row[names.index("value")])] + if tall + else zip(names, row, strict=True) + ) + for key, cell in pairs: + if isinstance(key, str) and key.strip(): + bindings.setdefault(" ".join(key.split()).casefold(), []).append(cell) + + +def _collect_python_report_bindings(bindings: dict[str, list], text: str) -> None: + documents = [] + try: + documents.append(json.loads(text)) + except (ValueError, RecursionError): + for line in text.splitlines(): + try: + documents.append(json.loads(line)) + except (ValueError, RecursionError): + continue + rows = [ + row + for document in documents + for row in (document if isinstance(document, list) else [document]) + if isinstance(row, dict) + ] + for row in rows: + if len(rows) > 1 and not {"metric", "value"}.issubset(" ".join(k.split()).casefold() for k in row): + continue + _collect_report_bindings(bindings, list(row), [list(row.values())]) + + def validate_answer_evidence(answer: AnalystAnswer, store: Store, sandbox: PythonSandbox) -> None: """Execute every cited item under the same read-only bindings used by analysis tools. - This is an execution-integrity check, not a correctness/lineage judge. Independent - evaluators still check values, missing/extra requested metrics, and unsupported claims. + Numeric answers must also appear in executed output, allowing display rounding. + Explicit SQL/JSON report bindings may not contradict the reported metric values. + This does not establish correctness or lineage: independent evaluators still check + reference values, missing/extra requested metrics, and unsupported claims. """ problems = [] + numeric_output = [] + report_bindings: dict[str, list] = {} for index, item in enumerate(answer.evidence, 1): try: if item.kind == "python": @@ -136,6 +210,10 @@ def validate_answer_evidence(answer: AnalystAnswer, store: Store, sandbox: Pytho ) if result.exit_code != 0: problems.append(f"Evidence {index}: program failed: {result.stderr[-300:]}") + else: + numeric_output.extend(NUMERIC_OUTPUT.findall(result.stdout)) + if answer.answer_type == "report": + _collect_python_report_bindings(report_bindings, result.stdout) else: code = item.code if isinstance(store, MultiStore): @@ -149,18 +227,39 @@ def validate_answer_evidence(answer: AnalystAnswer, store: Store, sandbox: Pytho code = f"-- store: {item.store}\n{code}" elif item.store is not None and item.store != store.id: raise QueryRefused("Evidence names a store that is not connected.") - store.query(code) + result = store.query(code) + numeric_output.extend(cell for row in result.rows for cell in row) + if answer.answer_type == "report": + _collect_report_bindings(report_bindings, result.columns, result.rows) except (QueryRefused, NotImplementedError) as error: problems.append(f"Evidence {index}: query failed: {str(error)[:300]}") if problems: raise ValueError("; ".join(problems)) + if answer.status == "answered" and answer.answer_type == "report": + for row in answer.rows: + observed = report_bindings.get(" ".join(row["metric"].split()).casefold(), []) + if observed and not all(_report_value_matches(row["value"], cell) for cell in observed): + raise ValueError( + f"Executable output contradicts reported metric {row['metric']!r}. " + "Return that metric's reported value in the value column, with units or " + "formatting in a separate column. A label in the unit column, another " + "metric, or a second query cannot repair a contradictory value binding." + ) + if answer.status == "answered" and answer.answer_type == "number": + if any(_number_matches(answer.value, cell) for cell in numeric_output): + return + raise ValueError( + "Cited evidence executes but does not output the reported numeric value in its " + "reported units. Include any unit conversion or rounding in the executable " + "evidence and output the final value." + ) def answer_schema(store: Store, sandbox: PythonSandbox) -> type[AnalystAnswer]: """Bind runtime-only validation to the native structured-output retry mechanism.""" class VerifiedAnalystAnswer(AnalystAnswer): - """Return an answer with unique metrics and evidence that successfully re-executes.""" + """Return unique metrics, executable evidence and numeric values present in its output.""" @model_validator(mode="after") def verify_before_returning(self): diff --git a/src/backintel/bench/acceptance.py b/src/backintel/bench/acceptance.py index 0c4b6b2..5feb097 100644 --- a/src/backintel/bench/acceptance.py +++ b/src/backintel/bench/acceptance.py @@ -46,9 +46,10 @@ def _fact(name: str, value) -> dict: class _Builder: - def __init__(self, settings: Settings, split: Split, output: Path): + def __init__(self, settings: Settings, split: Split, output: Path, *, generation: int = 1): self.settings, self.split, self.output = settings, split, output - self.dataset = f"backintel_acceptance_v1_{split}" + self.version = VERSION if generation == 1 else f"{VERSION}-reserved-{generation}" + self.dataset = f"backintel_{self.version.replace('-', '_')}_{split}" self.scenarios: list[Scenario] = [] self.worlds: dict[str, dict] = {} self.checks: list[dict] = [] @@ -68,7 +69,7 @@ def add( declaration: dict | None = None, level: int = 3, ) -> None: - world_name = "bi_" + _digest([VERSION, self.split, name])[:16] + world_name = "bi_" + _digest([self.version, self.split, name])[:16] path = self.settings.stores_dir / "sqlite" / f"{world_name}.sqlite" path.parent.mkdir(parents=True, exist_ok=True) # Rebuild only owned, deterministic fixture paths. No existing benchmark store is changed. @@ -94,7 +95,7 @@ def add( if not matches: raise AssertionError(f"Independent SQL reference mismatch: {name}: {value!r} != {want!r}") self.checks.append({"case": name, "sql": sql, "matches": True, "observed": value}) - identifier = f"{VERSION}-{self.split}-{name}" + identifier = f"{self.version}-{self.split}-{name}" self.scenarios.append( Scenario( id=identifier, @@ -124,7 +125,7 @@ def finish(self) -> AcceptanceSuite: data_hash = _digest(self.worlds) manifest_path = self.output / f"{self.split}.json" payload = { - "version": VERSION, + "version": self.version, "dataset": self.dataset, "split": self.split, "reference_hash": reference_hash, @@ -151,9 +152,17 @@ def _financial_cases(b: _Builder, variant: int) -> None: sales = "sales_lines" if not variant else "revenue_entries" expenses = "expense_lines" if not variant else "cost_entries" year, month = (2024, 3) if not variant else (2025, 7) + if variant == 2: + sales, expenses, year, month = "receipts", "expenditures", 2026, 2 + elif variant == 3: + sales, expenses, year, month = "sales_postings", "expense_postings", 2026, 5 period = f"{year}-{month:02d}" start, end = f"{period}-01", f"{year}-{month + 1:02d}-01" numbers = [12000, 8000, -1500, 5000] if not variant else [17003, 9401, -1107, 6309] + if variant == 2: + numbers = [14127, 8263, -2609, 10033] + elif variant == 3: + numbers = [11029, 19241, -3717, 9013] rows = [ (f"E{i + 71}", f"{period}-{day:02d}", amount, "GBP") for i, (day, amount) in enumerate(zip((2, 8, 15, 23), numbers, strict=True)) @@ -164,6 +173,10 @@ def _financial_cases(b: _Builder, variant: int) -> None: coverage_schema = "ledger TEXT, period_start TEXT, period_end_exclusive TEXT, complete INTEGER" revenue = sum(numbers) costs = [10100, 3300, 700] if not variant else [11011, 5401, 1207] + if variant == 2: + costs = [8451, 2067, 999] + elif variant == 3: + costs = [16007, 3901, 1723] cost_rows = [ (f"C{i + 31}", f"{period}-{day:02d}", amount, "GBP") for i, (day, amount) in enumerate(zip((3, 10, 21), costs, strict=True)) @@ -273,6 +286,13 @@ def _report_cases(b: _Builder, variant: int) -> None: review_table = "reviews" if not variant else "feedback" base = datetime(2024 + variant, 2 + variant, 1, 8) durations = [3, 6, 4, 10, 5] if not variant else [2, 7, 6, 12, 4] + people = (1, 1, 2, 3, 3) + if variant == 2: + order_table, review_table = "dispatches", "ratings" + durations, people = [7, 3, 9, 4, 6], (1, 2, 2, 2, 3) + elif variant == 3: + order_table, review_table, base = "consignments", "customer_reviews", datetime(2026, 3, 1, 8) + durations, people = [5, 7, 2, 9, 8], (1, 1, 1, 2, 4) estimates = [4, 5, 5, 8, 5] rows = [ ( @@ -283,7 +303,7 @@ def _report_cases(b: _Builder, variant: int) -> None: (base + timedelta(days=i + d)).isoformat(), (base + timedelta(days=i + estimates[i])).isoformat(), ) - for i, (d, person) in enumerate(zip(durations, (1, 1, 2, 3, 3), strict=True)) + for i, (d, person) in enumerate(zip(durations, people, strict=True)) ] rows.append(("O90", "U4", "delivered", base.isoformat(), None, (base + timedelta(days=9)).isoformat())) rows.append(("O91", "U5", "canceled", base.isoformat(), None, (base + timedelta(days=9)).isoformat())) @@ -296,6 +316,10 @@ def _report_cases(b: _Builder, variant: int) -> None: ("R6", "O25", 5), ("R7", "O90", 2), ] + if variant == 2: + reviews = [(r[0], r[1], score) for r, score in zip(reviews, (4, 1, 5, 3, 2, 4, 1), strict=True)] + elif variant == 3: + reviews = [(r[0], r[1], score) for r, score in zip(reviews, (2, 4, 1, 5, 3, 2, 4), strict=True)] schema = ( "order_id TEXT, customer_person_id TEXT, status TEXT, purchased_at TEXT, delivered_at " "TEXT, estimated_at TEXT" @@ -390,10 +414,18 @@ def _report_cases(b: _Builder, variant: int) -> None: def _replay_cases(b: _Builder, variant: int) -> None: # Different periods and baseline distributions keep the reserved split distinct. year, previous_month = (2024, 5) if not variant else (2025, 8) + if variant == 2: + year, previous_month = 2026, 8 + elif variant == 3: + year, previous_month = 2026, 1 current_month = previous_month + 1 previous, current = f"{year}-{previous_month:02d}", f"{year}-{current_month:02d}" cutoff = datetime(year, current_month + 1, 1) durations = [3, 4, 5, 4] if not variant else [5, 6, 7, 6] + if variant == 2: + durations = [7, 8, 9, 8] + elif variant == 3: + durations = [2, 3, 4, 3] schema = "order_id TEXT, purchased_at TEXT, delivered_at TEXT, estimated_at TEXT" for name, shift, duplicate, delay in ( ("stable", 0, False, "none"), @@ -503,8 +535,13 @@ def observed(period, unique=unique): def _price_replay_cases(b: _Builder, variant: int) -> None: previous, current = ("2024-05", "2024-06") if not variant else ("2025-08", "2025-09") prices = (1000, 2000) if not variant else (1750, 3500) + if variant == 2: + previous, current, prices = "2026-08", "2026-09", (1375, 3125) + elif variant == 3: + previous, current, prices = "2026-01", "2026-02", (1625, 2875) for name, changed in (("mix-only", False), ("shift", True)): - current_prices = (prices[0] * 16 // 10 if changed else prices[0], prices[1]) + multiplier = {2: 17, 3: 18}.get(variant, 16) + current_prices = (prices[0] * multiplier // 10 if changed else prices[0], prices[1]) rows = [ ("L41", previous + "-10", "P11", 100, prices[0]), ("L42", previous + "-11", "P92", 10, prices[1]), @@ -557,13 +594,25 @@ def _price_replay_cases(b: _Builder, variant: int) -> None: def build_acceptance( - settings: Settings, split: Split = "development", *, output: Path | None = None + settings: Settings, + split: Split = "development", + *, + output: Path | None = None, + reserved_generation: int = 1, ) -> AcceptanceSuite: """Build owned immutable SQLite fixtures and hidden reference manifest without any model calls.""" if split not in ("development", "acceptance"): raise ValueError("split must be development or acceptance") - builder = _Builder(settings, split, output or repo_root() / "benchmarks" / "acceptance" / VERSION) - variant = int(split == "acceptance") + if reserved_generation not in (1, 2, 3) or (split != "acceptance" and reserved_generation != 1): + raise ValueError("Reserved generation must be 1, 2 or 3 and applies only to acceptance") + version = VERSION if reserved_generation == 1 else f"{VERSION}-reserved-{reserved_generation}" + builder = _Builder( + settings, + split, + output or repo_root() / "benchmarks" / "acceptance" / version, + generation=reserved_generation, + ) + variant = reserved_generation if split == "acceptance" else 0 _financial_cases(builder, variant) _report_cases(builder, variant) _replay_cases(builder, variant) diff --git a/tests/test_acceptance_suite.py b/tests/test_acceptance_suite.py index f8ce655..6618db9 100644 --- a/tests/test_acceptance_suite.py +++ b/tests/test_acceptance_suite.py @@ -3,6 +3,8 @@ import json import sqlite3 +import pytest + from backintel.bench.acceptance import build_acceptance, verify_acceptance_data from backintel.settings import Settings from backintel.stores import open_store @@ -95,3 +97,58 @@ def test_changed_fixture_is_detected_before_it_can_be_a_comparable_run(tmp_path) with sqlite3.connect(path) as con: con.execute("UPDATE sales_lines SET amount_minor=0") assert not verify_acceptance_data(suite, settings) + + +def test_refreshed_reserved_worlds_preserve_old_evidence_and_have_independent_checks(tmp_path): + settings, original = build(tmp_path, "acceptance") + original_manifest = original.manifest_path.read_bytes() + refreshed = build_acceptance( + settings, "acceptance", output=tmp_path / "fresh-hidden", reserved_generation=2 + ) + assert len(refreshed.scenarios) == 17 + assert original.manifest_path.read_bytes() == original_manifest + assert {s.store for s in original.scenarios}.isdisjoint(s.store for s in refreshed.scenarios) + assert original.data_hash != refreshed.data_hash + assert original.reference_hash != refreshed.reference_hash + assert all(s.id.startswith("acceptance-v1-reserved-2-acceptance-") for s in refreshed.scenarios) + old_profit = next(s for s in original.scenarios if s.id.endswith("-profit-complete")) + new_profit = next(s for s in refreshed.scenarios if s.id.endswith("-profit-complete")) + assert old_profit.expected != new_profit.expected + manifest = json.loads(refreshed.manifest_path.read_text()) + assert len(manifest["reference_checks"]) == sum(s.answerable for s in refreshed.scenarios) + assert all(check["matches"] for check in manifest["reference_checks"]) + assert verify_acceptance_data(original, settings) and verify_acceptance_data(refreshed, settings) + rebuilt = build_acceptance( + settings, "acceptance", output=tmp_path / "fresh-hidden", reserved_generation=2 + ) + assert refreshed.data_hash == rebuilt.data_hash + assert refreshed.reference_hash == rebuilt.reference_hash + + +def test_third_reserved_generation_retains_both_consumed_generations(tmp_path): + settings, first = build(tmp_path, "acceptance") + second = build_acceptance(settings, "acceptance", output=tmp_path / "second", reserved_generation=2) + original_files = {p: p.read_bytes() for p in (first.manifest_path, second.manifest_path)} + third = build_acceptance(settings, "acceptance", output=tmp_path / "third", reserved_generation=3) + assert len(third.scenarios) == 17 + assert all(p.read_bytes() == data for p, data in original_files.items()) + previous_stores = {s.store for suite in (first, second) for s in suite.scenarios} + assert previous_stores.isdisjoint(s.store for s in third.scenarios) + assert third.reference_hash not in {first.reference_hash, second.reference_hash} + manifest = json.loads(third.manifest_path.read_text()) + assert len(manifest["reference_checks"]) == 13 + assert all(c["matches"] for c in manifest["reference_checks"]) + assert all(verify_acceptance_data(suite, settings) for suite in (first, second, third)) + + +@pytest.mark.parametrize("split,generation", [("development", 2), ("acceptance", 4)]) +def test_reserved_refresh_cannot_silently_replace_development_or_choose_arbitrary_worlds( + tmp_path, split, generation +): + with pytest.raises(ValueError, match="Reserved generation"): + build_acceptance( + Settings.from_env({"BACKINTEL_DATA_DIR": str(tmp_path)}), + split, + output=tmp_path / "hidden", + reserved_generation=generation, + ) diff --git a/tests/test_agent_reliability.py b/tests/test_agent_reliability.py index 11a1129..fad8250 100644 --- a/tests/test_agent_reliability.py +++ b/tests/test_agent_reliability.py @@ -138,6 +138,217 @@ def test_native_structured_output_retries_failed_evidence(store): assert len(model.bindings) == 2 +def test_numeric_unit_conversion_is_repaired_through_real_finalization(store): + # Different local data/amounts from the live benchmark; no evaluator truth is used. + bad = response(value=0.1, unit="EUR", method="Convert minor to major units.") + repaired = response( + value=0.1, + unit="EUR", + evidence=[{"kind": "sql", "code": "SELECT SUM(amount) / 100.0 FROM sales"}], + ) + model = ScriptedModel( + responses=[ + tool_call("VerifiedAnalystAnswer", bad, 0), + tool_call("VerifiedAnalystAnswer", repaired, 1), + ] + ) + result = build_agent(model, store, TrustedFixturePython()).invoke( + {"messages": [HumanMessage(content="Report the total in major currency units.")]} + ) + assert result["structured_response"].value == 0.1 + assert len(model.bindings) == 2 + assert any(isinstance(m, ToolMessage) and "unit conversion" in m.content for m in result["messages"]) + + +@pytest.mark.parametrize( + "kind,code", [("sql", "SELECT SUM(amount) * 100 FROM sales"), ("python", "print(1000)")] +) +def test_numeric_output_must_match_the_answer_not_only_execute(store, kind, code): + with pytest.raises(ValidationError, match="reported numeric value"): + answer_schema(store, TrustedFixturePython()).model_validate( + response(evidence=[{"kind": kind, "code": code}]) + ) + + +def test_numeric_claim_in_note_or_program_source_is_not_executed_output(store): + with pytest.raises(ValidationError, match="reported numeric value"): + answer_schema(store, TrustedFixturePython()).model_validate( + response(evidence=[{"kind": "python", "code": "total = 10\nprint('finished')", "note": "10"}]) + ) + + +@pytest.mark.parametrize( + "kind,code,value", + [ + ("sql", "SELECT SUM(amount) / 3 FROM sales", 3.33), + ("python", "print('1e1')", 10), + ("python", "print('10,000.00')", 10000), + ], +) +def test_numeric_output_accepts_display_rounding_and_numeric_notation(store, kind, code, value): + answer_schema(store, TrustedFixturePython()).model_validate( + response(value=value, evidence=[{"kind": kind, "code": code}]) + ) + + +def test_unavailable_report_metric_requires_declined_completion(store): + partial = response( + answer_type="report", + value=None, + rows=[ + {"metric": "observed_total", "value": 10, "unit": "units"}, + {"metric": "current_median", "value": None, "unit": "days"}, + ], + ) + assert AnalystAnswer.model_validate(partial).rows # Historical failures remain gradable. + declined = { + "status": "cannot_answer", + "method": "Checked the assigned observations.", + "reason": "No current-period observations exist to calculate the required median.", + } + model = ScriptedModel( + responses=[ + tool_call("VerifiedAnalystAnswer", partial, 0), + tool_call("VerifiedAnalystAnswer", declined, 1), + ] + ) + result = build_agent(model, store, TrustedFixturePython()).invoke( + {"messages": [HumanMessage(content="Report observed totals and the current median.")]} + ) + assert result["structured_response"].status == "cannot_answer" + assert len(model.bindings) == 2 + assert any(isinstance(m, ToolMessage) and "unavailable metric" in m.content for m in result["messages"]) + + +def test_report_label_must_be_its_metric_value_in_real_finalization(store): + bad = response( + answer_type="report", + value=None, + rows=[{"metric": "status", "value": "STABLE", "unit": "label"}], + evidence=[{"kind": "sql", "code": "SELECT 'status' AS metric, 0 AS value, 'STABLE' AS unit"}], + ) + good = { + **bad, + "evidence": [ + {"kind": "sql", "code": "SELECT 'status' AS metric, 'STABLE' AS value, 'label' AS unit"} + ], + } + model = ScriptedModel( + responses=[tool_call("VerifiedAnalystAnswer", bad, 0), tool_call("VerifiedAnalystAnswer", good, 1)] + ) + result = build_agent(model, store, TrustedFixturePython()).invoke( + {"messages": [HumanMessage(content="Report the observed status with executable evidence.")]} + ) + assert result["structured_response"].rows[0]["value"] == "STABLE" + assert len(model.bindings) == 2 + assert any( + isinstance(m, ToolMessage) and "contradicts reported metric" in m.content for m in result["messages"] + ) + + +@pytest.mark.parametrize( + "code", + [ + "SELECT 'total' AS metric, 5 AS value UNION ALL SELECT 'other', 10", + "SELECT 5 AS total, 10 AS other", + ], +) +def test_another_metric_or_query_cannot_rescue_a_contradictory_report_binding(store, code): + report = response( + answer_type="report", + value=None, + rows=[{"metric": "total", "value": 10, "unit": "units"}], + evidence=[{"kind": "sql", "code": code}, {"kind": "sql", "code": "SELECT 10 AS total"}], + ) + with pytest.raises(ValidationError, match="contradicts reported metric"): + answer_schema(store, TrustedFixturePython()).model_validate(report) + + +@pytest.mark.parametrize( + "code", + [ + "SELECT 'total' AS metric, SUM(amount) AS value FROM sales", + "SELECT SUM(amount) AS total FROM sales", + "SELECT amount AS total FROM sales", + ], +) +def test_explicit_sql_binding_check_preserves_valid_and_unbound_population_outputs(store, code): + answer_schema(store, TrustedFixturePython()).model_validate( + response( + answer_type="report", + value=None, + rows=[{"metric": "total", "value": 10, "unit": "units"}], + evidence=[{"kind": "sql", "code": code}], + ) + ) + + +@pytest.mark.parametrize( + "document", + [ + {"metric": "status", "value": 0, "unit": "STABLE"}, + {"status": 0, "unit": "STABLE"}, + ], +) +def test_structured_python_cannot_put_report_labels_in_the_unit_field(store, document): + program = "print(" + repr(json.dumps(document)) + ")" + with pytest.raises(ValidationError, match="contradicts reported metric"): + answer_schema(store, TrustedFixturePython()).model_validate( + response( + answer_type="report", + value=None, + rows=[{"metric": "status", "value": "STABLE", "unit": "label"}], + evidence=[{"kind": "python", "code": program}], + ) + ) + + +@pytest.mark.parametrize( + "document", + [ + {"status": "STABLE"}, + [{"metric": "status", "value": "STABLE", "unit": "label"}], + ], +) +def test_structured_python_preserves_valid_named_report_values(store, document): + answer_schema(store, TrustedFixturePython()).model_validate( + response( + answer_type="report", + value=None, + rows=[{"metric": "status", "value": "STABLE", "unit": "label"}], + evidence=[{"kind": "python", "code": "print(" + repr(json.dumps(document)) + ")"}], + ) + ) + + +def test_json_lines_report_checks_named_values_across_output_lines(store): + lines = [ + json.dumps({"metric": "status", "value": "STABLE"}), + json.dumps({"metric": "count", "value": 10}), + ] + code = "\n".join("print(" + repr(line) + ")" for line in lines) + answer_schema(store, TrustedFixturePython()).model_validate( + response( + answer_type="report", + value=None, + rows=[{"metric": "status", "value": "STABLE", "unit": "label"}], + evidence=[{"kind": "python", "code": code}], + ) + ) + + +@pytest.mark.parametrize("value,sql_value", [(False, 0), (True, 1)]) +def test_sql_boolean_representation_remains_a_valid_report_binding(store, value, sql_value): + answer_schema(store, TrustedFixturePython()).model_validate( + response( + answer_type="report", + value=None, + rows=[{"metric": "flag", "value": value, "unit": "boolean"}], + evidence=[{"kind": "sql", "code": f"SELECT 'flag' AS metric, {sql_value} AS value"}], + ) + ) + + def test_failed_finalization_remains_an_execution_failure(store): model = ScriptedModel( responses=[