From 74803c2ff542cdb3638fe0dc0cae82fa26aa2a10 Mon Sep 17 00:00:00 2001 From: arpitkumar2004 Date: Wed, 2 Sep 2026 02:27:37 +0530 Subject: [PATCH 1/3] properly connected the pipeline: ready for the full execution. Excludes the evaluation corpus build output: `.gitignore` now allowlists what may be committed under `data/` instead of denying specific directories. The previous rule was `data/eval*/corpus/`, which matched `data/eval/corpus/` but not the second corpus at `data/eval/corpus_no_layout/`, so 3.3 GB of regenerable output was staged and GitHub rejected the push on four files between 783 MB and 878 MB. `queries.json` and `provenance.json` are still committed - they are the dataset definition, not output. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 7 + .gitignore | 30 +- CHANGELOG.md | 47 + Documentations/eval_method.md | 24 +- configs/config.yaml | 27 + data/eval/filings/provenance.json | 120 + data/eval/queries.json | 2684 +++++++++++++++++ data/eval_small/filings/provenance.json | 120 + data/eval_small/queries.json | 670 ++++ docureason/pipeline.py | 41 +- docureason/tripath/config.py | 143 +- docureason/tripath/evaluation/ablation.py | 641 +++- docureason/tripath/evaluation/baselines.py | 373 +++ docureason/tripath/evaluation/edgar.py | 282 ++ docureason/tripath/evaluation/eval_harness.py | 117 +- docureason/tripath/evaluation/figures.py | 929 ++++++ docureason/tripath/evaluation/gold_queries.py | 495 +++ docureason/tripath/evaluation/statistics.py | 316 ++ docureason/tripath/indexing/dense_index.py | 70 +- docureason/tripath/indexing/sparse_index.py | 20 +- docureason/tripath/indexing/table_store.py | 348 +++ .../tripath/ingestion/figure_captioner.py | 70 +- docureason/tripath/ingestion/ocr_fallback.py | 95 +- docureason/tripath/ingestion/rasterizer.py | 230 ++ .../tripath/retrieval/hybrid_retriever.py | 181 +- docureason/tripath/retrieval/index_backed.py | 28 +- docureason/tripath/retrieval/ranker.py | 50 +- docureason/tripath/retrieval/table_sql.py | 108 +- docureason/tripath/serving/query_service.py | 116 +- docureason/tripath/utils/__init__.py | 3 + docureason/tripath/utils/logger.py | 15 +- docureason/tripath/utils/timing.py | 108 + pyproject.toml | 15 +- requirements.lock | 1 + scripts/__init__.py | 7 + scripts/build_eval_corpus.py | 201 ++ scripts/build_kaggle_eval_notebook.py | 882 ++++++ scripts/check_metric_regression.py | 153 + scripts/plot_results.py | 66 + scripts/run_ablations.py | 209 ++ scripts/run_eval_suite.py | 231 ++ .../fixtures/metric_regression_baseline.json | 113 + tests/fixtures/metric_regression_fixture.json | 1088 +++++++ tests/tripath/test_ablation_runner.py | 330 ++ tests/tripath/test_eval_suite_runner.py | 100 + tests/tripath/test_figures.py | 244 ++ tests/tripath/test_gold_queries.py | 184 ++ tests/tripath/test_metric_regression_gate.py | 94 + tests/tripath/test_naive_baseline.py | 177 ++ tests/tripath/test_naive_baseline_models.py | 61 + tests/tripath/test_rasterizer.py | 164 + tests/tripath/test_response_cache.py | 233 ++ tests/tripath/test_statistics.py | 127 + tests/tripath/test_table_store.py | 150 + 54 files changed, 13169 insertions(+), 169 deletions(-) create mode 100644 data/eval/filings/provenance.json create mode 100644 data/eval/queries.json create mode 100644 data/eval_small/filings/provenance.json create mode 100644 data/eval_small/queries.json create mode 100644 docureason/tripath/evaluation/baselines.py create mode 100644 docureason/tripath/evaluation/edgar.py create mode 100644 docureason/tripath/evaluation/figures.py create mode 100644 docureason/tripath/evaluation/gold_queries.py create mode 100644 docureason/tripath/evaluation/statistics.py create mode 100644 docureason/tripath/indexing/table_store.py create mode 100644 docureason/tripath/ingestion/rasterizer.py create mode 100644 docureason/tripath/utils/timing.py create mode 100644 scripts/__init__.py create mode 100644 scripts/build_eval_corpus.py create mode 100644 scripts/build_kaggle_eval_notebook.py create mode 100644 scripts/check_metric_regression.py create mode 100644 scripts/plot_results.py create mode 100644 scripts/run_ablations.py create mode 100644 scripts/run_eval_suite.py create mode 100644 tests/fixtures/metric_regression_baseline.json create mode 100644 tests/fixtures/metric_regression_fixture.json create mode 100644 tests/tripath/test_ablation_runner.py create mode 100644 tests/tripath/test_eval_suite_runner.py create mode 100644 tests/tripath/test_figures.py create mode 100644 tests/tripath/test_gold_queries.py create mode 100644 tests/tripath/test_metric_regression_gate.py create mode 100644 tests/tripath/test_naive_baseline.py create mode 100644 tests/tripath/test_naive_baseline_models.py create mode 100644 tests/tripath/test_rasterizer.py create mode 100644 tests/tripath/test_response_cache.py create mode 100644 tests/tripath/test_statistics.py create mode 100644 tests/tripath/test_table_store.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57c4f30..886f381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,6 +77,13 @@ jobs: key: hf-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} restore-keys: hf-${{ runner.os }}- + - name: Metric regression gate + # 359 lines of metric arithmetic had no test that would notice a change. This + # scores a frozen fixture (no models, no network) and fails on any movement + # beyond one percentage point. It is a blocking gate, not advisory: an advisory + # metric check reads as enforced and is worse than none (audit P3-6). + run: python scripts/check_metric_regression.py --tolerance 0.01 + - name: Run pytest with coverage run: | pytest --cov=docureason --cov-report=xml --cov-report=term-missing tests/ diff --git a/.gitignore b/.gitignore index 0bdc19d..07d4807 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ plan/ artifacts/ -notebooks/ +# notebooks/* rather than notebooks/: git cannot re-include a file inside an excluded +# directory, so the negation below would never fire against a directory pattern. +notebooks/* +# ...except the generated Kaggle runner: it is the only way to reproduce the full +# evaluation without a local GPU, so it is worth reviewing. Regenerate it with +# scripts/build_kaggle_eval_notebook.py rather than editing the JSON. +!notebooks/docureason_full_eval_kaggle.ipynb .ipynb_checkpoints/ @@ -223,3 +229,25 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# --------------------------------------------------------------------------- +# Evaluation corpus build output (scripts/build_eval_corpus.py) +# --------------------------------------------------------------------------- +# Ingested corpora and raw filings are large (1.2 GB for eight 10-Ks) and fully +# regenerable: provenance.json carries every filing URL plus its SHA-256, and the +# corpus is a deterministic function of those filings and the pipeline config. +# +# queries.json and provenance.json are NOT ignored — together they are the dataset +# definition, and a query set that lives only on one machine cannot be evaluated +# against by anyone else. +# This is an allowlist, not a denylist, on purpose: the previous rule was +# `data/eval*/corpus/`, which matched data/eval/corpus/ but not the second corpus at +# data/eval/corpus_no_layout/, so 3.3 GB of build output was committed and GitHub +# rejected the push. Naming what may be committed cannot miss a new directory name. +data/* +!data/eval*/ +data/eval*/* +!data/eval*/queries.json +!data/eval*/filings/ +data/eval*/filings/* +!data/eval*/filings/provenance.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 141b0b6..ceb99dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,53 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +### Measurability Release — audit remediation (Sprint 2) + +Continues [`Documentations/audit.md`](Documentations/audit.md) §13, Sprint 2. Sprint 1 made the architecture real; this sprint makes it **measurable** — a real corpus, a baseline to compare against, executed ablations, and intervals around every number. + +#### Added — evaluation infrastructure +- **A real evaluation corpus from SEC EDGAR** (`docureason/tripath/evaluation/edgar.py`, `scripts/build_eval_corpus.py`). Everything scoreable before came from eight fixture files totalling ~3 KB with invented answers (`benchmark_dataset.py` asserted `"answer": "2.4M"` against a table containing 100M and 125M), so no retrieval number was falsifiable. The fetcher pulls 10-K primary documents with the SEC-required `User-Agent` (`DOCUREASON_SEC_USER_AGENT`, no default — requests are refused rather than sent under a fabricated identity), rate-limits itself well under the 10 req/s fair-access limit, caches by accession number, and writes `provenance.json` with each filing's URL, accession, filing date and SHA-256. +- **Stratified gold queries whose labels are derivable** (`docureason/tripath/evaluation/gold_queries.py`). Table-cell queries know their gold region by construction (row label × column header); prose and figure queries are anchored on a phrase **verified unique across the whole corpus**, and a candidate whose anchor is not unique is discarded rather than emitted — scoring a correct region as a miss would measure the label format, not retrieval. Rejections are counted by reason, and every record carries `provenance: "programmatic"`: these are derived labels, not human annotation, which over-represents lookup and cannot express multi-hop reasoning. +- **The naive-RAG baseline** (`docureason/tripath/evaluation/baselines.py`). The docs quoted improvements over "naive RAG" with no naive RAG in the repository, so nothing achieved could be attributed to the tri-path architecture rather than the corpus or the generator. Recursive character splitting, one flat exact index, top-5, **the same generator and the same pinned embedding model** — only the retrieval architecture differs. Chunks record every region they overlap, so a splitter that cuts across gold units is not penalised for the label format. +- **A real ablation runner** (`docureason/tripath/evaluation/ablation.py`). The module it replaces subtracted one pre-computed `average_recall_at_5` from another and had **zero callers**, while `Documentations/eval_method.md` §21 published seven deltas to one decimal place that nothing in the codebase could have produced. Each ablation now re-runs every query with the component disabled and reports a **paired** comparison: mean delta, bootstrap CI and a sign-flip p-value. Each row carries its published claim beside the measurement, and an ablation that could not be executed is reported `executed: false` with a reason — never as a delta of zero. +- **Five ablation switches that did not exist.** `retrieval.enable_sparse_retrieval`, `retrieval.enable_table_sql`, `reranker.enable_cross_encoder`, `reranker.enable_heading_penalty` and `ingestion.enable_layout_parser`. Four of the seven documented ablations were not operations the system supported: there was no way to turn those components off, so "remove X and re-measure" could only ever have been asserted. Each switch is also reported through the `degradation` channel, because an ablated response still means something different from a healthy one. +- **Bootstrap intervals and paired significance testing** (`docureason/tripath/evaluation/statistics.py`). Percentile bootstrap for a metric's mean, paired bootstrap for the difference between two systems, and an exact sign-flip permutation test (enumerated for n ≤ 14, sampled with an add-one correction above it, so a sampled p-value is never reported as 0). Paired because query-to-query variance dwarfs between-system variance. Pure Python against a seeded `random` — the CI gate needs determinism more than speed. +- **`confidence_intervals`, `stage_latency_ms` and `per_stratum` in every suite report.** A corpus-wide mean can hold steady while tabular lookup collapses and prose improves; the strata are the only place that shows. +- **Per-stage latency instrumentation** (`docureason/tripath/utils/timing.py`). Reported latency was one number measured *around* the call (audit P0-5), so an embedding reload, a cross-encoder cold start and a DuckDB retry were indistinguishable. `StageTimer` accumulates `route`, `dense_search`, `sparse_search`, `index_merge`, `corpus_scan`, `table_sql`, `fuse`, `chart_enrich`, `parent_expand`, `rerank`, `generate` and `attribute`, and reports a **signed residual** — total minus the sum of the stages — so a decomposition with a hole in it says so instead of quietly describing 60% of the request. A negative residual is the parallel dispatch overlapping, and is reported rather than clamped. +- **A metric regression gate in CI** (`scripts/check_metric_regression.py`, `tests/fixtures/`). 359 lines of metric arithmetic had no test that would notice a change. A frozen fixture — 20 cases across four strata, fixed retrieval output, no models, no network — is scored and compared against a committed baseline of 99 values; anything moving more than 0.01 fails the build. Latency and throughput are excluded because they measure the runner. It is a **blocking** gate: an advisory metric check reads as enforced and is worse than none (P3-6). +- **DuckDB tables registered at ingestion** (`docureason/tripath/indexing/table_store.py`). `TableSQLRetriever` re-created every table it touched on **every query** — one `CREATE TABLE` plus one `INSERT` per row into a throwaway `:memory:` database — and re-inferred column types each time, so the same column could be `DECIMAL` for one question and `VARCHAR` for the next. Tables are now registered once into `tables.duckdb` with `tables_index.json` beside the corpus, and queries run against a read-only connection. On the 8-filing corpus that is **1,952 tables / 17,310 rows** prepared once instead of per query. A region absent from the index falls back to the in-memory path, so a partially built store degrades rather than failing. +- **Parallel modality dispatch** (`serving.enable_parallel_paths`). The three retrieval paths are independent and now run concurrently, with results collected in a fixed task order so ranking is unchanged. A failing path no longer takes the query down: it returns empty, is logged, and surfaces as `mode: "failed"` in that modality's stats. The dense and sparse index builders gained load locks and a metadata cache — without them three threads could each load the embedding model, and `search()` re-read the whole metadata sidecar from disk on every query. +- **A bounded response cache** (`serving.response_cache_size`, default 64). Keyed on the query **and** a fingerprint of the effective config, so a per-query override or an ablation switch can never be served a response computed under the opposite setting. A hit is marked `cached: true` and its `latency_breakdown` describes the **cache lookup**, with the original run preserved under `latency_breakdown_first_run` — replaying the original timings would credit the pipeline with work it did not do and drag every reported percentile toward the cache's. +- **`ServingConfig`.** `configs/serving.yaml` existed but was never loaded (`load_from_yaml` reads `config.yaml` only), so both settings above were unreachable configuration for behaviour the service did not have. + +#### Fixed +- **Per-query overrides mutated the shared service.** `apply_query_overrides` mutated `self` and returned it, so one request carrying `{"final_top_k": 1}` permanently narrowed the `QueryService` every later request used, and an ablation run left its own switches behind for the next one. It now returns a deep copy. Overrides are also driven by a single table shared with the REST layer and the ablation definitions, and an unknown key is recorded in `source_map["unknown_overrides"]` instead of silently doing nothing. +- **SEC filings were parsed without a layout parser.** `_build_regions` routed only `.pdf/.docx/.pptx/.xlsx` to Docling, so an HTML filing reached the rule-based segmenter, which saw raw markup and produced **no table regions at all**. `.html`/`.htm` are now layout-parsed: one Apple 10-K yields 62 tables and 1,107 body regions where it previously yielded neither. +- **Figures were captioned from the wrong image, or from a blank one** (P1-3, P1-4). The captioner returned `page.images[0]` for *every* figure on a page, so three charts on one page were all described from whichever image came first; when a page had no embedded image it built a **blank white canvas**, cropped it and captioned that, and the result was indexed as a figure. A new `PageRasterizer` (`pypdfium2`) renders the page and crops each figure at its own bbox, and a crop with no visual variation is refused and counted as `blank_crop_skipped`. +- **OCR read only the first embedded image per page** (P2-13), so a scan split across two images OCR'd to half its text — and a page with no embedded image had a blank canvas handed to EasyOCR, costing seconds per page to read nothing. Pages are now rendered and blank renders are skipped before OCR runs. +- **`image_rendering_dpi` had no consumer.** It was in the config, in `config.yaml` and in the startup banner, and reached no code (§6). It is now the rasterization DPI for both figure crops and OCR. +- **The scanned-page detector was off by one page** (P1-2). Character counts were accumulated under Docling's 1-based `page_no` and looked up under a 0-based page loop index, so page *N*'s text was compared against page *N+1*'s content. Conversion now happens in one place (`page_index_from_region`), and OCR regions are emitted 1-based to match Docling. +- **A pinned model revision turned a network hiccup into an empty index.** Pinning makes the loader resolve that commit against the hub; when the hub was unreachable but the commit was already cached, the load failed outright and the dense index silently degraded. Both loaders now retry with `local_files_only=True` before giving up. +- **Windows console logging lost every message containing an arrow or em dash.** `UnicodeEncodeError` inside `logging` printed a traceback instead of the line, so ingestion logs were unreadable exactly where they mattered. The stream handler is now reconfigured to UTF-8. +- **`bootstrap_ci` iteration count is a parameter.** The ablation runner computes intervals for eight systems and every stratum; at the default 10,000 resamples that dominated the sweep. + +#### Measured, on 8 real 10-K filings +| | | +| :--- | ---: | +| filings (Apple, Microsoft, JPMorgan, J&J, Exxon, Walmart, Caterpillar, Pfizer) | 8 | +| regions | 20,966 | +| chunks | 24,919 | +| DuckDB tables registered / rows | 1,952 / 17,310 | +| gold queries (tabular_lookup / numeric / prose) | 150 (50/50/50) | +| gold-id and anchor-uniqueness validation | PASS | +| hermetic unit tests | 202 (was 95) | + +#### Known gaps after this sprint +- **The `figure` stratum is empty**: Docling's HTML pipeline emits no figure regions for these filings, so the corpus cannot support figure queries. Recorded in the generation report as `figure:no_regions_in_corpus` rather than as a silent zero. +- **The corpus is 8 filings, not the 20–50 the roadmap calls for.** Ingestion is ~2.4 min/filing on CPU and gold ids only exist after it; `scripts/build_eval_corpus.py --filings 24` scales without code changes. +- **`no_layout_parser` cannot be measured from a single corpus.** It is ingestion-scoped, so it needs a second corpus built with `enable_layout_parser: false`; the runner reports it as not executed until one is supplied. +- Gold labels are **programmatically derived, not annotated**. Real NLI entailment (P0-2), authentication (S1) and the remaining P1 findings are unchanged. + ### Architecture Release — audit remediation (Sprint 1) Continues the remediation of [`Documentations/audit.md`](Documentations/audit.md) (§13, Sprint 1). diff --git a/Documentations/eval_method.md b/Documentations/eval_method.md index 3c567dd..353b003 100644 --- a/Documentations/eval_method.md +++ b/Documentations/eval_method.md @@ -595,6 +595,25 @@ Quantifies the exact contribution of each architectural block by measuring perfo $$\Delta \text{Metric} = \text{Metric}_{\text{Full\_Pipeline}} - \text{Metric}_{\text{Ablated\_Pipeline}}$$ +> **Status of the numbers in this table.** The deltas below were published before any +> runner existed that could produce them, and four of the seven components had no switch +> to turn them off, so "remove X and re-measure" was not an operation this system +> supported. Treat every figure in the last two columns as an **unverified expectation**. +> +> `docureason/tripath/evaluation/ablation.py` now executes the matrix for real: +> +> ```bash +> python scripts/run_ablations.py --queries data/eval/queries.json --limit 20 --stratified +> ``` +> +> It re-runs every query with the component disabled, reports a paired delta with a +> bootstrap confidence interval and a sign-flip p-value, and prints each measured delta +> **beside the claim below** with a verdict (`consistent_in_magnitude`, `same_direction_different_magnitude`, +> `contradicted`, `not_significant`, `not_measurable`). `no_layout_parser` is +> ingestion-scoped and is reported as *not executed* until a corpus ingested with +> `ingestion.enable_layout_parser: false` is supplied. Where the measurement and this +> table disagree, the measurement is the result. + #### Master Ablation Test Matrix Table | Ablated Component | Primary Metric Impact | Expected Performance Drop ($\Delta$) | Diagnostic Conclusion | @@ -666,6 +685,7 @@ All evaluation modules are located in `docureason/tripath/evaluation/` and `docu # Execute automated evaluation harness python scripts/evaluate_system.py -# Run ablation studies -python -c "from docureason.tripath.evaluation.ablation import AblationStudy; AblationStudy().run()" +# Run ablation studies (the old AblationStudy().run() had no callers and computed +# nothing; it now raises and points here) +python scripts/run_ablations.py --queries data/eval/queries.json --limit 20 --stratified ``` diff --git a/configs/config.yaml b/configs/config.yaml index afd80d5..58ad3a8 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -123,3 +123,30 @@ attribution: entailment_threshold: 0.5 # Minimum probability to classify claim as entailed flag_threshold_precision: 0.5 # Response precision threshold to flag 'needs_review' enable_nli: true # Enable automated hallucination verification pass + +# ------------------------------------------------------------------------------ +# 8. Online Serving Layer +# ------------------------------------------------------------------------------ +# These two settings previously lived in configs/serving.yaml, which load_from_yaml +# never read (it loads config.yaml only), so both described behaviour the service did +# not have. +serving: + response_cache_size: 64 # Distinct (query, config) responses kept per service; 0 disables + enable_parallel_paths: true # Run the text/table/vision retrievers concurrently + +# ------------------------------------------------------------------------------ +# Ablation switches (Documentations/eval_method.md §21) +# ------------------------------------------------------------------------------ +# Left at their defaults here and flipped per query by scripts/run_ablations.py. Each +# one names a component the ablation matrix claims a delta for; before these existed, +# four of the seven rows could not be measured because the component could not be +# turned off. Setting any of them to false is reported in the response `degradation` +# list, so an ablated run is never mistaken for a healthy one: +# +# ingestion.enable_layout_parser false -> rule-based segmentation (needs re-ingestion) +# retrieval.enable_sparse_retrieval false -> dense-only retrieval +# retrieval.enable_table_sql false -> no DuckDB text-to-SQL path +# reranker.enable_cross_encoder false -> token-similarity fallback reranker +# reranker.enable_heading_penalty false -> no short-heading penalty +# reranker.enable_parent_expansion false -> child chunks only +# attribution.enable_nli false -> no claim-support verification diff --git a/data/eval/filings/provenance.json b/data/eval/filings/provenance.json new file mode 100644 index 0000000..690129c --- /dev/null +++ b/data/eval/filings/provenance.json @@ -0,0 +1,120 @@ +{ + "source": "SEC EDGAR", + "form": "10-K", + "filing_count": 8, + "user_agent": "DocuReason evaluation corpus builder (kumararpit17773@gmail.com)", + "filings": [ + { + "company": "Apple", + "cik": 320193, + "form": "10-K", + "filing_date": "2025-10-31", + "accession": "0000320193-25-000079", + "document": "aapl-20250927.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019325000079/aapl-20250927.htm", + "path": "data\\eval\\filings\\apple_10-K_2025-10-31.html", + "bytes": 1520208, + "sha256": "548ae59778cf08ee0f2ee088e7ece20d947076c3c01f74d2d65db4c2777e436a", + "fetched_at": "2026-09-01T14:01:56Z", + "from_cache": false + }, + { + "company": "Microsoft", + "cik": 789019, + "form": "10-K", + "filing_date": "2026-07-29", + "accession": "0001193125-26-323660", + "document": "msft-20260630.htm", + "url": "https://www.sec.gov/Archives/edgar/data/789019/000119312526323660/msft-20260630.htm", + "path": "data\\eval\\filings\\microsoft_10-K_2026-07-29.html", + "bytes": 8585501, + "sha256": "4bcf2da2871acc19e32c5a62f36f6d83d6aae03cdc4aaa9a86a9a21ee8223f02", + "fetched_at": "2026-09-01T14:01:57Z", + "from_cache": false + }, + { + "company": "JPMorgan Chase", + "cik": 19617, + "form": "10-K", + "filing_date": "2026-02-13", + "accession": "0001628280-26-008131", + "document": "jpm-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/19617/000162828026008131/jpm-20251231.htm", + "path": "data\\eval\\filings\\jpmorgan_chase_10-K_2026-02-13.html", + "bytes": 12927325, + "sha256": "4d9febdbc2038dcdca8726053286df4cbbfd48885051cbd781efcc3becb66a23", + "fetched_at": "2026-09-01T14:01:59Z", + "from_cache": false + }, + { + "company": "Johnson & Johnson", + "cik": 200406, + "form": "10-K", + "filing_date": "2026-02-11", + "accession": "0000200406-26-000016", + "document": "jnj-20251228.htm", + "url": "https://www.sec.gov/Archives/edgar/data/200406/000020040626000016/jnj-20251228.htm", + "path": "data\\eval\\filings\\johnson__johnson_10-K_2026-02-11.html", + "bytes": 3701587, + "sha256": "082f4e489d4c19f585cae4c8b62acea92bd6727576f7b0e450b9dfe8ecd42694", + "fetched_at": "2026-09-01T14:02:00Z", + "from_cache": false + }, + { + "company": "Exxon Mobil", + "cik": 34088, + "form": "10-K", + "filing_date": "2026-02-18", + "accession": "0000034088-26-000045", + "document": "xom-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/34088/000003408826000045/xom-20251231.htm", + "path": "data\\eval\\filings\\exxon_mobil_10-K_2026-02-18.html", + "bytes": 5591068, + "sha256": "3591db2246ab14c52465d32f69b459b000919c141a5656c81cf59acf0a805be8", + "fetched_at": "2026-09-01T14:02:03Z", + "from_cache": false + }, + { + "company": "Walmart", + "cik": 104169, + "form": "10-K", + "filing_date": "2026-03-13", + "accession": "0000104169-26-000055", + "document": "wmt-20260131.htm", + "url": "https://www.sec.gov/Archives/edgar/data/104169/000010416926000055/wmt-20260131.htm", + "path": "data\\eval\\filings\\walmart_10-K_2026-03-13.html", + "bytes": 2323981, + "sha256": "276a6accff6066f00342ba03b390c6c2bd51aa7246f0346df22df0b839a5bf5c", + "fetched_at": "2026-09-01T14:02:09Z", + "from_cache": false + }, + { + "company": "Caterpillar", + "cik": 18230, + "form": "10-K", + "filing_date": "2026-02-13", + "accession": "0000018230-26-000008", + "document": "cat-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/18230/000001823026000008/cat-20251231.htm", + "path": "data\\eval\\filings\\caterpillar_10-K_2026-02-13.html", + "bytes": 6100469, + "sha256": "090ce2bd6d7dd1e85d4397ab152d3c3c3fb059fd4af0eb9aa979c1eb2ff92f15", + "fetched_at": "2026-09-01T14:02:12Z", + "from_cache": false + }, + { + "company": "Pfizer", + "cik": 78003, + "form": "10-K", + "filing_date": "2026-02-26", + "accession": "0000078003-26-000026", + "document": "pfe-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/78003/000007800326000026/pfe-20251231.htm", + "path": "data\\eval\\filings\\pfizer_10-K_2026-02-26.html", + "bytes": 5222324, + "sha256": "175e07c21ee258eddd9952e443d34df2a297d0c38e2d1312dff0a64a31c401ab", + "fetched_at": "2026-09-01T14:02:13Z", + "from_cache": false + } + ] +} \ No newline at end of file diff --git a/data/eval/queries.json b/data/eval/queries.json new file mode 100644 index 0000000..eb4a8f8 --- /dev/null +++ b/data/eval/queries.json @@ -0,0 +1,2684 @@ +{ + "dataset": "docureason-edgar-eval", + "built_at": "2026-09-01T14:33:45Z", + "seed": 0, + "form": "10-K", + "filing_count": 8, + "provenance": "Gold regions are derived programmatically from the ingested corpus, not human-annotated: table cells give a known row/column intersection, and prose and figure anchors are phrases verified unique across the corpus. Derivable queries over-represent lookup and cannot express multi-hop reasoning.", + "generation_report": { + "query_count": 150, + "documents": 8, + "regions_scanned": 20966, + "per_stratum": { + "tabular_lookup": 50, + "numeric": 50, + "prose": 50 + }, + "rejected": { + "tabular_lookup:no_cell": 12, + "numeric:no_cell": 27, + "prose:too_short": 108, + "prose:no_unique_phrase": 18, + "figure:no_regions_in_corpus": 1 + } + }, + "validation": { + "queries": 150, + "regions_in_corpus": 20966, + "dangling_gold_ids": [], + "non_unique_anchors": [], + "valid": true + }, + "queries": [ + { + "id": "q-0001", + "stratum": "tabular_lookup", + "query": "In the table reporting December 31,\n(in billions), what value is shown for col_3?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1635" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "December 31,\n(in billions)", + "derivation": "table_cell", + "anchor": { + "row_label": "December 31,\n(in billions)", + "column": "col_3", + "cell_value": "December 31,\n(in billions)" + }, + "provenance": "programmatic" + }, + { + "id": "q-0002", + "stratum": "tabular_lookup", + "query": "In the table reporting Assumption, what value is shown for col_2?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1032" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "Assumption", + "derivation": "table_cell", + "anchor": { + "row_label": "Assumption", + "column": "col_2", + "cell_value": "Assumption" + }, + "provenance": "programmatic" + }, + { + "id": "q-0003", + "stratum": "tabular_lookup", + "query": "In the table reporting ASSET & WEALTH MANAGEMENT, what value is shown for col_2?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-834" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "ASSET & WEALTH MANAGEMENT", + "derivation": "table_cell", + "anchor": { + "row_label": "ASSET & WEALTH MANAGEMENT", + "column": "col_2", + "cell_value": "ASSET & WEALTH MANAGEMENT" + }, + "provenance": "programmatic" + }, + { + "id": "q-0004", + "stratum": "tabular_lookup", + "query": "In the table reporting Accrued compensation and related items, what value is shown for col_2?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1649" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "Accrued compensation and related items", + "derivation": "table_cell", + "anchor": { + "row_label": "Accrued compensation and related items", + "column": "col_2", + "cell_value": "Accrued compensation and related items" + }, + "provenance": "programmatic" + }, + { + "id": "q-0005", + "stratum": "tabular_lookup", + "query": "In the table reporting U.S. GSEs and government agencies(a), what value is shown for col_25?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-2045" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "104800", + "derivation": "table_cell", + "anchor": { + "row_label": "U.S. GSEs and government agencies(a)", + "column": "col_25", + "cell_value": "104800" + }, + "provenance": "programmatic" + }, + { + "id": "q-0006", + "stratum": "tabular_lookup", + "query": "In the table reporting Collections of finance receivables, what value is shown for col_25?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-868" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "-", + "derivation": "table_cell", + "anchor": { + "row_label": "Collections of finance receivables", + "column": "col_25", + "cell_value": "-" + }, + "provenance": "programmatic" + }, + { + "id": "q-0007", + "stratum": "tabular_lookup", + "query": "In the table reporting Net income, what value is shown for col_23?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-664" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "2821", + "derivation": "table_cell", + "anchor": { + "row_label": "Net income", + "column": "col_23", + "cell_value": "2821" + }, + "provenance": "programmatic" + }, + { + "id": "q-0008", + "stratum": "tabular_lookup", + "query": "In the table reporting Available-for-sale debt securities in an unrealized loss position:, what value is shown for col_35?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1629" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "Available-for-sale debt securities in an unrealized loss position:", + "derivation": "table_cell", + "anchor": { + "row_label": "Available-for-sale debt securities in an unrealized loss position:", + "column": "col_35", + "cell_value": "Available-for-sale debt securities in an unrealized loss position:" + }, + "provenance": "programmatic" + }, + { + "id": "q-0009", + "stratum": "tabular_lookup", + "query": "In the table reporting 2025 Annual Report, what value is shown for col_2?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-2011" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "2025 Annual Report", + "derivation": "table_cell", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_2", + "cell_value": "2025 Annual Report" + }, + "provenance": "programmatic" + }, + { + "id": "q-0010", + "stratum": "tabular_lookup", + "query": "In the table reporting Pfizer Inc., what value is shown for col_4?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-693" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_4", + "cell_value": "2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0011", + "stratum": "tabular_lookup", + "query": "In the table reporting Consolidated Comprehensive Income (Loss) for the Years Ended December 31,, what value is shown for col_17?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-933" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "Consolidated Comprehensive Income (Loss) for the Years Ended December 31,", + "derivation": "table_cell", + "anchor": { + "row_label": "Consolidated Comprehensive Income (Loss) for the Years Ended December 31,", + "column": "col_17", + "cell_value": "Consolidated Comprehensive Income (Loss) for the Years Ended December 31," + }, + "provenance": "programmatic" + }, + { + "id": "q-0012", + "stratum": "tabular_lookup", + "query": "In the table reporting JPMorgan Chase & Co./2025 Form 10-K, what value is shown for col_2?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1593" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "JPMorgan Chase & Co./2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_2", + "cell_value": "JPMorgan Chase & Co./2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0013", + "stratum": "tabular_lookup", + "query": "In the table reporting Development costs incurred during the year, what value is shown for col_11?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2743" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "16845", + "derivation": "table_cell", + "anchor": { + "row_label": "Development costs incurred during the year", + "column": "col_11", + "cell_value": "16845" + }, + "provenance": "programmatic" + }, + { + "id": "q-0014", + "stratum": "tabular_lookup", + "query": "In the table reporting Item 6., what value is shown for col_5?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-33" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "Reserved", + "derivation": "table_cell", + "anchor": { + "row_label": "Item 6.", + "column": "col_5", + "cell_value": "Reserved" + }, + "provenance": "programmatic" + }, + { + "id": "q-0015", + "stratum": "tabular_lookup", + "query": "In the table reporting As of or for the year ended December 31, 2024 (in millions, except ratios), what value is shown for col_19?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3593" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "Term loans by origination year(c)", + "derivation": "table_cell", + "anchor": { + "row_label": "As of or for the year ended December 31, 2024 (in millions, except ratios)", + "column": "col_19", + "cell_value": "Term loans by origination year(c)" + }, + "provenance": "programmatic" + }, + { + "id": "q-0016", + "stratum": "tabular_lookup", + "query": "In the table reporting Table of Contents, what value is shown for col_3?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-339" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Table of Contents", + "derivation": "table_cell", + "anchor": { + "row_label": "Table of Contents", + "column": "col_3", + "cell_value": "Table of Contents" + }, + "provenance": "programmatic" + }, + { + "id": "q-0017", + "stratum": "tabular_lookup", + "query": "In the table reporting Due after one year through five years, what value is shown for col_13?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1656" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "1504", + "derivation": "table_cell", + "anchor": { + "row_label": "Due after one year through five years", + "column": "col_13", + "cell_value": "1504" + }, + "provenance": "programmatic" + }, + { + "id": "q-0018", + "stratum": "tabular_lookup", + "query": "In the table reporting Less: Imputed interest, what value is shown for col_3?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-3232" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "Less: Imputed interest", + "derivation": "table_cell", + "anchor": { + "row_label": "Less: Imputed interest", + "column": "col_3", + "cell_value": "Less: Imputed interest" + }, + "provenance": "programmatic" + }, + { + "id": "q-0019", + "stratum": "tabular_lookup", + "query": "In the table reporting JPMorgan Chase & Co./2025 Form 10-K, what value is shown for col_3?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3335" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "JPMorgan Chase & Co./2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_3", + "cell_value": "JPMorgan Chase & Co./2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0020", + "stratum": "tabular_lookup", + "query": "In the table reporting Table of Contents, what value is shown for col_3?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1020" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Table of Contents", + "derivation": "table_cell", + "anchor": { + "row_label": "Table of Contents", + "column": "col_3", + "cell_value": "Table of Contents" + }, + "provenance": "programmatic" + }, + { + "id": "q-0021", + "stratum": "tabular_lookup", + "query": "In the table reporting Canada, what value is shown for col_3?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1813" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Canada", + "derivation": "table_cell", + "anchor": { + "row_label": "Canada", + "column": "col_3", + "cell_value": "Canada" + }, + "provenance": "programmatic" + }, + { + "id": "q-0022", + "stratum": "tabular_lookup", + "query": "In the table reporting (millions of dollars), what value is shown for col_12?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-765" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Total", + "derivation": "table_cell", + "anchor": { + "row_label": "(millions of dollars)", + "column": "col_12", + "cell_value": "Total" + }, + "provenance": "programmatic" + }, + { + "id": "q-0023", + "stratum": "tabular_lookup", + "query": "In the table reporting Table of Contents, what value is shown for col_9?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1419" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Financial Table of Contents", + "derivation": "table_cell", + "anchor": { + "row_label": "Table of Contents", + "column": "col_9", + "cell_value": "Financial Table of Contents" + }, + "provenance": "programmatic" + }, + { + "id": "q-0024", + "stratum": "tabular_lookup", + "query": "In the table reporting Total, what value is shown for col_7?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1002" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "3954", + "derivation": "table_cell", + "anchor": { + "row_label": "Total", + "column": "col_7", + "cell_value": "3954" + }, + "provenance": "programmatic" + }, + { + "id": "q-0025", + "stratum": "tabular_lookup", + "query": "In the table reporting Cost\n of sales, what value is shown for col_5?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77-region-938" + ], + "document_id": "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "-89587", + "derivation": "table_cell", + "anchor": { + "row_label": "Cost\n of sales", + "column": "col_5", + "cell_value": "-89587" + }, + "provenance": "programmatic" + }, + { + "id": "q-0026", + "stratum": "tabular_lookup", + "query": "In the table reporting PART II, what value is shown for col_2?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-789" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "PART II", + "derivation": "table_cell", + "anchor": { + "row_label": "PART II", + "column": "col_2", + "cell_value": "PART II" + }, + "provenance": "programmatic" + }, + { + "id": "q-0027", + "stratum": "tabular_lookup", + "query": "In the table reporting Table of Contents, what value is shown for col_8?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2316" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Financial Table of Contents", + "derivation": "table_cell", + "anchor": { + "row_label": "Table of Contents", + "column": "col_8", + "cell_value": "Financial Table of Contents" + }, + "provenance": "programmatic" + }, + { + "id": "q-0028", + "stratum": "tabular_lookup", + "query": "In the table reporting Walmart International retail units, what value is shown for col_13?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-429" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": "4257", + "derivation": "table_cell", + "anchor": { + "row_label": "Walmart International retail units", + "column": "col_13", + "cell_value": "4257" + }, + "provenance": "programmatic" + }, + { + "id": "q-0029", + "stratum": "tabular_lookup", + "query": "In the table reporting Gross profit, what value is shown for col_20?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-585" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": "26.8", + "derivation": "table_cell", + "anchor": { + "row_label": "Gross profit", + "column": "col_20", + "cell_value": "26.8" + }, + "provenance": "programmatic" + }, + { + "id": "q-0030", + "stratum": "tabular_lookup", + "query": "In the table reporting 2025 Annual Report, what value is shown for col_2?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-220" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "2025 Annual Report", + "derivation": "table_cell", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_2", + "cell_value": "2025 Annual Report" + }, + "provenance": "programmatic" + }, + { + "id": "q-0031", + "stratum": "tabular_lookup", + "query": "In the table reporting Total available-for-sale securities with gross unrealized losses, what value is shown for col_5?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3407" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "16015", + "derivation": "table_cell", + "anchor": { + "row_label": "Total available-for-sale securities with gross unrealized losses", + "column": "col_5", + "cell_value": "16015" + }, + "provenance": "programmatic" + }, + { + "id": "q-0032", + "stratum": "tabular_lookup", + "query": "In the table reporting Impairments, what value is shown for col_31?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1572" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "-", + "derivation": "table_cell", + "anchor": { + "row_label": "Impairments", + "column": "col_31", + "cell_value": "-" + }, + "provenance": "programmatic" + }, + { + "id": "q-0033", + "stratum": "tabular_lookup", + "query": "In the table reporting Class, what value is shown for col_3?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-38" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Class", + "derivation": "table_cell", + "anchor": { + "row_label": "Class", + "column": "col_3", + "cell_value": "Class" + }, + "provenance": "programmatic" + }, + { + "id": "q-0034", + "stratum": "tabular_lookup", + "query": "In the table reporting Total Firm assets, what value is shown for col_2?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1718" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "Total Firm assets", + "derivation": "table_cell", + "anchor": { + "row_label": "Total Firm assets", + "column": "col_2", + "cell_value": "Total Firm assets" + }, + "provenance": "programmatic" + }, + { + "id": "q-0035", + "stratum": "tabular_lookup", + "query": "In the table reporting Expected volatility of the Company's stock, what value is shown for col_2?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1209" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "Expected volatility of the Company's stock", + "derivation": "table_cell", + "anchor": { + "row_label": "Expected volatility of the Company's stock", + "column": "col_2", + "cell_value": "Expected volatility of the Company's stock" + }, + "provenance": "programmatic" + }, + { + "id": "q-0036", + "stratum": "tabular_lookup", + "query": "In the table reporting (in millions), what value is shown for col_11?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1001" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "December 31,\n2024", + "derivation": "table_cell", + "anchor": { + "row_label": "(in millions)", + "column": "col_11", + "cell_value": "December 31,\n2024" + }, + "provenance": "programmatic" + }, + { + "id": "q-0037", + "stratum": "tabular_lookup", + "query": "In the table reporting 2.610% notes due 2030, what value is shown for col_11?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2255" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "2000", + "derivation": "table_cell", + "anchor": { + "row_label": "2.610% notes due 2030", + "column": "col_11", + "cell_value": "2000" + }, + "provenance": "programmatic" + }, + { + "id": "q-0038", + "stratum": "tabular_lookup", + "query": "In the table reporting Consumer, excluding credit card, what value is shown for col_16?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-528" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "$", + "derivation": "table_cell", + "anchor": { + "row_label": "Consumer, excluding credit card", + "column": "col_16", + "cell_value": "$" + }, + "provenance": "programmatic" + }, + { + "id": "q-0039", + "stratum": "tabular_lookup", + "query": "In the table reporting Corporate Total(millions of dollars), what value is shown for col_12?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-782" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "2023", + "derivation": "table_cell", + "anchor": { + "row_label": "Corporate Total(millions of dollars)", + "column": "col_12", + "cell_value": "2023" + }, + "provenance": "programmatic" + }, + { + "id": "q-0040", + "stratum": "tabular_lookup", + "query": "In the table reporting Alliance revenues(b), what value is shown for col_19?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2086" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "7582", + "derivation": "table_cell", + "anchor": { + "row_label": "Alliance revenues(b)", + "column": "col_19", + "cell_value": "7582" + }, + "provenance": "programmatic" + }, + { + "id": "q-0041", + "stratum": "tabular_lookup", + "query": "In the table reporting JPMorgan Chase & Co./2025 Form 10-K, what value is shown for col_2?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3509" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "JPMorgan Chase & Co./2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_2", + "cell_value": "JPMorgan Chase & Co./2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0042", + "stratum": "tabular_lookup", + "query": "In the table reporting Pfizer Inc., what value is shown for col_5?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1574" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_5", + "cell_value": "2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0043", + "stratum": "tabular_lookup", + "query": "In the table reporting JPMorgan Chase & Co./2025 Form 10-K, what value is shown for col_2?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1873" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "JPMorgan Chase & Co./2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_2", + "cell_value": "JPMorgan Chase & Co./2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0044", + "stratum": "tabular_lookup", + "query": "In the table reporting Chicago, Illinois, what value is shown for col_3?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-199" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "Chicago, Illinois", + "derivation": "table_cell", + "anchor": { + "row_label": "Chicago, Illinois", + "column": "col_3", + "cell_value": "Chicago, Illinois" + }, + "provenance": "programmatic" + }, + { + "id": "q-0045", + "stratum": "tabular_lookup", + "query": "In the table reporting (MILLIONS), what value is shown for col_31?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-3447" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "2024", + "derivation": "table_cell", + "anchor": { + "row_label": "(MILLIONS)", + "column": "col_31", + "cell_value": "2024" + }, + "provenance": "programmatic" + }, + { + "id": "q-0046", + "stratum": "tabular_lookup", + "query": "In the table reporting See accompanying notes to Consolidated Financial Statements., what value is shown for col_6?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-955" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "See accompanying notes to Consolidated Financial Statements.", + "derivation": "table_cell", + "anchor": { + "row_label": "See accompanying notes to Consolidated Financial Statements.", + "column": "col_6", + "cell_value": "See accompanying notes to Consolidated Financial Statements." + }, + "provenance": "programmatic" + }, + { + "id": "q-0047", + "stratum": "tabular_lookup", + "query": "In the table reporting Table of Contents, what value is shown for col_8?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2753" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Financial Table of Contents", + "derivation": "table_cell", + "anchor": { + "row_label": "Table of Contents", + "column": "col_8", + "cell_value": "Financial Table of Contents" + }, + "provenance": "programmatic" + }, + { + "id": "q-0048", + "stratum": "tabular_lookup", + "query": "In the table reporting Accumulated benefit obligation, what value is shown for col_3?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2046" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "Accumulated benefit obligation", + "derivation": "table_cell", + "anchor": { + "row_label": "Accumulated benefit obligation", + "column": "col_3", + "cell_value": "Accumulated benefit obligation" + }, + "provenance": "programmatic" + }, + { + "id": "q-0049", + "stratum": "tabular_lookup", + "query": "In the table reporting Part I, what value is shown for col_4?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-41" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "Item 1.", + "derivation": "table_cell", + "anchor": { + "row_label": "Part I", + "column": "col_4", + "cell_value": "Item 1." + }, + "provenance": "programmatic" + }, + { + "id": "q-0050", + "stratum": "tabular_lookup", + "query": "In the table reporting Pfizer Inc., what value is shown for col_5?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2434" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "2025 Form 10-K", + "derivation": "table_cell", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_5", + "cell_value": "2025 Form 10-K" + }, + "provenance": "programmatic" + }, + { + "id": "q-0051", + "stratum": "numeric", + "query": "What is the Total lending- and deposit-related fees figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3220" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "7606", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total lending- and deposit-related fees", + "column": "col_8", + "cell_value": "7606" + }, + "provenance": "programmatic" + }, + { + "id": "q-0052", + "stratum": "numeric", + "query": "What is the Total noninterest expense(d) figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4761" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "1825", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total noninterest expense(d)", + "column": "col_8", + "cell_value": "1825" + }, + "provenance": "programmatic" + }, + { + "id": "q-0053", + "stratum": "numeric", + "query": "What is the 2025 Annual Report figure reported under col_6?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-427" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "19", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_6", + "cell_value": "19" + }, + "provenance": "programmatic" + }, + { + "id": "q-0054", + "stratum": "numeric", + "query": "What is the Net periodic benefit cost (credit) figure reported under col_31?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1478" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "530", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Net periodic benefit cost (credit)", + "column": "col_31", + "cell_value": "530" + }, + "provenance": "programmatic" + }, + { + "id": "q-0055", + "stratum": "numeric", + "query": "What is the Actual annual rate of return on plan assets figure reported under col_20?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1024" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "-4.6", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Actual annual rate of return on plan assets", + "column": "col_20", + "cell_value": "-4.6" + }, + "provenance": "programmatic" + }, + { + "id": "q-0056", + "stratum": "numeric", + "query": "What is the United States figure reported under col_8?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2294" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "2593", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "United States", + "column": "col_8", + "cell_value": "2593" + }, + "provenance": "programmatic" + }, + { + "id": "q-0057", + "stratum": "numeric", + "query": "What is the Value of restricted stock units figure reported under col_6?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2499" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "2025", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Value of restricted stock units", + "column": "col_6", + "cell_value": "2025" + }, + "provenance": "programmatic" + }, + { + "id": "q-0058", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_8?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1205" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "41", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_8", + "cell_value": "41" + }, + "provenance": "programmatic" + }, + { + "id": "q-0059", + "stratum": "numeric", + "query": "What is the Over (under) funded status figure reported under col_32?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1667" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "-2214", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Over (under) funded status", + "column": "col_32", + "cell_value": "-2214" + }, + "provenance": "programmatic" + }, + { + "id": "q-0060", + "stratum": "numeric", + "query": "What is the Goodwill, related to acquisitions figure reported under col_17?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1116" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "7569", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Goodwill, related to acquisitions", + "column": "col_17", + "cell_value": "7569" + }, + "provenance": "programmatic" + }, + { + "id": "q-0061", + "stratum": "numeric", + "query": "What is the Emerging Markets figure reported under col_25?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-3447" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "9313", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Emerging Markets", + "column": "col_25", + "cell_value": "9313" + }, + "provenance": "programmatic" + }, + { + "id": "q-0062", + "stratum": "numeric", + "query": "What is the 2025 Annual Report figure reported under col_5?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1931" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "89", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_5", + "cell_value": "89" + }, + "provenance": "programmatic" + }, + { + "id": "q-0063", + "stratum": "numeric", + "query": "What is the Loans at fair value figure reported under col_20?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1228" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "1502", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Loans at fair value", + "column": "col_20", + "cell_value": "1502" + }, + "provenance": "programmatic" + }, + { + "id": "q-0064", + "stratum": "numeric", + "query": "What is the Liquid securities and other cash collateral held against derivative receivables figure reported under col_5?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1461" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "-28891", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Liquid securities and other cash collateral held against derivative receivables", + "column": "col_5", + "cell_value": "-28891" + }, + "provenance": "programmatic" + }, + { + "id": "q-0065", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_9?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-640" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "19", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_9", + "cell_value": "19" + }, + "provenance": "programmatic" + }, + { + "id": "q-0066", + "stratum": "numeric", + "query": "What is the S&P Financials Index figure reported under col_23?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-441" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "135.32", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "S&P Financials Index", + "column": "col_23", + "cell_value": "135.32" + }, + "provenance": "programmatic" + }, + { + "id": "q-0067", + "stratum": "numeric", + "query": "What is the JPMorgan Chase & Co./2025 Form 10-K figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1593" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "135", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_8", + "cell_value": "135" + }, + "provenance": "programmatic" + }, + { + "id": "q-0068", + "stratum": "numeric", + "query": "What is the JPMorgan Chase & Co./2025 Form 10-K figure reported under col_9?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4521" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "289", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_9", + "cell_value": "289" + }, + "provenance": "programmatic" + }, + { + "id": "q-0069", + "stratum": "numeric", + "query": "What is the 2025 Annual Report figure reported under col_6?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-220" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "3", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_6", + "cell_value": "3" + }, + "provenance": "programmatic" + }, + { + "id": "q-0070", + "stratum": "numeric", + "query": "What is the Expected stock price volatility (based on implied volatility, after consideration of historical volatility) figure reported under col_14?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-3157" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "26.56", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Expected stock price volatility (based on implied volatility, after consideration of historical volatility)", + "column": "col_14", + "cell_value": "26.56" + }, + "provenance": "programmatic" + }, + { + "id": "q-0071", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_9?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2069" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "66", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_9", + "cell_value": "66" + }, + "provenance": "programmatic" + }, + { + "id": "q-0072", + "stratum": "numeric", + "query": "What is the Fully taxable-equivalent adjustments figure reported under col_14?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-617" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "3782", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Fully taxable-equivalent adjustments", + "column": "col_14", + "cell_value": "3782" + }, + "provenance": "programmatic" + }, + { + "id": "q-0073", + "stratum": "numeric", + "query": "What is the Total securities loaned and other figure reported under col_16?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3500" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "976", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total securities loaned and other", + "column": "col_16", + "cell_value": "976" + }, + "provenance": "programmatic" + }, + { + "id": "q-0074", + "stratum": "numeric", + "query": "What is the Asia-Pacific figure reported under col_11?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-825" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "16107", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Asia-Pacific", + "column": "col_11", + "cell_value": "16107" + }, + "provenance": "programmatic" + }, + { + "id": "q-0075", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_7?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-536" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "14", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_7", + "cell_value": "14" + }, + "provenance": "programmatic" + }, + { + "id": "q-0076", + "stratum": "numeric", + "query": "What is the Corporate debt securities figure reported under col_13?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-2452" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "322", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Corporate debt securities", + "column": "col_13", + "cell_value": "322" + }, + "provenance": "programmatic" + }, + { + "id": "q-0077", + "stratum": "numeric", + "query": "What is the December 29, 2024 figure reported under col_13?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1656" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "1", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "December 29, 2024", + "column": "col_13", + "cell_value": "1" + }, + "provenance": "programmatic" + }, + { + "id": "q-0078", + "stratum": "numeric", + "query": "What is the Total at fair value figure reported under col_28?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2010" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "250", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total at fair value", + "column": "col_28", + "cell_value": "250" + }, + "provenance": "programmatic" + }, + { + "id": "q-0079", + "stratum": "numeric", + "query": "What is the MP&E capital expenditures figure reported under col_26?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-820" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "-1988", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "MP&E capital expenditures", + "column": "col_26", + "cell_value": "-1988" + }, + "provenance": "programmatic" + }, + { + "id": "q-0080", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_8?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-358" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "7", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_8", + "cell_value": "7" + }, + "provenance": "programmatic" + }, + { + "id": "q-0081", + "stratum": "numeric", + "query": "What is the Vested figure reported under col_23?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1224" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "253.98", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Vested", + "column": "col_23", + "cell_value": "253.98" + }, + "provenance": "programmatic" + }, + { + "id": "q-0082", + "stratum": "numeric", + "query": "What is the Long-term debt figure reported under col_13?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-2012" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "46673", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Long-term debt", + "column": "col_13", + "cell_value": "46673" + }, + "provenance": "programmatic" + }, + { + "id": "q-0083", + "stratum": "numeric", + "query": "What is the All other investing activities, net figure reported under col_20?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4808" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "25", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "All other investing activities, net", + "column": "col_20", + "cell_value": "25" + }, + "provenance": "programmatic" + }, + { + "id": "q-0084", + "stratum": "numeric", + "query": "What is the Postretirement benefits reserves figure reported under col_5?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1883" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "-1278", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Postretirement benefits reserves", + "column": "col_5", + "cell_value": "-1278" + }, + "provenance": "programmatic" + }, + { + "id": "q-0085", + "stratum": "numeric", + "query": "What is the Report of Independent Registered Public Accounting Firm figure reported under col_3?", + "relevant_ids": [ + "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4-region-1804" + ], + "document_id": "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4", + "source": "microsoft_10-K_2026-07-29.html", + "ground_truth_answer": "86", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Report of Independent Registered Public Accounting Firm", + "column": "col_3", + "cell_value": "86" + }, + "provenance": "programmatic" + }, + { + "id": "q-0086", + "stratum": "numeric", + "query": "What is the Self-insurance(2) figure reported under col_14?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-1099" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": "4976", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Self-insurance(2)", + "column": "col_14", + "cell_value": "4976" + }, + "provenance": "programmatic" + }, + { + "id": "q-0087", + "stratum": "numeric", + "query": "What is the Inter-Segment figure reported under col_26?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-614" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "6", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Inter-Segment", + "column": "col_26", + "cell_value": "6" + }, + "provenance": "programmatic" + }, + { + "id": "q-0088", + "stratum": "numeric", + "query": "What is the Provision for taxes on income figure reported under col_15?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-2173" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "307", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Provision for taxes on income", + "column": "col_15", + "cell_value": "307" + }, + "provenance": "programmatic" + }, + { + "id": "q-0089", + "stratum": "numeric", + "query": "What is the Total assets acquired in loan satisfactions figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1300" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "112", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total assets acquired in loan satisfactions", + "column": "col_8", + "cell_value": "112" + }, + "provenance": "programmatic" + }, + { + "id": "q-0090", + "stratum": "numeric", + "query": "What is the JPMorgan Chase & Co./2025 Form 10-K figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-2450" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "187", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_8", + "cell_value": "187" + }, + "provenance": "programmatic" + }, + { + "id": "q-0091", + "stratum": "numeric", + "query": "What is the Employee benefits figure reported under col_7?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4525" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "1165", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Employee benefits", + "column": "col_7", + "cell_value": "1165" + }, + "provenance": "programmatic" + }, + { + "id": "q-0092", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_8?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1646" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "53", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_8", + "cell_value": "53" + }, + "provenance": "programmatic" + }, + { + "id": "q-0093", + "stratum": "numeric", + "query": "What is the Pfizer Inc. figure reported under col_8?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2573" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": "76", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Pfizer Inc.", + "column": "col_8", + "cell_value": "76" + }, + "provenance": "programmatic" + }, + { + "id": "q-0094", + "stratum": "numeric", + "query": "What is the All Other Segment figure reported under col_25?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-2138" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "14", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "All Other Segment", + "column": "col_25", + "cell_value": "14" + }, + "provenance": "programmatic" + }, + { + "id": "q-0095", + "stratum": "numeric", + "query": "What is the Canada figure reported under col_5?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1044" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "402", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Canada", + "column": "col_5", + "cell_value": "402" + }, + "provenance": "programmatic" + }, + { + "id": "q-0096", + "stratum": "numeric", + "query": "What is the (Amounts in millions, except per share data) figure reported under col_13?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-1096" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": "2025", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "(Amounts in millions, except per share data)", + "column": "col_13", + "cell_value": "2025" + }, + "provenance": "programmatic" + }, + { + "id": "q-0097", + "stratum": "numeric", + "query": "What is the Effective income tax rate figure reported under col_11?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1258" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": "33", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Effective income tax rate", + "column": "col_11", + "cell_value": "33" + }, + "provenance": "programmatic" + }, + { + "id": "q-0098", + "stratum": "numeric", + "query": "What is the JPMorgan Chase & Co./2025 Form 10-K figure reported under col_8?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-660" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": "63", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "JPMorgan Chase & Co./2025 Form 10-K", + "column": "col_8", + "cell_value": "63" + }, + "provenance": "programmatic" + }, + { + "id": "q-0099", + "stratum": "numeric", + "query": "What is the Asset-backed securities figure reported under col_5?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1641" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": "7", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Asset-backed securities", + "column": "col_5", + "cell_value": "7" + }, + "provenance": "programmatic" + }, + { + "id": "q-0100", + "stratum": "numeric", + "query": "What is the 2025 Annual Report figure reported under col_6?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-731" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": "39", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "2025 Annual Report", + "column": "col_6", + "cell_value": "39" + }, + "provenance": "programmatic" + }, + { + "id": "q-0101", + "stratum": "prose", + "query": "What does the filing say about collaboration with valneva se?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1360" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "collaboration with valneva se", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0102", + "stratum": "prose", + "query": "What does the filing say about or otherwise disrupting critical?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-289" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "or otherwise disrupting critical", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0103", + "stratum": "prose", + "query": "What does the filing say about the accounting treatment for?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77-region-731" + ], + "document_id": "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "the accounting treatment for", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0104", + "stratum": "prose", + "query": "What does the filing say about the asia-pacific africa region?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-496" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "the asia-pacific africa region", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0105", + "stratum": "prose", + "query": "What does the filing say about device market demand between?", + "relevant_ids": [ + "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4-region-168" + ], + "document_id": "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4", + "source": "microsoft_10-K_2026-07-29.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "device market demand between", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0106", + "stratum": "prose", + "query": "What does the filing say about union disputes or other?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-316" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "union disputes or other", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0107", + "stratum": "prose", + "query": "What does the filing say about other employment taxes and?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77-region-892" + ], + "document_id": "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "other employment taxes and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0108", + "stratum": "prose", + "query": "What does the filing say about new products in the?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-1053" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "new products in the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0109", + "stratum": "prose", + "query": "What does the filing say about climate change see the?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-542" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "climate change see the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0110", + "stratum": "prose", + "query": "What does the filing say about included in other income deductions?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2059" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "included in other income deductions", + "phrase_words": 5 + }, + "provenance": "programmatic" + }, + { + "id": "q-0111", + "stratum": "prose", + "query": "What does the filing say about approximates fair value or?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1789" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "approximates fair value or", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0112", + "stratum": "prose", + "query": "What does the filing say about ms wengel was appointed executive vice?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-440" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "ms wengel was appointed executive vice", + "phrase_words": 6 + }, + "provenance": "programmatic" + }, + { + "id": "q-0113", + "stratum": "prose", + "query": "What does the filing say about fees recorded in primarily?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1168" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "fees recorded in primarily", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0114", + "stratum": "prose", + "query": "What does the filing say about of changing assumptions for?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-657" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "of changing assumptions for", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0115", + "stratum": "prose", + "query": "What does the filing say about sold of approximately million?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-2019" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "sold of approximately million", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0116", + "stratum": "prose", + "query": "What does the filing say about february the firm redeemed?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4299" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "february the firm redeemed", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0117", + "stratum": "prose", + "query": "What does the filing say about improper payments or providing?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-356" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "improper payments or providing", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0118", + "stratum": "prose", + "query": "What does the filing say about tables show information by?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77-region-916" + ], + "document_id": "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "tables show information by", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0119", + "stratum": "prose", + "query": "What does the filing say about consolidated financial statements and involved?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1559" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "consolidated financial statements and involved", + "phrase_words": 5 + }, + "provenance": "programmatic" + }, + { + "id": "q-0120", + "stratum": "prose", + "query": "What does the filing say about costs related asset impairments?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-2144" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "costs related asset impairments", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0121", + "stratum": "prose", + "query": "What does the filing say about exchange reported on july?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-37" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "exchange reported on july", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0122", + "stratum": "prose", + "query": "What does the filing say about need to rely on?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-184" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "need to rely on", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0123", + "stratum": "prose", + "query": "What does the filing say about copilot are key tools?", + "relevant_ids": [ + "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4-region-185" + ], + "document_id": "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4", + "source": "microsoft_10-K_2026-07-29.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "copilot are key tools", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0124", + "stratum": "prose", + "query": "What does the filing say about yet adopted were assessed?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1163" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "yet adopted were assessed", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0125", + "stratum": "prose", + "query": "What does the filing say about costs and less flexibility?", + "relevant_ids": [ + "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4-region-408" + ], + "document_id": "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4", + "source": "microsoft_10-K_2026-07-29.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "costs and less flexibility", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0126", + "stratum": "prose", + "query": "What does the filing say about to support customer and?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-159" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "to support customer and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0127", + "stratum": "prose", + "query": "What does the filing say about global health crises pandemics?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-292" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "global health crises pandemics", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0128", + "stratum": "prose", + "query": "What does the filing say about estate loans excluding those?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-3636" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "estate loans excluding those", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0129", + "stratum": "prose", + "query": "What does the filing say about government agency securities of?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-2236" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "government agency securities of", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0130", + "stratum": "prose", + "query": "What does the filing say about to the information set?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-399" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "to the information set", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0131", + "stratum": "prose", + "query": "What does the filing say about year the following activity?", + "relevant_ids": [ + "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4-region-1633" + ], + "document_id": "doc-microsoft_10-K_2026-07-29-2b2764e68b92caf4", + "source": "microsoft_10-K_2026-07-29.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "year the following activity", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0132", + "stratum": "prose", + "query": "What does the filing say about and ecommerce businesses in?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-351" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "and ecommerce businesses in", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0133", + "stratum": "prose", + "query": "What does the filing say about maintenance costs of the?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-236" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "maintenance costs of the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0134", + "stratum": "prose", + "query": "What does the filing say about actual and estimated payments?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-1168" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "actual and estimated payments", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0135", + "stratum": "prose", + "query": "What does the filing say about we rely extensively on?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-301" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "we rely extensively on", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0136", + "stratum": "prose", + "query": "What does the filing say about financial products and of?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-843" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "financial products and of", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0137", + "stratum": "prose", + "query": "What does the filing say about uses formulations and product?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-339" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "uses formulations and product", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0138", + "stratum": "prose", + "query": "What does the filing say about class of the selected?", + "relevant_ids": [ + "doc-pfizer_10-K_2026-02-26-14f04faab20489c5-region-1143" + ], + "document_id": "doc-pfizer_10-K_2026-02-26-14f04faab20489c5", + "source": "pfizer_10-K_2026-02-26.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "class of the selected", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0139", + "stratum": "prose", + "query": "What does the filing say about use the mortality assumption?", + "relevant_ids": [ + "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7-region-744" + ], + "document_id": "doc-caterpillar_10-K_2026-02-13-91280835de60a5a7", + "source": "caterpillar_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "use the mortality assumption", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0140", + "stratum": "prose", + "query": "What does the filing say about regulations and interpretations could?", + "relevant_ids": [ + "doc-walmart_10-K_2026-03-13-22563e5dea3b635e-region-357" + ], + "document_id": "doc-walmart_10-K_2026-03-13-22563e5dea3b635e", + "source": "walmart_10-K_2026-03-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "regulations and interpretations could", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0141", + "stratum": "prose", + "query": "What does the filing say about equipment increased reflecting the?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-576" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "equipment increased reflecting the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0142", + "stratum": "prose", + "query": "What does the filing say about plans at december and?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1582" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "plans at december and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0143", + "stratum": "prose", + "query": "What does the filing say about data centers retail and?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77-region-319" + ], + "document_id": "doc-apple_10-K_2025-10-31-bcda7cdbd0907f77", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "data centers retail and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0144", + "stratum": "prose", + "query": "What does the filing say about revenue and market share?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-150" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "revenue and market share", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0145", + "stratum": "prose", + "query": "What does the filing say about and production activities unaudited?", + "relevant_ids": [ + "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8-region-2562" + ], + "document_id": "doc-exxon_mobil_10-K_2026-02-18-c6ded73193db5de8", + "source": "exxon_mobil_10-K_2026-02-18.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "and production activities unaudited", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0146", + "stratum": "prose", + "query": "What does the filing say about owning and managing real?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-152" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "owning and managing real", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0147", + "stratum": "prose", + "query": "What does the filing say about tcja foreign undistributed earnings?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1444" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "tcja foreign undistributed earnings", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0148", + "stratum": "prose", + "query": "What does the filing say about is liquidity requirement for?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-1141" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "is liquidity requirement for", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0149", + "stratum": "prose", + "query": "What does the filing say about were well-capitalized and met?", + "relevant_ids": [ + "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4-region-4597" + ], + "document_id": "doc-jpmorgan_chase_10-K_2026-02-13-0b600893751cd5f4", + "source": "jpmorgan_chase_10-K_2026-02-13.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "were well-capitalized and met", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0150", + "stratum": "prose", + "query": "What does the filing say about on the restricted share?", + "relevant_ids": [ + "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730-region-1760" + ], + "document_id": "doc-johnson__johnson_10-K_2026-02-11-f9e17634dccb0730", + "source": "johnson__johnson_10-K_2026-02-11.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "on the restricted share", + "phrase_words": 4 + }, + "provenance": "programmatic" + } + ] +} \ No newline at end of file diff --git a/data/eval_small/filings/provenance.json b/data/eval_small/filings/provenance.json new file mode 100644 index 0000000..690129c --- /dev/null +++ b/data/eval_small/filings/provenance.json @@ -0,0 +1,120 @@ +{ + "source": "SEC EDGAR", + "form": "10-K", + "filing_count": 8, + "user_agent": "DocuReason evaluation corpus builder (kumararpit17773@gmail.com)", + "filings": [ + { + "company": "Apple", + "cik": 320193, + "form": "10-K", + "filing_date": "2025-10-31", + "accession": "0000320193-25-000079", + "document": "aapl-20250927.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019325000079/aapl-20250927.htm", + "path": "data\\eval\\filings\\apple_10-K_2025-10-31.html", + "bytes": 1520208, + "sha256": "548ae59778cf08ee0f2ee088e7ece20d947076c3c01f74d2d65db4c2777e436a", + "fetched_at": "2026-09-01T14:01:56Z", + "from_cache": false + }, + { + "company": "Microsoft", + "cik": 789019, + "form": "10-K", + "filing_date": "2026-07-29", + "accession": "0001193125-26-323660", + "document": "msft-20260630.htm", + "url": "https://www.sec.gov/Archives/edgar/data/789019/000119312526323660/msft-20260630.htm", + "path": "data\\eval\\filings\\microsoft_10-K_2026-07-29.html", + "bytes": 8585501, + "sha256": "4bcf2da2871acc19e32c5a62f36f6d83d6aae03cdc4aaa9a86a9a21ee8223f02", + "fetched_at": "2026-09-01T14:01:57Z", + "from_cache": false + }, + { + "company": "JPMorgan Chase", + "cik": 19617, + "form": "10-K", + "filing_date": "2026-02-13", + "accession": "0001628280-26-008131", + "document": "jpm-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/19617/000162828026008131/jpm-20251231.htm", + "path": "data\\eval\\filings\\jpmorgan_chase_10-K_2026-02-13.html", + "bytes": 12927325, + "sha256": "4d9febdbc2038dcdca8726053286df4cbbfd48885051cbd781efcc3becb66a23", + "fetched_at": "2026-09-01T14:01:59Z", + "from_cache": false + }, + { + "company": "Johnson & Johnson", + "cik": 200406, + "form": "10-K", + "filing_date": "2026-02-11", + "accession": "0000200406-26-000016", + "document": "jnj-20251228.htm", + "url": "https://www.sec.gov/Archives/edgar/data/200406/000020040626000016/jnj-20251228.htm", + "path": "data\\eval\\filings\\johnson__johnson_10-K_2026-02-11.html", + "bytes": 3701587, + "sha256": "082f4e489d4c19f585cae4c8b62acea92bd6727576f7b0e450b9dfe8ecd42694", + "fetched_at": "2026-09-01T14:02:00Z", + "from_cache": false + }, + { + "company": "Exxon Mobil", + "cik": 34088, + "form": "10-K", + "filing_date": "2026-02-18", + "accession": "0000034088-26-000045", + "document": "xom-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/34088/000003408826000045/xom-20251231.htm", + "path": "data\\eval\\filings\\exxon_mobil_10-K_2026-02-18.html", + "bytes": 5591068, + "sha256": "3591db2246ab14c52465d32f69b459b000919c141a5656c81cf59acf0a805be8", + "fetched_at": "2026-09-01T14:02:03Z", + "from_cache": false + }, + { + "company": "Walmart", + "cik": 104169, + "form": "10-K", + "filing_date": "2026-03-13", + "accession": "0000104169-26-000055", + "document": "wmt-20260131.htm", + "url": "https://www.sec.gov/Archives/edgar/data/104169/000010416926000055/wmt-20260131.htm", + "path": "data\\eval\\filings\\walmart_10-K_2026-03-13.html", + "bytes": 2323981, + "sha256": "276a6accff6066f00342ba03b390c6c2bd51aa7246f0346df22df0b839a5bf5c", + "fetched_at": "2026-09-01T14:02:09Z", + "from_cache": false + }, + { + "company": "Caterpillar", + "cik": 18230, + "form": "10-K", + "filing_date": "2026-02-13", + "accession": "0000018230-26-000008", + "document": "cat-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/18230/000001823026000008/cat-20251231.htm", + "path": "data\\eval\\filings\\caterpillar_10-K_2026-02-13.html", + "bytes": 6100469, + "sha256": "090ce2bd6d7dd1e85d4397ab152d3c3c3fb059fd4af0eb9aa979c1eb2ff92f15", + "fetched_at": "2026-09-01T14:02:12Z", + "from_cache": false + }, + { + "company": "Pfizer", + "cik": 78003, + "form": "10-K", + "filing_date": "2026-02-26", + "accession": "0000078003-26-000026", + "document": "pfe-20251231.htm", + "url": "https://www.sec.gov/Archives/edgar/data/78003/000007800326000026/pfe-20251231.htm", + "path": "data\\eval\\filings\\pfizer_10-K_2026-02-26.html", + "bytes": 5222324, + "sha256": "175e07c21ee258eddd9952e443d34df2a297d0c38e2d1312dff0a64a31c401ab", + "fetched_at": "2026-09-01T14:02:13Z", + "from_cache": false + } + ] +} \ No newline at end of file diff --git a/data/eval_small/queries.json b/data/eval_small/queries.json new file mode 100644 index 0000000..e041c07 --- /dev/null +++ b/data/eval_small/queries.json @@ -0,0 +1,670 @@ +{ + "dataset": "docureason-edgar-eval", + "built_at": "2026-09-01T15:04:52Z", + "seed": 0, + "form": "10-K", + "filing_count": 1, + "provenance": "Gold regions are derived programmatically from the ingested corpus, not human-annotated: table cells give a known row/column intersection, and prose and figure anchors are phrases verified unique across the corpus. Derivable queries over-represent lookup and cannot express multi-hop reasoning.", + "generation_report": { + "query_count": 36, + "documents": 1, + "regions_scanned": 1169, + "per_stratum": { + "tabular_lookup": 12, + "numeric": 12, + "prose": 12 + }, + "rejected": { + "tabular_lookup:no_cell": 1, + "numeric:no_cell": 5, + "prose:too_short": 23, + "prose:no_unique_phrase": 3, + "figure:no_regions_in_corpus": 1 + } + }, + "validation": { + "queries": 36, + "regions_in_corpus": 1169, + "dangling_gold_ids": [], + "non_unique_anchors": [], + "valid": true + }, + "queries": [ + { + "id": "q-0001", + "stratum": "tabular_lookup", + "query": "In the table reporting U.S., what value is shown for col_11?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-984" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "35664", + "derivation": "table_cell", + "anchor": { + "row_label": "U.S.", + "column": "col_11", + "cell_value": "35664" + }, + "provenance": "programmatic" + }, + { + "id": "q-0002", + "stratum": "tabular_lookup", + "query": "In the table reporting Ending\n balances, what value is shown for col_4?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-799" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "$", + "derivation": "table_cell", + "anchor": { + "row_label": "Ending\n balances", + "column": "col_4", + "cell_value": "$" + }, + "provenance": "programmatic" + }, + { + "id": "q-0003", + "stratum": "tabular_lookup", + "query": "In the table reporting Thereafter, what value is shown for col_3?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-904" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "Thereafter", + "derivation": "table_cell", + "anchor": { + "row_label": "Thereafter", + "column": "col_3", + "cell_value": "Thereafter" + }, + "provenance": "programmatic" + }, + { + "id": "q-0004", + "stratum": "tabular_lookup", + "query": "In the table reporting Other\n non-current assets, what value is shown for col_3?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-761" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "Other\n non-current assets", + "derivation": "table_cell", + "anchor": { + "row_label": "Other\n non-current assets", + "column": "col_3", + "cell_value": "Other\n non-current assets" + }, + "provenance": "programmatic" + }, + { + "id": "q-0005", + "stratum": "tabular_lookup", + "query": "In the table reporting Exhibit\n Number, what value is shown for col_26?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-1097" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "Filing\n Date/\nPeriod End Date", + "derivation": "table_cell", + "anchor": { + "row_label": "Exhibit\n Number", + "column": "col_26", + "cell_value": "Filing\n Date/\nPeriod End Date" + }, + "provenance": "programmatic" + }, + { + "id": "q-0006", + "stratum": "tabular_lookup", + "query": "In the table reporting Services\n (1), what value is shown for col_2?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-670" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "Services\n (1)", + "derivation": "table_cell", + "anchor": { + "row_label": "Services\n (1)", + "column": "col_2", + "cell_value": "Services\n (1)" + }, + "provenance": "programmatic" + }, + { + "id": "q-0007", + "stratum": "tabular_lookup", + "query": "In the table reporting One Apple Park\n Way, what value is shown for col_3?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-19" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "One Apple Park\n Way", + "derivation": "table_cell", + "anchor": { + "row_label": "One Apple Park\n Way", + "column": "col_3", + "cell_value": "One Apple Park\n Way" + }, + "provenance": "programmatic" + }, + { + "id": "q-0008", + "stratum": "tabular_lookup", + "query": "In the table reporting 0.000% Notes due 2025, what value is shown for col_2?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-27" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "0.000% Notes due 2025", + "derivation": "table_cell", + "anchor": { + "row_label": "0.000% Notes due 2025", + "column": "col_2", + "cell_value": "0.000% Notes due 2025" + }, + "provenance": "programmatic" + }, + { + "id": "q-0009", + "stratum": "tabular_lookup", + "query": "In the table reporting Total\n term debt principal, what value is shown for col_5?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-865" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "91281", + "derivation": "table_cell", + "anchor": { + "row_label": "Total\n term debt principal", + "column": "col_5", + "cell_value": "91281" + }, + "provenance": "programmatic" + }, + { + "id": "q-0010", + "stratum": "tabular_lookup", + "query": "In the table reporting Exhibit\n Number, what value is shown for col_8?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-1148" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "Exhibit\n Description", + "derivation": "table_cell", + "anchor": { + "row_label": "Exhibit\n Number", + "column": "col_8", + "cell_value": "Exhibit\n Description" + }, + "provenance": "programmatic" + }, + { + "id": "q-0011", + "stratum": "tabular_lookup", + "query": "In the table reporting Basic\n earnings per share, what value is shown for col_5?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-677" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "7.49", + "derivation": "table_cell", + "anchor": { + "row_label": "Basic\n earnings per share", + "column": "col_5", + "cell_value": "7.49" + }, + "provenance": "programmatic" + }, + { + "id": "q-0012", + "stratum": "tabular_lookup", + "query": "In the table reporting Net\n income, what value is shown for col_5?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-515" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "112010", + "derivation": "table_cell", + "anchor": { + "row_label": "Net\n income", + "column": "col_5", + "cell_value": "112010" + }, + "provenance": "programmatic" + }, + { + "id": "q-0013", + "stratum": "numeric", + "query": "What is the Municipal\n securities figure reported under col_23?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-702" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "405", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Municipal\n securities", + "column": "col_23", + "cell_value": "405" + }, + "provenance": "programmatic" + }, + { + "id": "q-0014", + "stratum": "numeric", + "query": "What is the Cost\n of sales figure reported under col_10?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-917" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "-58617", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Cost\n of sales", + "column": "col_10", + "cell_value": "-58617" + }, + "provenance": "programmatic" + }, + { + "id": "q-0015", + "stratum": "numeric", + "query": "What is the iPad figure reported under col_11?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-670" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "26694", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "iPad", + "column": "col_11", + "cell_value": "26694" + }, + "provenance": "programmatic" + }, + { + "id": "q-0016", + "stratum": "numeric", + "query": "What is the Total\n operating expenses figure reported under col_11?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-426" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "8", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Total\n operating expenses", + "column": "col_11", + "cell_value": "8" + }, + "provenance": "programmatic" + }, + { + "id": "q-0017", + "stratum": "numeric", + "query": "What is the Consolidated Statements of Cash Flows for\n the years ended September 27, 2025, September 28, 2024 and September 30, 2023 figure reported under col_8?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-1065" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "33", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Consolidated Statements of Cash Flows for\n the years ended September 27, 2025, September 28, 2024 and September 30, 2023", + "column": "col_8", + "cell_value": "33" + }, + "provenance": "programmatic" + }, + { + "id": "q-0018", + "stratum": "numeric", + "query": "What is the Foreign\n exchange contracts figure reported under col_11?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-744" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "64069", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Foreign\n exchange contracts", + "column": "col_11", + "cell_value": "64069" + }, + "provenance": "programmatic" + }, + { + "id": "q-0019", + "stratum": "numeric", + "query": "What is the S&P\n 500 Index figure reported under col_38?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-352" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "217", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "S&P\n 500 Index", + "column": "col_38", + "cell_value": "217" + }, + "provenance": "programmatic" + }, + { + "id": "q-0020", + "stratum": "numeric", + "query": "What is the Research\n and development figure reported under col_4?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-504" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "34550", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Research\n and development", + "column": "col_4", + "cell_value": "34550" + }, + "provenance": "programmatic" + }, + { + "id": "q-0021", + "stratum": "numeric", + "query": "What is the Interest\n Rate\nSensitive\n Instrument figure reported under col_21?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-474" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "2025", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Interest\n Rate\nSensitive\n Instrument", + "column": "col_21", + "cell_value": "2025" + }, + "provenance": "programmatic" + }, + { + "id": "q-0022", + "stratum": "numeric", + "query": "What is the Operating\n income/(loss) figure reported under col_23?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-938" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "12454", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Operating\n income/(loss)", + "column": "col_23", + "cell_value": "12454" + }, + "provenance": "programmatic" + }, + { + "id": "q-0023", + "stratum": "numeric", + "query": "What is the Other\n non-current assets figure reported under col_11?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-761" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "55335", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Other\n non-current assets", + "column": "col_11", + "cell_value": "55335" + }, + "provenance": "programmatic" + }, + { + "id": "q-0024", + "stratum": "numeric", + "query": "What is the Decreases\n related to settlements with taxing authorities figure reported under col_17?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-799" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": "-19", + "derivation": "table_cell_numeric", + "anchor": { + "row_label": "Decreases\n related to settlements with taxing authorities", + "column": "col_17", + "cell_value": "-19" + }, + "provenance": "programmatic" + }, + { + "id": "q-0025", + "stratum": "prose", + "query": "What does the filing say about its success and could?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-245" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "its success and could", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0026", + "stratum": "prose", + "query": "What does the filing say about the company's own advertising?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-151" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "the company's own advertising", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0027", + "stratum": "prose", + "query": "What does the filing say about first quarter of the?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-377" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "first quarter of the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0028", + "stratum": "prose", + "query": "What does the filing say about targeted improvements to the?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-457" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "targeted improvements to the", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0029", + "stratum": "prose", + "query": "What does the filing say about of supplier arrangements licensed?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-903" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "of supplier arrangements licensed", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0030", + "stratum": "prose", + "query": "What does the filing say about company's share repurchase programs?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-869" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "company's share repurchase programs", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0031", + "stratum": "prose", + "query": "What does the filing say about affected products services and?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-203" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "affected products services and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0032", + "stratum": "prose", + "query": "What does the filing say about make judgments assumptions and?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-463" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "make judgments assumptions and", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0033", + "stratum": "prose", + "query": "What does the filing say about unsecured obligations with interest?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-839" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "unsecured obligations with interest", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0034", + "stratum": "prose", + "query": "What does the filing say about company combines and accounts?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-660" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "company combines and accounts", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0035", + "stratum": "prose", + "query": "What does the filing say about inc the company as?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-991" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "inc the company as", + "phrase_words": 4 + }, + "provenance": "programmatic" + }, + { + "id": "q-0036", + "stratum": "prose", + "query": "What does the filing say about claims such insurance coverage?", + "relevant_ids": [ + "doc-apple_10-K_2025-10-31-5e604ff0e793065a-region-269" + ], + "document_id": "doc-apple_10-K_2025-10-31-5e604ff0e793065a", + "source": "apple_10-K_2025-10-31.html", + "ground_truth_answer": null, + "derivation": "unique_phrase", + "anchor": { + "phrase": "claims such insurance coverage", + "phrase_words": 4 + }, + "provenance": "programmatic" + } + ] +} \ No newline at end of file diff --git a/docureason/pipeline.py b/docureason/pipeline.py index 7a2fd2c..b351016 100644 --- a/docureason/pipeline.py +++ b/docureason/pipeline.py @@ -10,6 +10,7 @@ from docureason.tripath.indexing.artifact_writer import ArtifactWriter, build_provenance from docureason.tripath.indexing.dense_index import DenseIndexBuilder from docureason.tripath.indexing.sparse_index import BM25SIndexBuilder +from docureason.tripath.indexing.table_store import DuckDBTableStore from docureason.tripath.ingestion.chunker import SectionAwareChunker from docureason.tripath.ingestion.docling_layout_parser import DoclingLayoutParser from docureason.tripath.ingestion.figure_captioner import FigureCaptioner @@ -27,6 +28,10 @@ logger = setup_logger("docureason.pipeline") +#: Formats handed to the Docling layout parser. Anything else falls through to the +#: rule-based segmenter, which cannot recover tables or reading order. +_LAYOUT_PARSED_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".html", ".htm"}) + class DocuReasonPipeline: """Build a corpus and index artifact from documents using vision-based layout parsing. @@ -105,6 +110,7 @@ def __init__( self.ocr_fallback = OCRFallback( languages=["en"], min_chars_threshold=self.config.ingestion.ocr_fallback_char_threshold, + image_rendering_dpi=self.config.ingestion.image_rendering_dpi, ) log_pipeline_flag( "ocr_languages", @@ -125,12 +131,17 @@ def __init__( ) self.quality_auditor = ArtifactQualityAuditor() - self.figure_captioner = FigureCaptioner(use_blip2=True, use_clip=True) + self.figure_captioner = FigureCaptioner( + use_blip2=True, + use_clip=True, + image_rendering_dpi=self.config.ingestion.image_rendering_dpi, + ) log_pipeline_flag("use_blip2", True, "BLIP-2 figure captioning enabled", logger) log_pipeline_flag("use_clip", True, "CLIP figure embeddings enabled", logger) self.dense_index = DenseIndexBuilder(output_dir=self.output_dir, config=self.config) self.sparse_index = BM25SIndexBuilder(output_dir=self.output_dir, config=self.config) + self.table_store = DuckDBTableStore(self.output_dir) @trace_pipeline_stage("Document Ingestion & Indexing Pipeline") def run(self) -> Dict[str, object]: @@ -176,6 +187,11 @@ def run(self) -> Dict[str, object]: # landed on disk rather than what was requested. index_stats = self._build_indices(chunks) + # Register every table region as a real DuckDB table, so the SQL retriever + # stops re-creating and re-populating each table on every query. + table_store_stats = self.table_store.build(document_objects).as_dict() + index_stats["tables"] = table_store_stats + audit = self._build_audit(document_objects, index_stats) (self.output_dir / "corpus.json").write_text(json.dumps(corpus, indent=2), encoding="utf-8") @@ -308,9 +324,11 @@ def _build_audit( "structured_tables_serialized": structured_tables, "table_structure_enabled": self.layout_parser.table_structure_enabled, "table_structure_requested": self.config.ingestion.enable_table_structure, + "layout_parser_enabled": self.config.ingestion.enable_layout_parser, "caption_tier_counts": dict(self.figure_captioner.tier_counts), "ocr_fallback_enabled": self.config.ingestion.enable_ocr_fallback, "ocr_char_threshold": self.config.ingestion.ocr_fallback_char_threshold, + "image_rendering_dpi": self.config.ingestion.image_rendering_dpi, "chunking": { "child_chunk_size": self.chunker.chunk_size, "chunk_overlap": self.chunker.overlap, @@ -322,6 +340,7 @@ def _build_audit( "dense_index_type": self.dense_index.index_type, "bm25_params": {"k1": self.sparse_index.bm25_k1, "b": self.sparse_index.bm25_b}, "index_stats": index_stats, + "table_store": index_stats.get("tables", {}), } audit["degradation"] = self._ingestion_degradation(layout_source_counts, index_stats) audit["provenance"] = build_provenance({"preset": self.config.preset}) @@ -341,6 +360,8 @@ def _ingestion_degradation( rule_regions = layout_source_counts.get("rule", 0) if total_regions and rule_regions / total_regions > 0.5: labels.append(f"layout:rule_fallback={rule_regions}/{total_regions}") + if not self.config.ingestion.enable_layout_parser: + labels.append("ablation:layout_parser_disabled") if self.layout_parser.table_structure_enabled is False: labels.append("table_structure:disabled") if self.layout_parser.table_structure_enabled is None and total_regions: @@ -357,6 +378,10 @@ def _ingestion_degradation( if stats.get("degraded"): labels.append(f"sparse_index:{modality}:{stats['degraded']}") + table_stats = index_stats.get("tables") or {} + if table_stats.get("degraded"): + labels.append(f"table_store:{table_stats['degraded']}") + return labels # ------------------------------------------------------------------ @@ -432,8 +457,15 @@ def _build_regions(self, path: Path) -> List[Region]: regions: List[Region] = [] raw_tables: Dict[int, Any] = {} - # Step 1 — vision-based layout parse. - if path.suffix.lower() in {".pdf", ".docx", ".pptx", ".xlsx"}: + # Step 1 — vision-based layout parse. Ablation A1 ("remove the Docling layout + # parser") skips it outright; previously the only way to reach the rule-based + # segmenter was to make Docling fail, so the -42.5% the ablation matrix claims + # for this component had never been measured. + # HTML is in this set because SEC filings are HTML: routing them to the loader + # instead meant the rule-based segmenter saw raw markup, so a 10-K's financial + # tables produced no table regions at all and the tabular strata of the + # evaluation corpus could not exist. + if self.config.ingestion.enable_layout_parser and path.suffix.lower() in _LAYOUT_PARSED_SUFFIXES: try: regions, raw_tables = self.layout_parser.parse(path) logger.debug("DoclingLayoutParser: %d region(s) from %s", len(regions), path.name) @@ -447,7 +479,8 @@ def _build_regions(self, path: Path) -> List[Region]: text = str(payload.get("text", "")) regions = self._segment_regions_rule_based(text) - # Step 3 — OCR supplement for scanned pages (PDF only, config-gated). + # Step 3 — OCR supplement for scanned pages (PDF only: there is no page raster + # to OCR for an HTML or Office document). if path.suffix.lower() == ".pdf" and self.config.ingestion.enable_ocr_fallback: try: ocr_regions = self.ocr_fallback.run(path, regions) diff --git a/docureason/tripath/config.py b/docureason/tripath/config.py index 0b6d655..2fb5911 100644 --- a/docureason/tripath/config.py +++ b/docureason/tripath/config.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +from copy import deepcopy from dataclasses import asdict, dataclass, field from dataclasses import fields as dataclass_fields from pathlib import Path @@ -167,6 +168,10 @@ class IngestionConfig: # two undocumented environment variables, so CPU users silently got degraded # table parsing while the docs claimed TableFormer had run (audit P0-6). enable_table_structure: Optional[bool] = None + # Ingestion-time ablation switch for the Docling layout parse (ablation A1). + # False forces the rule-based line segmenter that used to be reachable only by + # making Docling fail, so "no layout parser" can be measured rather than assumed. + enable_layout_parser: bool = True @dataclass @@ -230,6 +235,11 @@ class RetrievalConfig: # readable. Set False to force the legacy scan (used by the retrieval # ablation and by tests that must not touch an index). enable_index_retrieval: bool = True + # Query-time ablation switches. Each names a retrieval path that the ablation + # matrix in Documentations/eval_method.md claims a delta for; without a switch + # those deltas could only be asserted, never measured. + enable_sparse_retrieval: bool = True + enable_table_sql: bool = True @dataclass @@ -251,6 +261,12 @@ class RerankerConfig: heading_penalty_multiplier: float = 0.5 final_top_k: int = 5 enable_parent_expansion: bool = True + # False forces the non-model Jaccard fallback scorer, which is what a + # "no cross-encoder" ablation means (ablation A3). + enable_cross_encoder: bool = True + # False disables the short-heading penalty entirely (ablation A5). Distinct from + # setting the offset to 0.0, which would still record the penalty as active. + enable_heading_penalty: bool = True @dataclass @@ -265,6 +281,22 @@ class GenerationConfig: max_new_tokens: int = 512 +@dataclass +class ServingConfig: + """Online serving behaviour: concurrency and caching. + + ``configs/serving.yaml`` existed but was never loaded (``load_from_yaml`` reads + ``config.yaml`` only), so both of these were previously unreachable settings for + behaviour the service did not have. + """ + + #: Distinct (query, config) responses kept in the per-service LRU. 0 disables it. + response_cache_size: int = 64 + #: Run the three modality retrievers concurrently instead of one after another. + #: Results are collected in a fixed order, so this changes latency, not ranking. + enable_parallel_paths: bool = True + + @dataclass class AttributionConfig: """Sentence-Level NLI Faithfulness Attribution Parameters.""" @@ -275,6 +307,58 @@ class AttributionConfig: enable_nli: bool = True +#: Every configurable layer, in banner order. One tuple, so adding a section does +#: not mean remembering to add it to the YAML loader, the env-override scan and the +#: summary independently. +_CONFIG_SECTIONS = ( + "ingestion", + "indexing", + "router", + "retrieval", + "reranker", + "generation", + "attribution", + "serving", +) + + +def _as_bool(value: Any) -> bool: + """Coerce a JSON or CLI value to bool, rejecting anything ambiguous.""" + if isinstance(value, bool): + return value + token = str(value).strip().lower() + if token in _TRUE_TOKENS: + return True + if token in _FALSE_TOKENS: + return False + raise ValueError(f"expected a boolean, got {value!r}") + + +#: Per-query override name -> (section, field, caster). One table, so a field is +#: addressable from a REST request body and from an ablation definition by the same +#: name; adding an ablation switch means adding one row here. +_QUERY_OVERRIDES: Dict[str, tuple] = { + "domain": ("indexing", "domain", str), + "index_type": ("indexing", "index_type", str), + "max_context_tokens": ("generation", "max_context_tokens", int), + "gen_backend": ("generation", "backend", str), + "final_top_k": ("reranker", "final_top_k", int), + "top_k_text": ("retrieval", "top_k_text", int), + "top_k_table": ("retrieval", "top_k_table", int), + "top_k_vision": ("retrieval", "top_k_vision", int), + # Ablation switches (Documentations/eval_method.md §21). + "enable_index_retrieval": ("retrieval", "enable_index_retrieval", _as_bool), + "enable_sparse_retrieval": ("retrieval", "enable_sparse_retrieval", _as_bool), + "enable_table_sql": ("retrieval", "enable_table_sql", _as_bool), + "enable_cross_encoder": ("reranker", "enable_cross_encoder", _as_bool), + "enable_heading_penalty": ("reranker", "enable_heading_penalty", _as_bool), + "enable_parent_expansion": ("reranker", "enable_parent_expansion", _as_bool), + "enable_nli": ("attribution", "enable_nli", _as_bool), + "enable_parallel_paths": ("serving", "enable_parallel_paths", _as_bool), + "response_cache_size": ("serving", "response_cache_size", int), +} + + # ---------------------------------------------------------------------- # Root Framework Configuration # ---------------------------------------------------------------------- @@ -293,6 +377,7 @@ class DocuReasonConfig: reranker: RerankerConfig = field(default_factory=RerankerConfig) generation: GenerationConfig = field(default_factory=GenerationConfig) attribution: AttributionConfig = field(default_factory=AttributionConfig) + serving: ServingConfig = field(default_factory=ServingConfig) source_map: Dict[str, str] = field(default_factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -449,7 +534,7 @@ def load_from_yaml(cls, yaml_path: Union[str, Path] = "configs/config.yaml") -> config.output_dir = data["output_dir"] config.source_map["output_dir"] = f"YAML File: {path}" - for section_name in ["ingestion", "indexing", "router", "retrieval", "reranker", "generation", "attribution"]: + for section_name in _CONFIG_SECTIONS: if section_name in data and isinstance(data[section_name], dict): section_obj = getattr(config, section_name) for k, v in data[section_name].items(): @@ -457,7 +542,7 @@ def load_from_yaml(cls, yaml_path: Union[str, Path] = "configs/config.yaml") -> setattr(section_obj, k, v) config.source_map[f"{section_name}.{k}"] = f"YAML File: {path}" - sections = ["ingestion", "indexing", "router", "retrieval", "reranker", "generation", "attribution"] + sections = list(_CONFIG_SECTIONS) for env_key, env_val in os.environ.items(): if not env_key.startswith("DOCUREASON_"): continue @@ -488,27 +573,44 @@ def load_from_yaml(cls, yaml_path: Union[str, Path] = "configs/config.yaml") -> return config def apply_query_overrides(self, overrides: Dict[str, Any]) -> DocuReasonConfig: - """Apply dynamic runtime per-query overrides (for FastAPI JSON requests).""" + """Return a **copy** of this config with per-query overrides applied. + + This used to mutate ``self`` and return it, so a single request carrying + ``{"final_top_k": 1}`` permanently narrowed the shared ``QueryService`` for + every request that followed, and an ablation run would leave its own switches + behind for the next one. Overrides are per query by definition; deep-copying + is what makes that true. + + Unknown keys are ignored but recorded in ``source_map`` under + ``"unknown_overrides"``, so a typo in a request body is visible rather than + silently doing nothing. + """ if not overrides: return self - if "domain" in overrides: - self.indexing.domain = str(overrides["domain"]) - self.source_map["indexing.domain"] = "Per-Query Runtime Override" - if "max_context_tokens" in overrides: - self.generation.max_context_tokens = int(overrides["max_context_tokens"]) - self.source_map["generation.max_context_tokens"] = "Per-Query Runtime Override" - if "final_top_k" in overrides: - self.reranker.final_top_k = int(overrides["final_top_k"]) - self.source_map["reranker.final_top_k"] = "Per-Query Runtime Override" - if "index_type" in overrides: - self.indexing.index_type = str(overrides["index_type"]) - self.source_map["indexing.index_type"] = "Per-Query Runtime Override" - if "gen_backend" in overrides: - self.generation.backend = str(overrides["gen_backend"]) - self.source_map["generation.backend"] = "Per-Query Runtime Override" - - return self + clone = deepcopy(self) + unknown: List[str] = [] + + for key, raw in overrides.items(): + spec = _QUERY_OVERRIDES.get(key) + if spec is None: + unknown.append(str(key)) + continue + section_name, field_name, caster = spec + section = getattr(clone, section_name) + try: + setattr(section, field_name, caster(raw)) + except (TypeError, ValueError) as exc: + logger.warning("Ignoring per-query override %s=%r: %s", key, raw, exc) + unknown.append(str(key)) + continue + clone.source_map[f"{section_name}.{field_name}"] = "Per-Query Runtime Override" + + if unknown: + clone.source_map["unknown_overrides"] = ", ".join(sorted(unknown)) + logger.warning("Per-query overrides ignored (unknown or invalid): %s", sorted(unknown)) + + return clone def summary_dict(self) -> Dict[str, Any]: """Return structured summary dictionary of active framework settings.""" @@ -522,6 +624,7 @@ def summary_dict(self) -> Dict[str, Any]: "reranker": asdict(self.reranker), "generation": asdict(self.generation), "attribution": asdict(self.attribution), + "serving": asdict(self.serving), "source_map": self.source_map, } diff --git a/docureason/tripath/evaluation/ablation.py b/docureason/tripath/evaluation/ablation.py index a0043c7..4e037b3 100644 --- a/docureason/tripath/evaluation/ablation.py +++ b/docureason/tripath/evaluation/ablation.py @@ -1,24 +1,629 @@ +"""ablation.py — measure what each architectural block actually contributes. + +What was here before: a 24-line function that subtracted one pre-computed +``average_recall_at_5`` from another, with no caller anywhere in the repository. +Meanwhile ``Documentations/eval_method.md`` §21 published a seven-row ablation matrix +with deltas to one decimal place (-42.5%, -61.2%, -21.4%, ...) that nothing in the +codebase could have produced. Three of those components could not even be switched +off: there was no flag for the sparse path, the cross-encoder, the heading penalty or +the SQL retriever, so "remove X and re-measure" was not an operation the system +supported. + +This runner executes the matrix. For each ablation it re-runs **every query** with +that component disabled, scores both runs through the same harness, and reports the +delta as a *paired* comparison: mean difference, bootstrap confidence interval and a +sign-flip p-value. Paired because query-to-query variance dwarfs the between-system +variance - the same query is hard for both systems, and only the paired difference +cancels that out. + +Two honesty mechanisms are built into the report: + +* ``documented_claim`` carries each row's published delta next to the measured one, so + a claim the measurement does not support is visible in the artifact rather than + surviving in a table nobody re-derived. +* an ablation that could not be executed is reported as ``executed: false`` with a + reason. It is never silently dropped, and it never appears as a delta of zero - + which is what "not measured" would otherwise look like. +""" + from __future__ import annotations -from typing import Any, Dict, List +import json +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple +from docureason.tripath.evaluation.statistics import DEFAULT_ITERATIONS, compare_paired +from docureason.tripath.utils import get_logger -class AblationStudy: - """Compare a full system run against ablated configurations.""" - - def run(self, baseline: Dict[str, Any], ablations: List[Dict[str, Any]]) -> Dict[str, Any]: - rows = [] - for ablation in ablations: - delta = round( - ablation.get("summary", {}).get("average_recall_at_5", 0.0) - - baseline.get("summary", {}).get("average_recall_at_5", 0.0), - 3, +logger = get_logger(__name__) + +#: Metrics compared for every ablation. Retrieval quality, ranking quality, ordering, +#: faithfulness and latency - enough to distinguish "found less" from "ranked worse". +COMPARED_METRICS = ( + "recall_at_5", + "precision_at_5", + "ndcg_at_5", + "mrr", + "map_at_10", + "hit_rate_at_5", + "attribution_precision", + "hallucination_rate", + "duplicate_rank_ratio", + # Answer quality: the parent-expansion row's primary metric is token_f1, so leaving + # it out of this tuple rendered that row as "n/a" - the comparison was never + # computed for the one metric its own claim is about. + "token_f1", + "exact_match", + "latency_ms", +) + +#: Metrics scored against gold **region ids**. Their values only mean something inside +#: the corpus those ids were derived from. An ingestion-scoped ablation rebuilds the +#: corpus, so the same sentence lives in a region with a different id and every metric +#: below collapses toward zero whether retrieval degraded or not. For such a row they +#: are reported as not comparable rather than as a delta: a -100% recall that actually +#: measures the label space would be the most misleading number in the artifact. +CORPUS_BOUND_METRICS = frozenset( + { + "recall_at_5", + "precision_at_5", + "ndcg_at_5", + "mrr", + "map_at_10", + "hit_rate_at_5", + } +) + +#: Printed in place of a corpus-bound delta on a cross-corpus row. +NOT_COMPARABLE_REASON = ( + "scored against gold region ids derived from the un-ablated corpus; this row ran a " + "corpus re-ingested with the component disabled, so the label space differs and the " + "delta would measure region identity, not retrieval" +) + +#: What a cross-corpus row reports instead. Scored against the ground-truth answer text, +#: which a corpus rebuild does not change, so it stays a paired measurement. +ANSWER_LEVEL_METRIC = "token_f1" + + +@dataclass(frozen=True) +class Ablation: + """One row of the ablation matrix.""" + + key: str + label: str + #: Per-query config overrides that disable the component. Empty for an ablation + #: that can only be applied at ingestion time. + overrides: Dict[str, Any] + #: ``"query"`` (a per-query override suffices) or ``"ingestion"`` (needs a corpus + #: rebuilt with the component off - the layout parser runs once, not per query). + scope: str + #: The metric the documented matrix claims this component moves. + primary_metric: str + #: The published delta from Documentations/eval_method.md §21, as a percentage. + documented_claim_pct: Optional[float] + rationale: str + + +#: The seven ablations from Documentations/eval_method.md §21, in that order. +ABLATIONS: Tuple[Ablation, ...] = ( + Ablation( + key="no_layout_parser", + label="Remove Docling layout parser", + overrides={}, + scope="ingestion", + primary_metric="recall_at_5", + documented_claim_pct=-42.5, + rationale="Layout-aware parsing is claimed to be essential for multi-column and tabular PDFs.", + ), + Ablation( + key="no_table_sql", + label="Remove TableSQL retriever (DuckDB)", + overrides={"enable_table_sql": False}, + scope="query", + primary_metric="recall_at_5", + documented_claim_pct=-61.2, + rationale="Dense vectors are claimed to fail at exact numerical arithmetic.", + ), + Ablation( + key="no_cross_encoder", + label="Remove cross-encoder reranker", + overrides={"enable_cross_encoder": False}, + scope="query", + primary_metric="ndcg_at_5", + documented_claim_pct=-21.4, + rationale="RRF rank output is claimed to need joint self-attention rescoring.", + ), + Ablation( + key="no_parent_expansion", + label="Remove parent-child expansion", + overrides={"enable_parent_expansion": False}, + scope="query", + primary_metric="token_f1", + documented_claim_pct=-18.6, + rationale="Isolated child chunks are claimed to lack context for QA.", + ), + Ablation( + key="no_heading_penalty", + label="Remove short-heading penalty", + overrides={"enable_heading_penalty": False}, + scope="query", + primary_metric="ndcg_at_5", + documented_claim_pct=-14.2, + rationale="Heading-only chunks are claimed to crowd out real content without the penalty.", + ), + Ablation( + key="no_sparse_path", + label="Remove BM25S sparse path", + overrides={"enable_sparse_retrieval": False}, + scope="query", + primary_metric="recall_at_5", + documented_claim_pct=-28.9, + rationale="Dense search is claimed to miss exact jargon and identifiers.", + ), + Ablation( + key="no_claim_verification", + label="Remove claim-support verification", + overrides={"enable_nli": False}, + scope="query", + primary_metric="hallucination_rate", + documented_claim_pct=24.8, + rationale="Claim checking is claimed to be vital against hallucination.", + ), +) + +ABLATIONS_BY_KEY: Dict[str, Ablation] = {ablation.key: ablation for ablation in ABLATIONS} + + +@dataclass +class SystemRun: + """Every per-query metric vector for one system configuration.""" + + name: str + #: metric name -> value per query, in query order (None where unmeasured). + metrics: Dict[str, List[Optional[float]]] = field(default_factory=dict) + #: The harness's own suite report for this configuration. + report: Dict[str, Any] = field(default_factory=dict) + failures: int = 0 + + def values(self, metric: str) -> List[Optional[float]]: + return self.metrics.get(metric, []) + + +class AblationRunner: + """Execute the ablation matrix against a live query service and an eval harness.""" + + def __init__( + self, + service: Any, + harness: Any, + ablations: Sequence[Ablation] = ABLATIONS, + compared_metrics: Sequence[str] = COMPARED_METRICS, + iterations: int = DEFAULT_ITERATIONS, + seed: int = 0, + ingestion_services: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Parameters + ---------- + service + A built ``QueryService`` (or anything with ``query(text, overrides=...)``). + harness + An ``EvaluationHarness``; the *same* one scores every configuration, which + is what makes the runs comparable. + ingestion_services + Optional ``{ablation_key: service}`` for ingestion-scoped ablations, each + built from a corpus ingested with that component off. Absent keys are + reported as not executed rather than skipped silently. + """ + self.service = service + self.harness = harness + self.ablations = list(ablations) + self.compared_metrics = list(compared_metrics) + self.iterations = iterations + self.seed = seed + self.ingestion_services = dict(ingestion_services or {}) + + # ------------------------------------------------------------------ + # Execution + # ------------------------------------------------------------------ + + def run(self, cases: Sequence[Dict[str, Any]], baseline: Optional[Any] = None) -> Dict[str, Any]: + """Run the full system, every ablation, and optionally a baseline system. + + *cases* are ``{"query", "relevant_ids", ...}`` dicts - the same shape the + harness suite takes. *baseline* is anything exposing ``query(text)``, scored + through the identical harness so the comparison is not confounded by scoring. + """ + if not cases: + raise ValueError("no evaluation cases: an ablation over zero queries measures nothing") + + started = time.perf_counter() + full = self._run_system("full", cases, overrides=None, service=self.service) + + rows: List[Dict[str, Any]] = [] + systems: Dict[str, Any] = {"full": full.report.get("summary_metrics", {})} + + for ablation in self.ablations: + service = self._service_for(ablation) + if service is None: + rows.append(self._not_executed(ablation)) + logger.warning("Ablation %s not executed: %s", ablation.key, rows[-1]["reason"]) + continue + + overrides = ablation.overrides or None + run = self._run_system(ablation.key, cases, overrides=overrides, service=service) + systems[ablation.key] = run.report.get("summary_metrics", {}) + # An ingestion-scoped row runs a different service over a different corpus, + # so metrics keyed on gold region ids stop being paired measurements of the + # same thing. Decided from the service identity rather than from scope alone, + # because scope says what the ablation is, not which corpus answered. + rows.append(self._compare(ablation, full, run, cross_corpus=service is not self.service)) + + report: Dict[str, Any] = { + "queries": len(cases), + "seed": self.seed, + "bootstrap_iterations": self.iterations, + "compared_metrics": list(self.compared_metrics), + "systems": systems, + "ablations": rows, + "wall_clock_sec": round(time.perf_counter() - started, 2), + "executed": sum(1 for row in rows if row.get("executed")), + "not_executed": [row["key"] for row in rows if not row.get("executed")], + } + + if baseline is not None: + report["baseline"] = self._run_baseline(baseline, cases, full) + + report["detailed_runs"] = {"full": full.report} + return report + + def _service_for(self, ablation: Ablation) -> Optional[Any]: + if ablation.scope == "query": + return self.service + return self.ingestion_services.get(ablation.key) + + def _not_executed(self, ablation: Ablation) -> Dict[str, Any]: + return { + "key": ablation.key, + "label": ablation.label, + "scope": ablation.scope, + "executed": False, + # Names the command that would close it. A reason a reader cannot act on is + # only marginally better than a silent zero. + "reason": ( + "ingestion-scoped: needs a corpus re-ingested with this component disabled. " + "Build one with scripts/build_eval_corpus.py --corpus-dir --no-layout-parser " + "--skip-derive, then pass --ingestion-corpus %s= to scripts/run_ablations.py" + % ablation.key + ), + "documented_claim_pct": ablation.documented_claim_pct, + "primary_metric": ablation.primary_metric, + } + + def _run_system( + self, + name: str, + cases: Sequence[Dict[str, Any]], + overrides: Optional[Dict[str, Any]], + service: Any, + ) -> SystemRun: + """Execute every case against *service* and score it through the harness.""" + run = SystemRun(name=name) + scored: List[Dict[str, Any]] = [] + + for case in cases: + query = str(case.get("query", "")) + started = time.perf_counter() + try: + response = service.query(query, overrides=overrides) if overrides else service.query(query) + except Exception as exc: + logger.error("System %s failed on query %r: %s", name, query, exc) + run.failures += 1 + continue + latency_ms = (time.perf_counter() - started) * 1000.0 + + scored.append( + self.harness.evaluate_single( + query=query, + results=response.get("results", []), + relevant_ids=case.get("relevant_ids", []), + answer=response.get("answer"), + ground_truth_answer=case.get("ground_truth_answer"), + evidence=response.get("retrieved_evidence"), + predicted_table=case.get("predicted_table"), + ground_truth_table=case.get("ground_truth_table"), + sql_executed=(response.get("sql_execution") or {}).get("executed"), + latency_ms=round(latency_ms, 3), + stratum=case.get("stratum"), + latency_breakdown=response.get("latency_breakdown"), + ) + ) + + for metric in self.compared_metrics: + run.metrics[metric] = [record.get(metric) for record in scored] + + latencies = [record.get("latency_ms", 0.0) for record in scored] + run.report = { + "total_queries": len(scored), + "failures": run.failures, + "summary_metrics": self.harness._aggregate_summary(scored, latencies), + # The runner's own budget: it computes intervals for every system and every + # stratum, so the default 10,000 resamples would dominate the sweep. + "confidence_intervals": self.harness.confidence_intervals(scored, iterations=self.iterations), + "stage_latency_ms": self.harness.stage_latency_summary(scored), + "per_stratum": self.harness.stratified_summary(scored, iterations=self.iterations), + } + logger.info("Ablation system %r: %d/%d queries scored", name, len(scored), len(cases)) + return run + + # ------------------------------------------------------------------ + # Comparison + # ------------------------------------------------------------------ + + def _compare( + self, + ablation: Ablation, + full: SystemRun, + ablated: SystemRun, + cross_corpus: bool = False, + ) -> Dict[str, Any]: + """Paired comparison of the ablated system against the full one, per metric. + + Sign convention: ``delta = ablated - full``. Removing a component that helps + therefore produces a negative delta on a quality metric, which is the same + direction the documented matrix uses. + + *cross_corpus* marks a row whose ablated system answered from its own corpus. + The answer-level metrics stay paired - the query and its ground-truth answer are + unchanged - so they carry the measurement for that row, while the gold-region + metrics are reported as not comparable. + """ + comparisons: Dict[str, Any] = {} + for metric in self.compared_metrics: + if cross_corpus and metric in CORPUS_BOUND_METRICS: + comparisons[metric] = {"comparable": False, "reason": NOT_COMPARABLE_REASON} + continue + comparison = compare_paired( + ablated.values(metric), + full.values(metric), + iterations=self.iterations, + seed=self.seed, + ) + payload = comparison.as_dict() + payload["mean_ablated"] = payload.pop("mean_a") + payload["mean_full"] = payload.pop("mean_b") + payload["relative_delta_pct"] = _relative_pct(payload["delta"], payload["mean_full"]) + comparisons[metric] = payload + + primary = comparisons.get(ablation.primary_metric) + primary_comparable = bool(primary) and primary.get("comparable") is not False + row: Dict[str, Any] = { + "key": ablation.key, + "label": ablation.label, + "scope": ablation.scope, + "executed": True, + "overrides": dict(ablation.overrides), + "rationale": ablation.rationale, + "primary_metric": ablation.primary_metric, + "documented_claim_pct": ablation.documented_claim_pct, + "measured_primary_delta_pct": (primary or {}).get("relative_delta_pct"), + "claim_supported": ( + _claim_verdict(ablation, primary) if primary_comparable else "not_comparable_across_corpora" + ), + "queries_scored": len(full.values(ablation.primary_metric)), + "comparisons": comparisons, + } + if cross_corpus: + row["cross_corpus"] = True + # The published claim for this row is about a metric the row cannot measure, + # so name the corpus-independent one that carries the result instead of + # leaving the reader to infer which numbers survived the corpus rebuild. + row["answer_metric"] = ANSWER_LEVEL_METRIC + row["measured_answer_delta_pct"] = (comparisons.get(ANSWER_LEVEL_METRIC) or {}).get( + "relative_delta_pct" + ) + # What the two ingestions actually produced: directly comparable where the + # gold-id metrics are not, and the evidence the row is really about. + row["corpora"] = { + "full": corpus_signature(self.service), + "ablated": corpus_signature(self._service_for(ablation)), + } + return row + + def _run_baseline(self, baseline: Any, cases: Sequence[Dict[str, Any]], full: SystemRun) -> Dict[str, Any]: + """Score a baseline system through the same harness and pair it against full.""" + run = self._run_system("baseline", cases, overrides=None, service=baseline) + comparisons: Dict[str, Any] = {} + for metric in self.compared_metrics: + comparison = compare_paired( + full.values(metric), + run.values(metric), + iterations=self.iterations, + seed=self.seed, + ) + payload = comparison.as_dict() + payload["mean_full"] = payload.pop("mean_a") + payload["mean_baseline"] = payload.pop("mean_b") + payload["relative_delta_pct"] = _relative_pct(payload["delta"], payload["mean_baseline"]) + comparisons[metric] = payload + return { + "system": getattr(baseline, "__class__", type(baseline)).__name__, + "summary_metrics": run.report.get("summary_metrics", {}), + "full_vs_baseline": comparisons, + } + + +def corpus_signature(service: Any) -> Dict[str, Any]: + """What an ingestion produced, read from that corpus's own audit artifact. + + For an ingestion-scoped ablation this is the part of the comparison that survives + the corpus rebuild: region and table counts are directly comparable where gold-id + metrics are not. Returns ``{}`` when there is no audit on disk - a stub service in a + test, or a corpus built before ``quality_audit.json`` existed - so the row reports no + signature instead of failing the sweep. + """ + output_dir = getattr(service, "output_dir", None) + if not output_dir: + return {} + try: + audit = json.loads((Path(output_dir) / "quality_audit.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + capabilities = audit.get("capabilities_used") or {} + return { + "corpus": str(output_dir), + "total_regions": audit.get("total_regions"), + "total_chunks": audit.get("total_chunks"), + "structured_tables_serialized": capabilities.get("structured_tables_serialized"), + "regions_by_layout_source": capabilities.get("regions_by_layout_source"), + "layout_parser_enabled": capabilities.get("layout_parser_enabled"), + } + + +def _relative_pct(delta: Optional[float], reference: Optional[float]) -> Optional[float]: + """*delta* as a percentage of *reference*, or None when that is undefined. + + Returns None rather than 0.0 when the reference is zero: a relative change against + a baseline of zero is not a number, and reporting it as 0% would read as "no + effect" when the truth is "no denominator". + """ + if delta is None or reference in (None, 0): + return None + return round((delta / reference) * 100.0, 2) + + +def _claim_verdict(ablation: Ablation, primary: Optional[Dict[str, Any]]) -> Optional[str]: + """Whether the measurement supports the published delta for this row. + + Deliberately coarse - direction and significance only. Claiming agreement to one + decimal place on a few hundred queries would repeat the mistake this runner exists + to correct. + """ + if primary is None or ablation.documented_claim_pct is None: + return None + measured = primary.get("relative_delta_pct") + if measured is None: + return "not_measurable" + if not primary.get("significant"): + return "not_significant" + claimed_sign = 1 if ablation.documented_claim_pct > 0 else -1 + measured_sign = 1 if measured > 0 else -1 + if claimed_sign != measured_sign: + return "contradicted" + ratio = abs(measured) / max(1e-9, abs(ablation.documented_claim_pct)) + if 0.5 <= ratio <= 2.0: + return "consistent_in_magnitude" + return "same_direction_different_magnitude" + + +def _cross_corpus_cell( + row: Dict[str, Any], + answer_metric: str, + answer_delta: Optional[float], + answer_comparison: Dict[str, Any], +) -> str: + """What to print for a row whose primary metric cannot cross the corpus rebuild. + + Preference order is by how much the number says. A relative answer-quality delta is + best; an absolute one still carries a direction; failing both, the corpus census is + the finding - "62 tables became 0" is the ablation, and it is always available. + """ + if answer_delta is not None: + return f"{answer_metric} {answer_delta:+.1f}%" + + absolute = answer_comparison.get("delta") + if isinstance(absolute, (int, float)) and absolute != 0: + return f"{answer_metric} {absolute:+.4f} abs" + + corpora = row.get("corpora") or {} + full, ablated = corpora.get("full") or {}, corpora.get("ablated") or {} + if full.get("total_regions") is not None and ablated.get("total_regions") is not None: + return ( + f"tables {full.get('structured_tables_serialized')}->{ablated.get('structured_tables_serialized')}, " + f"regions {full['total_regions']}->{ablated['total_regions']}" + ) + return f"{answer_metric} n/a" + + +def markdown_matrix(report: Dict[str, Any]) -> str: + """Render the ablation report as the matrix table, measured next to claimed.""" + lines = [ + "| Ablated component | Primary metric | Measured Δ | Claimed Δ | p | Verdict |", + "| :--- | :--- | ---: | ---: | ---: | :--- |", + ] + for row in report.get("ablations", []): + if not row.get("executed"): + # The published claim stays in the row even when nothing measured it: dropping + # it here would quietly retire an unverified number instead of showing it + # standing unverified. + claimed = row.get("documented_claim_pct") + lines.append( + "| {label} | {metric} | not executed | {claimed} | — | {reason} |".format( + label=row["label"], + metric=row["primary_metric"], + claimed="—" if claimed is None else f"{claimed:+.1f}%", + reason=row["reason"], + ) ) - rows.append( - { - "name": ablation.get("name", "unnamed"), - "average_recall_at_5": ablation.get("summary", {}).get("average_recall_at_5", 0.0), - "delta_vs_full": delta, - } + continue + primary = row["comparisons"].get(row["primary_metric"], {}) + measured = row.get("measured_primary_delta_pct") + claimed = row.get("documented_claim_pct") + if primary.get("comparable") is False: + # The claim for this row is about a metric the row cannot measure. Show the + # delta it can measure, labelled with its metric, rather than an empty cell + # that reads as "no effect". + answer_metric = row.get("answer_metric", ANSWER_LEVEL_METRIC) + answer_delta = row.get("measured_answer_delta_pct") + answer_comparison = row["comparisons"].get(answer_metric, {}) + measured_cell = _cross_corpus_cell(row, answer_metric, answer_delta, answer_comparison) + p_cell = "n/a" if answer_comparison.get("p_value") is None else f"{answer_comparison['p_value']:.4f}" + else: + measured_cell = "n/a" if measured is None else f"{measured:+.1f}%" + p_cell = "n/a" if primary.get("p_value") is None else f"{primary['p_value']:.4f}" + lines.append( + "| {label} | {metric} | {measured} | {claimed} | {p} | {verdict} |".format( + label=row["label"], + metric=row["primary_metric"], + measured=measured_cell, + claimed="—" if claimed is None else f"{claimed:+.1f}%", + p=p_cell, + verdict=row.get("claim_supported") or "n/a", ) - return {"baseline": baseline, "ablations": rows} + ) + return "\n".join(lines) + + +class AblationStudy: + """Deprecated shim for the original delta calculator. + + Kept because ``Documentations/eval_method.md`` documents ``AblationStudy().run()`` + as the entry point. It now delegates to :class:`AblationRunner`, which needs a live + service, so the old signature raises with instructions rather than returning a + plausible-looking number computed from nothing. + """ + + def run(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + raise NotImplementedError( + "AblationStudy computed a delta between two pre-computed summaries and was never called. " + "Use AblationRunner(service, harness).run(cases), or scripts/run_ablations.py." + ) + + +__all__ = [ + "ABLATIONS", + "ABLATIONS_BY_KEY", + "ANSWER_LEVEL_METRIC", + "COMPARED_METRICS", + "CORPUS_BOUND_METRICS", + "NOT_COMPARABLE_REASON", + "Ablation", + "AblationRunner", + "AblationStudy", + "SystemRun", + "corpus_signature", + "markdown_matrix", +] diff --git a/docureason/tripath/evaluation/baselines.py b/docureason/tripath/evaluation/baselines.py new file mode 100644 index 0000000..db7d00f --- /dev/null +++ b/docureason/tripath/evaluation/baselines.py @@ -0,0 +1,373 @@ +"""baselines.py — the naive RAG baseline every reported number needs. + +The report and the docs quote deltas against "naive RAG" without a naive RAG existing +anywhere in the repository, so nothing the tri-path system achieved could be +attributed to the tri-path architecture rather than to the corpus, the embedding +model, or the generator. A baseline is the cheapest thing in evaluation and the one +that makes every other number mean something. + +This is deliberately the *textbook* pipeline, so the comparison isolates retrieval +architecture and nothing else: + +* recursive character splitting on paragraph / sentence / word boundaries - no layout + parsing, no section awareness, no parent-child hierarchy; +* the same embedding model at the same pinned revision as the full system; +* one flat exact inner-product index over all chunks, no per-modality split, no BM25, + no fusion, no reranking, no SQL path; +* top-k by cosine similarity, k=5; +* **the same generator**, called the same way. + +Deviating on the generator or the embedding model would make the delta unreadable: +the point is that only the retrieval architecture differs. + +Region attribution +------------------ +A naive splitter cuts across the region boundaries that gold labels are written +against, so each chunk records every region it overlaps (``region_ids``) and names +the region contributing the most characters as its primary ``region_id``. Scoring a +chunk as a miss because its dominant region was the neighbour of the gold one would +measure the label format, not the retriever. +""" + +from __future__ import annotations + +import re +import time +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from docureason.tripath.config import DocuReasonConfig +from docureason.tripath.ingestion.identity import canonical_region_id +from docureason.tripath.model_pins import resolve_revision +from docureason.tripath.utils import StageTimer, get_logger + +logger = get_logger(__name__) + +#: Split points tried in order, coarsest first - the standard recursive-splitter +#: ladder. The last is a hard character cut, which is why the splitter always makes +#: progress even on text with no whitespace at all. +_SEPARATORS = ("\n\n", "\n", ". ", " ", "") + + +@dataclass +class NaiveChunk: + """One fixed-size window of a document, with the regions it overlaps.""" + + chunk_id: str + document_id: str + text: str + region_id: str + region_ids: List[str] = field(default_factory=list) + + def as_payload(self, score: float, rank: int, mode: str) -> Dict[str, Any]: + return { + "id": self.chunk_id, + "chunk_id": self.chunk_id, + "document_id": self.document_id, + "region_id": self.region_id, + "region_ids": list(self.region_ids), + "text": self.text, + "score": round(float(score), 4), + "rank": rank, + "modality": "text", + "retrieval_mode": mode, + } + + +def recursive_split(text: str, chunk_size: int, overlap: int) -> List[Tuple[int, int]]: + """``(start, end)`` character spans for *text*, recursively split. + + Spans rather than strings: the caller needs the offsets to work out which regions + each chunk overlaps, and slicing twice would be the only alternative. + """ + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + stride = max(1, chunk_size - max(0, overlap)) + spans: List[Tuple[int, int]] = [] + position = 0 + length = len(text) + + while position < length: + end = min(position + chunk_size, length) + if end < length: + end = _best_break(text, position, end) + spans.append((position, end)) + if end >= length: + break + position = max(position + stride, position + 1) if end - position >= stride else end + return spans + + +def _best_break(text: str, start: int, hard_end: int) -> int: + """The latest natural boundary at or before *hard_end*, else *hard_end*.""" + window = text[start:hard_end] + for separator in _SEPARATORS: + if not separator: + break + cut = window.rfind(separator) + # Refuse a break in the first quarter of the window: it would emit a sliver + # and push the real content into the next chunk. + if cut > len(window) // 4: + return start + cut + len(separator) + return hard_end + + +class NaiveRAGBaseline: + """Recursive-split + flat-index + top-k retrieval, sharing the full system's generator.""" + + def __init__( + self, + config: Optional[DocuReasonConfig] = None, + chunk_size: int = 1000, + chunk_overlap: int = 200, + top_k: int = 5, + generator: Optional[Any] = None, + ) -> None: + self.config = config or DocuReasonConfig.load_preset("balanced") + self.chunk_size = chunk_size + self.chunk_overlap = chunk_overlap + self.top_k = top_k + + self.chunks: List[NaiveChunk] = [] + self.degradation: List[str] = [] + self._model: Any = None + self._index: Any = None + self._generator = generator + + self.model_name = "sentence-transformers/all-MiniLM-L6-v2" + self.model_revision = resolve_revision(self.model_name) + + # ------------------------------------------------------------------ + # Build + # ------------------------------------------------------------------ + + def build(self, documents: Sequence[Any]) -> Dict[str, Any]: + """Split every document, embed the chunks, and build one flat index.""" + self.chunks = [] + self.degradation = [] + for document in documents: + self.chunks.extend(self._split_document(document)) + + stats: Dict[str, Any] = { + "documents": len(documents), + "chunks": len(self.chunks), + "chunk_size": self.chunk_size, + "chunk_overlap": self.chunk_overlap, + "top_k": self.top_k, + "embedding_model": self.model_name, + "embedding_revision": self.model_revision or "hub-default", + "index_type": "flat_inner_product", + } + + if not self.chunks: + self.degradation.append("baseline:empty_corpus") + stats["degraded"] = "empty_corpus" + return stats + + vectors = self._encode([chunk.text for chunk in self.chunks]) + if vectors is None: + self.degradation.append("baseline:embedder_unavailable") + stats["degraded"] = "embedder_unavailable" + stats["index_type"] = "keyword_overlap_fallback" + return stats + + try: + import faiss + + index = faiss.IndexFlatIP(vectors.shape[1]) + index.add(vectors) + if index.ntotal != len(self.chunks): + raise RuntimeError(f"index holds {index.ntotal} of {len(self.chunks)} chunks") + self._index = index + stats["ntotal"] = int(index.ntotal) + except Exception as exc: + # Same policy as the full system's index builders: a missing optional + # dependency degrades and says so, rather than answering every query with + # an empty list and looking like a corpus with no relevant documents. + logger.warning("NaiveRAGBaseline: flat index unavailable (%s) — using keyword overlap", exc) + self.degradation.append("baseline:faiss_unavailable") + stats["degraded"] = "faiss_unavailable" + stats["index_type"] = "keyword_overlap_fallback" + + return stats + + def _split_document(self, document: Any) -> List[NaiveChunk]: + """Concatenate the document's regions, then split without regard for them.""" + pieces: List[str] = [] + offsets: List[Tuple[int, int, str]] = [] # (start, end, region_id) + cursor = 0 + + for index, region in enumerate(getattr(document, "regions", []) or []): + text = str(getattr(region, "text", "") or "") + if not text.strip(): + continue + region_id = canonical_region_id(document.id, region, index) + pieces.append(text) + offsets.append((cursor, cursor + len(text), region_id)) + cursor += len(text) + 2 # the "\n\n" join below + + if not pieces: + return [] + + full_text = "\n\n".join(pieces) + chunks: List[NaiveChunk] = [] + for number, (start, end) in enumerate(recursive_split(full_text, self.chunk_size, self.chunk_overlap)): + body = full_text[start:end].strip() + if not body: + continue + overlapped = _overlapping_regions(start, end, offsets) + chunks.append( + NaiveChunk( + chunk_id=f"{document.id}-naive-{number}", + document_id=str(document.id), + text=body, + region_id=overlapped[0] if overlapped else "", + region_ids=overlapped, + ) + ) + return chunks + + # ------------------------------------------------------------------ + # Query + # ------------------------------------------------------------------ + + def query(self, text: str) -> Dict[str, Any]: + """Retrieve top-k chunks and generate an answer, in the same response shape. + + The shape matches :meth:`QueryService.query` closely enough for the evaluation + harness to score both systems through one code path - which is the point, since + a baseline scored by different code is not a baseline. + """ + timer = StageTimer() + with timer.stage("dense_search"): + results = self._search(text) + with timer.stage("generate"): + generation = self._generate(text, results) + + degradation = list(self.degradation) + engine = generation.get("engine", "unknown") + if engine not in ("local_slm_deepseek_r1", "cloud_gemini_api"): + degradation.append(f"generation:{engine}") + + return { + "query": text, + "system": "naive_rag_baseline", + "results": results, + "retrieved_evidence": results[: self.top_k], + "answer": generation.get("answer", ""), + "generation": generation, + "generation_engine": engine, + "citations": generation.get("citations", []), + "retrieval_stats": { + "mode": "flat_index" if self._index is not None else "keyword_overlap", + "candidates": len(self.chunks), + "returned": len(results), + }, + "latency_breakdown": timer.as_dict(), + "degradation": degradation, + "cached": False, + } + + def _search(self, text: str) -> List[Dict[str, Any]]: + if not self.chunks: + return [] + + if self._index is not None: + query_vector = self._encode([text]) + if query_vector is not None: + try: + import faiss # noqa: F401 + + scores, indices = self._index.search(query_vector, min(self.top_k, len(self.chunks))) + payloads = [] + for rank, (score, position) in enumerate(zip(scores[0], indices[0]), start=1): + if position < 0 or position >= len(self.chunks): + continue + payloads.append(self.chunks[position].as_payload(score, rank, "flat_index")) + return payloads + except Exception as exc: + logger.warning("NaiveRAGBaseline: flat search failed (%s) — using keyword overlap", exc) + + return self._keyword_search(text) + + def _keyword_search(self, text: str) -> List[Dict[str, Any]]: + """Term-overlap ranking, used only when the model or index is unavailable.""" + terms = set(re.findall(r"\w+", text.lower())) + if not terms: + return [] + scored: List[Tuple[float, NaiveChunk]] = [] + for chunk in self.chunks: + words = set(re.findall(r"\w+", chunk.text.lower())) + overlap = len(terms & words) / max(1, len(terms)) + if overlap > 0: + scored.append((overlap, chunk)) + scored.sort(key=lambda pair: pair[0], reverse=True) + return [ + chunk.as_payload(score, rank, "keyword_overlap") + for rank, (score, chunk) in enumerate(scored[: self.top_k], start=1) + ] + + def _generate(self, text: str, results: List[Dict[str, Any]]) -> Dict[str, Any]: + generator = self._get_generator() + if generator is None: + return {"answer": "", "engine": "unavailable", "citations": []} + # No SQL payload: the baseline has no SQL path, and passing an empty one is + # what "the same generator, less input" means. + return generator.generate(text, results, {"executed": False, "sql_results": []}) + + def _get_generator(self) -> Optional[Any]: + if self._generator is not None: + return self._generator + try: + from docureason.tripath.generation.generate import GenerationModule + + self._generator = GenerationModule(config=self.config) + except Exception as exc: + logger.warning("NaiveRAGBaseline: generator unavailable: %s", exc) + self._generator = None + return self._generator + + # ------------------------------------------------------------------ + # Embedding + # ------------------------------------------------------------------ + + def _encode(self, texts: List[str]) -> Optional[Any]: + model = self._get_model() + if model is None: + return None + try: + started = time.perf_counter() + vectors = model.encode(texts, batch_size=32, show_progress_bar=False, normalize_embeddings=True) + logger.debug("NaiveRAGBaseline: encoded %d text(s) in %.3fs", len(texts), time.perf_counter() - started) + return vectors.astype("float32") + except Exception as exc: + logger.warning("NaiveRAGBaseline: encoding failed: %s", exc) + return None + + def _get_model(self) -> Optional[Any]: + if self._model is not None: + return self._model + for kwargs in ({"revision": self.model_revision} if self.model_revision else {}, {"local_files_only": True}): + try: + from sentence_transformers import SentenceTransformer + + self._model = SentenceTransformer(self.model_name, **kwargs) + return self._model + except Exception as exc: + logger.debug("NaiveRAGBaseline: model load attempt failed (%s)", exc) + logger.warning("NaiveRAGBaseline: embedding model unavailable — falling back to keyword overlap") + return None + + +def _overlapping_regions(start: int, end: int, offsets: List[Tuple[int, int, str]]) -> List[str]: + """Region ids overlapping ``[start, end)``, the largest contributor first.""" + overlaps: List[Tuple[int, str]] = [] + for region_start, region_end, region_id in offsets: + shared = min(end, region_end) - max(start, region_start) + if shared > 0: + overlaps.append((shared, region_id)) + overlaps.sort(key=lambda pair: pair[0], reverse=True) + return [region_id for _, region_id in overlaps] + + +__all__ = ["NaiveChunk", "NaiveRAGBaseline", "recursive_split"] diff --git a/docureason/tripath/evaluation/edgar.py b/docureason/tripath/evaluation/edgar.py new file mode 100644 index 0000000..35364aa --- /dev/null +++ b/docureason/tripath/evaluation/edgar.py @@ -0,0 +1,282 @@ +"""edgar.py — fetch real SEC filings to evaluate against. + +Everything the harness could score before came from eight fixture files totalling +about 3 KB, plus one PDF, with hand-written questions whose "answers" were invented +(``benchmark_dataset.py``: ``"answer": "2.4M"`` against a table that does not contain +2.4M). Audit §10 is blunt about the consequence: with a corpus that small and gold +labels that arbitrary, no retrieval number the system reports is falsifiable. This +module fetches the real thing. + +Why EDGAR: filings are public, free, stable at a permanent URL, and they are exactly +the document class the system claims to serve - multi-column prose, dense financial +tables, footnotes and scale qualifiers ("in thousands") all in one file. + +Access rules, which this module follows rather than works around: + +* SEC requires a descriptive ``User-Agent`` with contact information. Set + ``DOCUREASON_SEC_USER_AGENT``; there is no usable default, and requests are refused + without one rather than sent with a fake identity. +* Fair-access asks for at most 10 requests/second. Requests are serialized with a + delay, so a corpus build is slow by design. +* Filings are cached on disk by accession number. A second build re-reads them + instead of re-fetching, so iterating on the query generator costs SEC nothing. + +Provenance is written next to the documents: URL, accession number, form type, filing +date and a SHA-256 of the bytes. Without it a corpus is unreproducible, and "we +evaluated on 24 filings" is not a checkable statement. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from docureason.tripath.utils import get_logger + +logger = get_logger(__name__) + +SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik:010d}.json" +ARCHIVE_URL = "https://www.sec.gov/Archives/edgar/data/{cik}/{accession}/{document}" + +#: Environment variable carrying the SEC-required contact identity. +USER_AGENT_ENV = "DOCUREASON_SEC_USER_AGENT" + +#: Seconds between requests. SEC asks for <=10/s; this is deliberately well under. +REQUEST_DELAY_SEC = 0.25 + +#: A spread of large filers across sectors, so a corpus built from the default set is +#: not four companies with the same disclosure template. (name, CIK). +DEFAULT_COMPANIES: Tuple[Tuple[str, int], ...] = ( + ("Apple", 320193), + ("Microsoft", 789019), + ("JPMorgan Chase", 19617), + ("Johnson & Johnson", 200406), + ("Exxon Mobil", 34088), + ("Walmart", 104169), + ("Caterpillar", 18230), + ("Pfizer", 78003), + ("Coca-Cola", 21344), + ("Boeing", 12927), + ("Verizon", 732712), + ("Procter & Gamble", 80424), +) + + +class EdgarAccessError(RuntimeError): + """Raised when EDGAR cannot be reached or refuses the request.""" + + +@dataclass +class FetchedFiling: + """One filing on disk, with enough provenance to re-fetch or verify it.""" + + company: str + cik: int + form: str + filing_date: str + accession: str + document: str + url: str + path: str + bytes: int + sha256: str + fetched_at: str + from_cache: bool = False + + def as_dict(self) -> Dict[str, Any]: + return asdict(self) + + +def user_agent() -> str: + """The configured SEC contact identity, or raise. + + Refusing is the point: sending a made-up identity to a government service to keep + a build green is not a tradeoff worth making, and SEC blocks anonymous scrapers + anyway - so a fake default would fail later and less clearly. + """ + value = os.getenv(USER_AGENT_ENV, "").strip() + if not value: + raise EdgarAccessError( + f"SEC requires a descriptive User-Agent with contact info. Set {USER_AGENT_ENV}, e.g.\n" + f' {USER_AGENT_ENV}="AcmeResearch corpus-builder (research@acme.example)"' + ) + return value + + +def _get(url: str, timeout: float = 30.0) -> bytes: + request = urllib.request.Request( + url, + headers={ + "User-Agent": user_agent(), + "Accept-Encoding": "gzip, deflate", + }, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = response.read() + if response.headers.get("Content-Encoding") == "gzip": + import gzip + + payload = gzip.decompress(payload) + return payload + except urllib.error.HTTPError as exc: + raise EdgarAccessError(f"{url} -> HTTP {exc.code} {exc.reason}") from exc + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise EdgarAccessError(f"{url} -> {exc}") from exc + + +def list_filings(cik: int, form: str = "10-K", limit: int = 2) -> List[Dict[str, Any]]: + """The most recent *limit* filings of type *form* for *cik*.""" + payload = json.loads(_get(SUBMISSIONS_URL.format(cik=cik)).decode("utf-8", errors="replace")) + recent = (payload.get("filings") or {}).get("recent") or {} + forms = recent.get("form") or [] + accessions = recent.get("accessionNumber") or [] + documents = recent.get("primaryDocument") or [] + dates = recent.get("filingDate") or [] + + out: List[Dict[str, Any]] = [] + for index, form_type in enumerate(forms): + if form_type != form: + continue + if index >= min(len(accessions), len(documents), len(dates)): + break + document = documents[index] + # Filings whose primary document is not HTML would land in the rule-based + # segmenter, producing a corpus entry with no tables and no reading order. + if not str(document).lower().endswith((".htm", ".html")): + continue + out.append( + { + "accession": accessions[index], + "document": document, + "filing_date": dates[index], + "form": form_type, + } + ) + if len(out) >= limit: + break + return out + + +def fetch_filings( + dest: Path | str, + companies: Sequence[Tuple[str, int]] = DEFAULT_COMPANIES, + form: str = "10-K", + per_company: int = 1, + limit: Optional[int] = None, + delay: float = REQUEST_DELAY_SEC, +) -> List[FetchedFiling]: + """Download up to *limit* filings into *dest*, round-robin across *companies*. + + Round-robin rather than company-by-company: a partial build (a rate limit, a + dropped connection) then still covers several issuers instead of exhausting one. + """ + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + user_agent() # fail before any request if the identity is not configured + + wanted: List[Tuple[str, int, Dict[str, Any]]] = [] + per_company_lists: List[List[Tuple[str, int, Dict[str, Any]]]] = [] + for name, cik in companies: + try: + filings = list_filings(cik, form=form, limit=per_company) + except EdgarAccessError as exc: + logger.warning("EDGAR listing failed for %s (CIK %d): %s", name, cik, exc) + continue + per_company_lists.append([(name, cik, filing) for filing in filings]) + time.sleep(delay) + + for depth in range(per_company): + for entries in per_company_lists: + if depth < len(entries): + wanted.append(entries[depth]) + if limit is not None: + wanted = wanted[:limit] + + fetched: List[FetchedFiling] = [] + for name, cik, filing in wanted: + accession = str(filing["accession"]).replace("-", "") + url = ARCHIVE_URL.format(cik=cik, accession=accession, document=filing["document"]) + target = dest / f"{_slug(name)}_{filing['form'].replace('/', '-')}_{filing['filing_date']}.html" + + if target.exists() and target.stat().st_size > 0: + body = target.read_bytes() + from_cache = True + else: + try: + body = _get(url) + except EdgarAccessError as exc: + logger.warning("EDGAR fetch failed for %s: %s", url, exc) + continue + target.write_bytes(body) + from_cache = False + time.sleep(delay) + + fetched.append( + FetchedFiling( + company=name, + cik=cik, + form=str(filing["form"]), + filing_date=str(filing["filing_date"]), + accession=str(filing["accession"]), + document=str(filing["document"]), + url=url, + path=str(target), + bytes=len(body), + sha256=hashlib.sha256(body).hexdigest(), + fetched_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + from_cache=from_cache, + ) + ) + logger.info( + "%s %s %s (%.1f KB)%s", + name, + filing["form"], + filing["filing_date"], + len(body) / 1024.0, + " [cached]" if from_cache else "", + ) + + _write_provenance(dest, fetched, form=form) + return fetched + + +def _write_provenance(dest: Path, fetched: Sequence[FetchedFiling], form: str) -> Path: + path = dest / "provenance.json" + path.write_text( + json.dumps( + { + "source": "SEC EDGAR", + "form": form, + "filing_count": len(fetched), + "user_agent": user_agent(), + "filings": [item.as_dict() for item in fetched], + }, + indent=2, + ), + encoding="utf-8", + ) + return path + + +def _slug(text: str) -> str: + keep = [ch.lower() if ch.isalnum() else "_" for ch in text] + return "".join(keep).strip("_").replace("__", "_") + + +__all__ = [ + "DEFAULT_COMPANIES", + "EdgarAccessError", + "FetchedFiling", + "USER_AGENT_ENV", + "fetch_filings", + "list_filings", + "user_agent", +] diff --git a/docureason/tripath/evaluation/eval_harness.py b/docureason/tripath/evaluation/eval_harness.py index 23785f6..38b2f95 100644 --- a/docureason/tripath/evaluation/eval_harness.py +++ b/docureason/tripath/evaluation/eval_harness.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Union from docureason.tripath.attribution.nli_attributor import ClaimSupportAttributor +from docureason.tripath.evaluation.statistics import DEFAULT_ITERATIONS, bootstrap_ci from docureason.tripath.evaluation.table_eval import TableEvaluator from docureason.tripath.utils import get_logger, trace_execution @@ -86,6 +87,14 @@ def _is_relevant(item: Dict[str, Any], gold: Set[str]) -> Optional[str]: value = item.get(field) if value is not None and str(value) in gold: return str(value) + + # A retriever whose units do not align to the gold units (the naive baseline + # splits across region boundaries by design) reports every region a chunk + # overlaps. Scoring such a chunk as a miss because its *dominant* region was + # the neighbour of the gold one would measure the label format, not retrieval. + for value in item.get("region_ids") or (): + if value is not None and str(value) in gold: + return str(value) return None def evaluate( @@ -112,8 +121,16 @@ def evaluate_single( sql_executed: Optional[bool] = None, latency_ms: Optional[float] = None, k_values: Optional[List[int]] = None, + stratum: Optional[str] = None, + latency_breakdown: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: - """Evaluates a single query run across 22 multi-dimensional evaluation metrics.""" + """Evaluates a single query run across 22 multi-dimensional evaluation metrics. + + *stratum* and *latency_breakdown* are carried through rather than computed: the + first lets :meth:`evaluate_suite` report per-stratum means (a corpus-wide mean + hides that tabular lookup and prose behave nothing alike), and the second lets + it decompose the latency the caller measured instead of only totalling it. + """ k_values = k_values or [5, 10, 20] relevant_ids = relevant_ids or [] gold: Set[str] = {str(r) for r in relevant_ids} @@ -232,6 +249,11 @@ def evaluate_single( est_cost_per_query = (est_tokens / 1_000_000.0) * 0.15 + 0.00005 metrics["est_cost_per_1k_queries_usd"] = round(est_cost_per_query * 1000.0, 5) + if stratum: + metrics["stratum"] = str(stratum) + if latency_breakdown: + metrics["latency_breakdown"] = latency_breakdown + return metrics def evaluate_suite( @@ -258,6 +280,8 @@ def evaluate_suite( ground_truth_table=case.get("ground_truth_table"), sql_executed=case.get("sql_executed"), latency_ms=case.get("latency_ms"), + stratum=case.get("stratum"), + latency_breakdown=case.get("latency_breakdown"), ) elapsed_ms = (time.perf_counter() - start_t) * 1000 if "latency_ms" not in m: @@ -278,10 +302,101 @@ def evaluate_suite( "suite_duration_sec": round(total_suite_sec, 3), "queries_per_second_qps": round(qps, 2), "summary_metrics": summary, + # A mean with no interval cannot distinguish a real gain from resampling + # noise, which on a suite this size is the difference of one query. + "confidence_intervals": self.confidence_intervals(results), + "stage_latency_ms": self.stage_latency_summary(results), + "per_stratum": self.stratified_summary(results), "target_verification": target_check, "detailed_results": results, } + #: Metrics that get a bootstrap interval. Deliberately short: an interval per + #: metric on all 22 would bury the four numbers any comparison actually turns on. + CI_METRICS = ("recall_at_5", "ndcg_at_5", "mrr", "map_at_10", "attribution_precision", "latency_ms") + + def confidence_intervals( + self, + results: List[Dict[str, Any]], + metrics: Optional[Iterable[str]] = None, + confidence: float = 0.95, + seed: int = 0, + iterations: int = DEFAULT_ITERATIONS, + ) -> Dict[str, Any]: + """95% percentile-bootstrap interval per headline metric. + + *iterations* is a parameter because the caller knows what the numbers are for: + a published report wants the default 10,000, while an ablation sweep computing + intervals for eight systems and every stratum wants far fewer and says so in + the artifact. + """ + out: Dict[str, Any] = {} + for name in metrics or self.CI_METRICS: + values = [r[name] for r in results if isinstance(r.get(name), (int, float))] + if not values: + continue + out[name] = bootstrap_ci(values, confidence=confidence, seed=seed, iterations=iterations).as_dict() + return out + + @staticmethod + def stage_latency_summary(results: List[Dict[str, Any]]) -> Dict[str, Any]: + """Per-stage mean and p95 across the suite, from each case's breakdown. + + Answers "which stage regressed", which a single end-to-end number never could. + Cases served from the response cache are excluded: their breakdown describes a + cache lookup, and averaging those in would flatter every stage at once. + """ + collected: Dict[str, List[float]] = {} + counted = 0 + for record in results: + breakdown = record.get("latency_breakdown") or {} + stages = breakdown.get("stages_ms") or {} + if not stages or breakdown.get("cached"): + continue + counted += 1 + for stage, elapsed in stages.items(): + collected.setdefault(stage, []).append(float(elapsed)) + + if not counted: + return {} + + summary: Dict[str, Any] = {"cases_with_breakdown": counted} + for stage, values in collected.items(): + ordered = sorted(values) + p95_index = max(0, int(round(0.95 * (len(ordered) - 1)))) + summary[stage] = { + "mean_ms": round(sum(ordered) / len(ordered), 3), + "p95_ms": round(ordered[p95_index], 3), + "observations": len(ordered), + } + return summary + + def stratified_summary( + self, + results: List[Dict[str, Any]], + iterations: int = DEFAULT_ITERATIONS, + ) -> Dict[str, Any]: + """Headline metrics per query stratum, with the interval for each. + + A corpus-wide mean can hold steady while tabular lookup collapses and prose + improves; the strata are the only place that shows. + """ + buckets: Dict[str, List[Dict[str, Any]]] = {} + for record in results: + stratum = record.get("stratum") + if stratum: + buckets.setdefault(str(stratum), []).append(record) + + out: Dict[str, Any] = {} + for stratum, records in sorted(buckets.items()): + latencies = [r.get("latency_ms", 0.0) for r in records] + out[stratum] = { + "n": len(records), + "summary_metrics": self._aggregate_summary(records, latencies), + "confidence_intervals": self.confidence_intervals(records, iterations=iterations), + } + return out + def verify_target_achievements(self, summary: Dict[str, float], targets: Dict[str, float]) -> Dict[str, Any]: """Compares achieved summary metrics against planned target thresholds.""" verification = {} diff --git a/docureason/tripath/evaluation/figures.py b/docureason/tripath/evaluation/figures.py new file mode 100644 index 0000000..fccc833 --- /dev/null +++ b/docureason/tripath/evaluation/figures.py @@ -0,0 +1,929 @@ +"""figures.py - publication figures and a derived-insight digest from the reports. + +Turns ``eval_suite.json`` and ``ablation_report.json`` into the small set of figures the +numbers actually support, plus an ``INSIGHTS.md`` whose every sentence is computed from +those files. Nothing here invents a series: a figure whose inputs are missing is skipped +and named in the digest, because an empty axis reads as "measured, and flat". + +Design constraints, held deliberately: + +* **One measure per axis.** Latency and [0, 1] quality metrics never share a chart - a + second y-scale is the one chart mistake that makes every comparison on it unreadable. +* **Intervals, not points.** Every estimate that has a bootstrap interval is drawn with + it. A bare bar at 0.42 over 20 queries claims a precision the data does not have. +* **The claim travels with the measurement.** The ablation figure plots each published + delta beside the measured one; a row that could not be measured is annotated in place + rather than dropped, so the figure cannot quietly retire an unverified number. +* **Colour carries identity only**, in a fixed order, and every bar is directly labelled + so nothing depends on hue - which also satisfies the contrast relief rule for the + third slot. + +The palette is the validated default categorical set (slots 1-3, light surface): worst +all-pairs CVD delta-E 9.2, normal-vision 24.0. Slot 3 sits below 3:1 against the surface, +so marks in it always carry a visible value label. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union + +from docureason.tripath.utils import get_logger + +logger = get_logger(__name__) + +# --- palette (light surface) ------------------------------------------------------- +SURFACE = "#fcfcfb" +INK = "#0b0b0b" +INK_SECONDARY = "#52514e" +INK_MUTED = "#898781" +GRID = "#e1e0d9" +AXIS = "#c3c2b7" +#: Categorical slots in fixed order. Never cycled: a fourth series folds into "other" +#: or becomes its own figure. +SERIES: Tuple[str, str, str] = ("#2a78d6", "#eb6834", "#1baf7a") + +#: Quality metrics live on [0, 1] and may share an axis. Latency does not, and gets its +#: own figures. +QUALITY_METRICS: Tuple[Tuple[str, str], ...] = ( + ("average_recall_at_5", "recall@5"), + ("average_precision_at_5", "precision@5"), + ("average_hit_rate_at_5", "hit rate@5"), + ("average_ndcg_at_5", "nDCG@5"), + ("mrr", "MRR"), + ("map_at_10", "MAP@10"), + ("attribution_precision", "attribution precision"), + ("hallucination_rate", "hallucination rate"), + ("token_f1", "token F1"), + ("exact_match", "exact match"), +) + +#: Summary key -> the per-case metric its bootstrap interval is stored under. +CI_ALIASES: Dict[str, str] = { + "average_recall_at_5": "recall_at_5", + "average_precision_at_5": "precision_at_5", + "average_hit_rate_at_5": "hit_rate_at_5", + "average_ndcg_at_5": "ndcg_at_5", + "mean_latency_ms": "latency_ms", +} + +DEFAULT_FORMATS: Tuple[str, ...] = ("png", "pdf") + + +def _pyplot(): + """Import matplotlib lazily, with a message that says what to install. + + Plotting is not a runtime dependency of the framework - a serving container has no + reason to carry it - so it lives in the ``figures`` extra and is imported here. + """ + try: + import matplotlib + except ImportError as exc: # pragma: no cover - exercised by the extra being absent + raise ImportError( + "figures require matplotlib: pip install 'docureason-framework[figures]'" + ) from exc + matplotlib.use("Agg") # headless: these are files, never a window + import matplotlib.pyplot as plt + + plt.rcParams.update( + { + "figure.facecolor": SURFACE, + "axes.facecolor": SURFACE, + "savefig.facecolor": SURFACE, + "font.family": "sans-serif", + "font.sans-serif": ["Segoe UI", "DejaVu Sans", "Arial", "sans-serif"], + "font.size": 9, + "axes.titlesize": 11, + "axes.titleweight": "semibold", + "axes.titlecolor": INK, + "axes.labelsize": 9, + "axes.labelcolor": INK_SECONDARY, + "axes.edgecolor": AXIS, + "axes.linewidth": 0.8, + "axes.spines.top": False, + "axes.spines.right": False, + "xtick.color": INK_MUTED, + "ytick.color": INK_SECONDARY, + "xtick.labelsize": 8, + "ytick.labelsize": 8.5, + "grid.color": GRID, + "grid.linewidth": 0.6, + "legend.frameon": False, + "legend.fontsize": 8.5, + "figure.dpi": 110, + } + ) + return plt + + +def _save(fig, out_dir: Path, stem: str, formats: Sequence[str]) -> List[Path]: + out_dir.mkdir(parents=True, exist_ok=True) + written: List[Path] = [] + for extension in formats: + path = out_dir / f"{stem}.{extension}" + # PNG for the notebook, PDF for a paper: same figure, no re-styling. + fig.savefig(path, dpi=200 if extension == "png" else None, bbox_inches="tight") + written.append(path) + fig.clf() + _pyplot().close(fig) + return written + + +def _number(value: Any) -> Optional[float]: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def _grid_x(ax) -> None: + ax.grid(axis="x", zorder=0) + ax.set_axisbelow(True) + + +def _caption(fig, text: str) -> None: + """One line under the figure saying what the reader is looking at and over what n.""" + fig.text(0.0, -0.02, text, ha="left", va="top", fontsize=7.5, color=INK_MUTED, wrap=True) + + +def _top_down(ax, count: int) -> None: + """Row 0 at the top. Idempotent, because a figure may draw two series on one axis.""" + ax.set_ylim(count - 0.6, -0.6) + + +def _forest( + ax, + labels: Sequence[str], + points: Sequence[Optional[float]], + lows: Sequence[Optional[float]], + highs: Sequence[Optional[float]], + color: str = SERIES[0], + label: Optional[str] = None, + offset: float = 0.0, + marker: str = "o", + filled: Optional[Sequence[bool]] = None, +) -> None: + """Dot-and-interval: the point estimate with the interval it actually carries.""" + positions = [index + offset for index in range(len(labels))] + labelled = False + for index, position in enumerate(positions): + point, low, high = points[index], lows[index], highs[index] + if point is None: + continue + if low is not None and high is not None: + ax.plot([low, high], [position, position], color=color, linewidth=2.0, solid_capstyle="round", zorder=2) + is_filled = True if filled is None else bool(filled[index]) + ax.plot( + [point], + [position], + marker=marker, + markersize=7.5, + color=color, + markerfacecolor=color if is_filled else SURFACE, + markeredgecolor=color, + markeredgewidth=1.6, + linestyle="none", + zorder=3, + # On the first mark actually drawn, not on index 0: the first row may be the + # one with nothing to plot, and the series would then vanish from the legend. + label=None if labelled or label is None else label, + ) + labelled = True + ax.set_yticks(range(len(labels))) + ax.set_yticklabels(labels) + + +def figure_headline_metrics(suite: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Every [0, 1] quality metric as a point estimate with its 95% bootstrap interval.""" + plt = _pyplot() + summary = suite.get("summary_metrics") or {} + intervals = suite.get("confidence_intervals") or {} + + labels, points, lows, highs = [], [], [], [] + for key, label in QUALITY_METRICS: + value = _number(summary.get(key)) + if value is None: + continue + interval = intervals.get(CI_ALIASES.get(key, key)) or {} + labels.append(label) + points.append(value) + lows.append(_number(interval.get("ci_low"))) + highs.append(_number(interval.get("ci_high"))) + + if not labels: + logger.warning("headline metrics: no quality metrics in the report; skipping") + return [] + + fig, ax = plt.subplots(figsize=(6.6, 0.42 * len(labels) + 1.3)) + _forest(ax, labels, points, lows, highs) + _top_down(ax, len(labels)) + for index, value in enumerate(points): + # Direct labels: the interval carries the uncertainty, the number carries the value, + # and neither depends on reading a position off an axis. + ax.annotate( + f"{value:.3f}", + (value, index), + xytext=(0, 9), + textcoords="offset points", + ha="center", + fontsize=7.5, + color=INK_SECONDARY, + ) + + ax.set_xlim(-0.03, 1.03) + ax.set_xlabel("score (0-1)") + ax.set_title("Retrieval, attribution and answer quality, with 95% bootstrap intervals") + _grid_x(ax) + scored = suite.get("queries_scored") + iterations = suite.get("bootstrap_iterations") + _caption( + fig, + f"n = {scored} gold queries; percentile bootstrap, {iterations} resamples. A metric with no " + "interval was not in the harness's interval set. hallucination_rate is inverted: lower is better.", + ) + return _save(fig, out_dir, "fig1_headline_metrics", formats) + + +def figure_per_stratum(suite: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Three metrics per stratum. The place a flat corpus-wide mean gives itself away.""" + plt = _pyplot() + per_stratum = suite.get("per_stratum") or {} + if not per_stratum: + logger.warning("per-stratum figure: report has no strata; skipping") + return [] + + metrics = (("average_recall_at_5", "recall@5"), ("average_ndcg_at_5", "nDCG@5"), ("token_f1", "token F1")) + strata = sorted(per_stratum) + height = 0.24 + + fig, ax = plt.subplots(figsize=(6.8, 1.15 * len(strata) + 1.5)) + for series_index, (key, label) in enumerate(metrics): + offsets = [index + (series_index - 1) * (height + 0.03) for index in range(len(strata))] + # A metric the stratum never had (prose queries carry no ground-truth answer, so + # token F1 does not exist for them) is annotated "n/a", never drawn as a zero bar: + # "measured zero" and "not measured" are different findings. + raw = [_number((per_stratum[stratum].get("summary_metrics") or {}).get(key)) for stratum in strata] + ax.barh( + offsets, + [value if value is not None else 0.0 for value in raw], + height=height, + color=SERIES[series_index], + label=label, + # A 2px surface gap between adjacent fills, so bars read as separate marks. + edgecolor=SURFACE, + linewidth=1.4, + zorder=2, + ) + for offset, value in zip(offsets, raw): + ax.annotate( + "n/a" if value is None else f"{value:.3f}", + (value or 0.0, offset), + xytext=(4, 0), + textcoords="offset points", + va="center", + fontsize=7.2, + color=INK_MUTED if value is None else INK_SECONDARY, + ) + + ax.set_yticks(range(len(strata))) + ax.set_yticklabels([f"{stratum}\n(n={per_stratum[stratum].get('n', '?')})" for stratum in strata]) + ax.invert_yaxis() + ax.set_xlim(0, 1.08) + ax.set_xlabel("score (0-1)") + ax.set_title("Quality by query stratum", pad=26) + ax.legend(loc="lower left", bbox_to_anchor=(0.0, 1.005), ncols=3, borderaxespad=0.0) + _grid_x(ax) + _caption( + fig, + "Every bar is labelled, so identity never rests on colour alone. A stratum whose " + "token F1 reads 0.000 had answers scored against a ground truth it did not match, not a missing metric.", + ) + return _save(fig, out_dir, "fig2_per_stratum", formats) + + +def _mean_residual_ms(suite: Dict[str, Any]) -> Tuple[Optional[float], Optional[float]]: + """Mean signed residual and mean total, over the cases that carried a breakdown.""" + residuals, totals = [], [] + for record in suite.get("per_query") or []: + breakdown = record.get("latency_breakdown") or {} + if breakdown.get("cached") or not breakdown.get("stages_ms"): + continue + residual, total = _number(breakdown.get("residual_ms")), _number(breakdown.get("total_ms")) + if residual is not None: + residuals.append(residual) + if total is not None: + totals.append(total) + mean_residual = sum(residuals) / len(residuals) if residuals else None + mean_total = sum(totals) / len(totals) if totals else None + return mean_residual, mean_total + + +def figure_stage_latency(suite: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Where the time goes, per stage: mean bar with a p95 tick on the same axis.""" + plt = _pyplot() + stages = suite.get("stage_latency_ms") or {} + rows = [ + (name, _number(block.get("mean_ms")), _number(block.get("p95_ms"))) + for name, block in stages.items() + if isinstance(block, dict) and _number(block.get("mean_ms")) is not None + ] + if not rows: + logger.warning("stage latency figure: no per-stage breakdown in the report; skipping") + return [] + + rows.sort(key=lambda row: row[1], reverse=True) + names = [row[0] for row in rows] + means = [row[1] for row in rows] + p95s = [row[2] for row in rows] + total_mean = sum(means) + + fig, ax = plt.subplots(figsize=(6.8, 0.36 * len(names) + 1.5)) + ax.barh(range(len(names)), means, height=0.6, color=SERIES[0], edgecolor=SURFACE, linewidth=1.2, zorder=2) + for index, (mean_ms, p95_ms) in enumerate(zip(means, p95s)): + share = (mean_ms / total_mean * 100.0) if total_mean else 0.0 + ax.annotate( + f"{mean_ms:,.0f} ms ({share:.0f}%)", + (mean_ms, index), + xytext=(5, 0), + textcoords="offset points", + va="center", + fontsize=7.5, + color=INK_SECONDARY, + ) + if p95_ms is not None: + # p95 as a tick on the same scale - a second statistic of the same measure, + # never a second axis. + ax.plot([p95_ms, p95_ms], [index - 0.28, index + 0.28], color=INK, linewidth=1.4, zorder=3) + + ax.set_yticks(range(len(names))) + ax.set_yticklabels(names) + ax.invert_yaxis() + ax.set_xlabel("milliseconds (bar = mean, vertical tick = p95)") + ax.set_title("Per-stage latency decomposition") + ax.set_xlim(0, max(max(means), max([value for value in p95s if value is not None], default=0)) * 1.28) + _grid_x(ax) + + residual, total = _mean_residual_ms(suite) + residual_note = ( + "residual not recorded" + if residual is None + else f"mean residual {residual:+,.0f} ms of {total:,.0f} ms total" + + (" - negative means the parallel paths overlapped" if residual < 0 else "") + ) + _caption( + fig, + f"{stages.get('cases_with_breakdown', '?')} uncached queries. Stages sum to {total_mean:,.0f} ms; " + f"{residual_note}. Cached responses are excluded: their breakdown describes a cache lookup.", + ) + return _save(fig, out_dir, "fig3_stage_latency", formats) + + +def figure_latency_distribution(suite: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """The whole latency distribution, because a mean hides the tail that hurts.""" + plt = _pyplot() + latencies = sorted( + value for value in (_number(record.get("latency_ms")) for record in suite.get("per_query") or []) if value + ) + if len(latencies) < 3: + logger.warning("latency distribution: fewer than 3 per-query latencies; skipping") + return [] + + cumulative = [(index + 1) / len(latencies) for index in range(len(latencies))] + fig, ax = plt.subplots(figsize=(6.6, 3.4)) + ax.step(latencies, cumulative, where="post", color=SERIES[0], linewidth=2.0, zorder=3) + + summary = suite.get("summary_metrics") or {} + for key, name in (("latency_p50_ms", "p50"), ("latency_p90_ms", "p90"), ("latency_p99_ms", "p99")): + value = _number(summary.get(key)) + if value is None: + continue + ax.axvline(value, color=INK_MUTED, linewidth=0.9, linestyle=(0, (4, 3)), zorder=2) + ax.annotate( + f"{name} {value:,.0f} ms", + (value, 0.02), + xytext=(3, 0), + textcoords="offset points", + rotation=90, + va="bottom", + fontsize=7.2, + color=INK_SECONDARY, + ) + + ax.set_ylim(0, 1.02) + ax.set_xlim(0, max(latencies) * 1.05) + ax.set_xlabel("end-to-end query latency (ms)") + ax.set_ylabel("fraction of queries at or below") + ax.set_title("Latency distribution (empirical CDF)") + ax.grid(axis="both") + ax.set_axisbelow(True) + _caption(fig, f"n = {len(latencies)} queries, measured around service.query() including generation and attribution.") + return _save(fig, out_dir, "fig4_latency_distribution", formats) + + +def figure_ablation_effects(report: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Measured effect beside the published claim, one row per ablation. + + The point of the figure: a row whose claim the measurement does not support should be + visibly unsupported, and a row that could not be measured should be visibly unmeasured + rather than absent. + """ + plt = _pyplot() + rows = report.get("ablations") or [] + if not rows: + logger.warning("ablation figure: report has no rows; skipping") + return [] + + labels, measured, lows, highs, claimed, notes, filled = [], [], [], [], [], [], [] + for row in rows: + primary = (row.get("comparisons") or {}).get(row.get("primary_metric")) or {} + label = str(row.get("label", row.get("key", "?"))).replace("Remove ", "") + labels.append(f"{label}\n({row.get('primary_metric')})") + claimed.append(_number(row.get("documented_claim_pct"))) + + if not row.get("executed"): + measured.append(None) + lows.append(None) + highs.append(None) + notes.append("not executed") + filled.append(False) + continue + if primary.get("comparable") is False: + measured.append(None) + lows.append(None) + highs.append(None) + notes.append("not comparable across corpora") + filled.append(False) + continue + + point = _number(row.get("measured_primary_delta_pct")) + reference = _number(primary.get("mean_full")) + measured.append(point) + if point is not None and reference not in (None, 0): + # The CI is in absolute units; the same denominator that produced the point + # estimate converts it, so the interval stays the interval of that estimate. + lows.append((_number(primary.get("delta_ci_low")) or 0.0) / reference * 100.0) + highs.append((_number(primary.get("delta_ci_high")) or 0.0) / reference * 100.0) + else: + lows.append(None) + highs.append(None) + p_value = _number(primary.get("p_value")) + notes.append( + f"{row.get('claim_supported', 'n/a')}" + ("" if p_value is None else f", p={p_value:.3f}") + if point is not None + else "no denominator" + ) + filled.append(bool(primary.get("significant"))) + + fig, ax = plt.subplots(figsize=(7.6, 0.78 * len(labels) + 1.7)) + ax.axvline(0, color=AXIS, linewidth=1.0, zorder=1) + _forest(ax, labels, measured, lows, highs, color=SERIES[0], label="measured (95% CI)", filled=filled) + _forest( + ax, + labels, + claimed, + [None] * len(labels), + [None] * len(labels), + color=SERIES[1], + label="published claim", + marker="D", + filled=[False] * len(labels), + ) + _top_down(ax, len(labels)) + + span = max([abs(value) for value in measured + claimed if value is not None] + [10.0]) + ax.set_xlim(-span * 1.35, span * 1.35) + for index, note in enumerate(notes): + ax.annotate( + note, + (ax.get_xlim()[1], index), + xytext=(-4, 0), + textcoords="offset points", + ha="right", + va="center", + fontsize=7.2, + color=INK_MUTED, + ) + + ax.set_xlabel("change in the row's primary metric when the component is removed (%)") + # Title padded so the legend can sit between it and the plot: "lower left" inside the + # axes landed on the bottom row's marks. + ax.set_title("Ablation effects: measured against published claim", pad=26) + ax.legend(loc="lower left", bbox_to_anchor=(0.0, 1.005), ncols=2, borderaxespad=0.0) + _grid_x(ax) + subset = report.get("query_subset") or {} + _caption( + fig, + f"n = {report.get('queries')} queries from {subset.get('source')}; " + f"{report.get('executed')} of {len(rows)} rows executed; full matrix: {subset.get('is_full_matrix')}. " + "A filled marker is significant at alpha=0.05 by paired sign-flip test; hollow is not. " + "Negative means removing the component made the metric worse, which is what a claim of " + "contribution predicts.", + ) + return _save(fig, out_dir, "fig5_ablation_effects", formats) + + +def figure_baseline_comparison(report: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Full system minus naive RAG, per metric, paired on the same queries.""" + plt = _pyplot() + baseline = report.get("baseline") or {} + comparisons = baseline.get("full_vs_baseline") or {} + if not comparisons: + logger.warning("baseline figure: no baseline comparison in the report; skipping") + return [] + + labels, points, lows, highs, filled, notes = [], [], [], [], [], [] + for key, label in QUALITY_METRICS + (("duplicate_rank_ratio", "duplicate rank ratio"),): + metric = CI_ALIASES.get(key, key) + comparison = comparisons.get(metric) + if not comparison: + continue + delta = _number(comparison.get("delta")) + if delta is None: + continue + labels.append(label) + points.append(delta) + lows.append(_number(comparison.get("delta_ci_low"))) + highs.append(_number(comparison.get("delta_ci_high"))) + filled.append(bool(comparison.get("significant"))) + p_value = _number(comparison.get("p_value")) + notes.append("" if p_value is None else f"p={p_value:.3f}") + + if not labels: + return [] + + fig, ax = plt.subplots(figsize=(7.0, 0.42 * len(labels) + 1.7)) + ax.axvline(0, color=AXIS, linewidth=1.0, zorder=1) + _forest(ax, labels, points, lows, highs, color=SERIES[0], filled=filled) + _top_down(ax, len(labels)) + for index, (delta, note) in enumerate(zip(points, notes)): + ax.annotate( + f"{delta:+.3f} {note}", + (delta, index), + xytext=(0, 9), + textcoords="offset points", + ha="center", + fontsize=7.2, + color=INK_SECONDARY, + ) + + span = max(abs(value) for value in points + [value for value in lows + highs if value is not None]) or 0.1 + ax.set_xlim(-span * 1.4, span * 1.4) + ax.set_xlabel(f"{baseline.get('system', 'baseline')} subtracted from the full system (absolute)") + ax.set_title("Full tri-path system vs the naive-RAG baseline") + _grid_x(ax) + _caption( + fig, + "Paired on the same queries and scored through the same harness, with the same generator and the " + "same embedding model, so the difference is the retrieval architecture. Filled markers are " + "significant at alpha=0.05; hollow are not. Latency is excluded here - it is not on this scale - " + "and appears in the latency figures.", + ) + return _save(fig, out_dir, "fig6_baseline_comparison", formats) + + +def figure_degradation(suite: Dict[str, Any], out_dir: Path, formats: Sequence[str]) -> List[Path]: + """Which fallback boundaries fired, and on how many queries. + + Worth a figure because it is the difference between a number produced by the + architecture and the same number produced by its fallbacks. + """ + plt = _pyplot() + counts = suite.get("degradation_counts") or {} + if not counts: + logger.info("degradation figure: no degradation reported; skipping (that is the good case)") + return [] + + rows = sorted(counts.items(), key=lambda item: item[1], reverse=True) + labels = [name for name, _ in rows] + values = [value for _, value in rows] + scored = suite.get("queries_scored") or max(values) + + fig, ax = plt.subplots(figsize=(7.0, 0.36 * len(labels) + 1.5)) + ax.barh(range(len(labels)), values, height=0.6, color=SERIES[0], edgecolor=SURFACE, linewidth=1.2, zorder=2) + for index, value in enumerate(values): + ax.annotate( + f"{value} of {scored} ({value / scored * 100:.0f}%)", + (value, index), + xytext=(5, 0), + textcoords="offset points", + va="center", + fontsize=7.5, + color=INK_SECONDARY, + ) + ax.set_yticks(range(len(labels))) + ax.set_yticklabels(labels) + ax.invert_yaxis() + ax.set_xlim(0, scored * 1.35) + ax.set_xlabel("queries affected") + ax.set_title("Degradation: which fallbacks fired during the run") + _grid_x(ax) + _caption( + fig, + "A label reaching 100% means that boundary fired on every query, so every metric above " + "describes the fallback rather than the component it replaced.", + ) + return _save(fig, out_dir, "fig7_degradation", formats) + + +def _sign_test_floor(n: int) -> Optional[float]: + """Smallest two-sided p an exact paired sign-flip test can return over *n* pairs. + + Worth stating: at n = 3 the floor is 0.25, so every row in such a run reads + "not significant" no matter how large the true effect. That is a property of the test, + not a finding about the system, and it is the single easiest number to misread. + """ + return 2.0 ** (1 - n) if 1 <= n <= 30 else None + + +def _insight_lines(suite: Optional[Dict[str, Any]], ablation: Optional[Dict[str, Any]]) -> List[str]: + lines: List[str] = ["# Insights", ""] + + if suite: + summary = suite.get("summary_metrics") or {} + intervals = suite.get("confidence_intervals") or {} + scored = suite.get("queries_scored") + lines += [f"## What the system scores, over {scored} gold queries", ""] + + for key, label in (("average_recall_at_5", "recall@5"), ("average_ndcg_at_5", "nDCG@5")): + value = _number(summary.get(key)) + interval = intervals.get(CI_ALIASES.get(key, key)) or {} + low, high = _number(interval.get("ci_low")), _number(interval.get("ci_high")) + if value is None: + continue + if low is not None and high is not None: + width = high - low + precision = ( + " The interval is wider than the estimate, so this run bounds the metric loosely rather " + "than measuring it." if value and width > value else "" + ) + lines.append(f"- **{label} = {value:.3f}**, 95% CI [{low:.3f}, {high:.3f}].{precision}") + else: + lines.append(f"- **{label} = {value:.3f}** (no interval recorded).") + + token_f1, exact = _number(summary.get("token_f1")), _number(summary.get("exact_match")) + if token_f1 is not None: + answer_note = ( + " Both are zero: the answers share no tokens with the gold strings, which for derived " + "table-cell labels usually means the generator returned the row rather than the cell." + if not token_f1 and not exact + else "" + ) + exact_text = "not measured" if exact is None else f"{exact:.3f}" + lines.append(f"- Answer quality: token F1 {token_f1:.3f}, exact match {exact_text}.{answer_note}") + + attribution, hallucination = _number(summary.get("attribution_precision")), _number(summary.get("hallucination_rate")) + if attribution is not None and hallucination is not None: + lines.append( + f"- Attribution precision {attribution:.3f}, hallucination rate {hallucination:.3f}. Read these as " + "claim-support coverage, not entailment: the attributor is lexical plus numeric consistency." + ) + + per_stratum = suite.get("per_stratum") or {} + scores = { + stratum: _number((block.get("summary_metrics") or {}).get("average_recall_at_5")) + for stratum, block in per_stratum.items() + } + scores = {stratum: value for stratum, value in scores.items() if value is not None} + if len(scores) >= 2: + best = max(scores, key=lambda name: scores[name]) + worst = min(scores, key=lambda name: scores[name]) + corpus_wide = _number(summary.get("average_recall_at_5")) + lines += [ + "", + "## Where it is strong and weak", + "", + f"- **{best}** recall@5 {scores[best]:.3f} vs **{worst}** {scores[worst]:.3f} - a spread of " + f"{scores[best] - scores[worst]:.3f}" + + (f" around a corpus-wide {corpus_wide:.3f}." if corpus_wide is not None else "."), + "- A single headline number cannot show that spread, which is why the strata are reported " + "separately rather than averaged away.", + ] + + if suite: + stages = suite.get("stage_latency_ms") or {} + rows = [ + (name, _number(block.get("mean_ms"))) + for name, block in stages.items() + if isinstance(block, dict) and _number(block.get("mean_ms")) is not None + ] + if rows: + rows.sort(key=lambda row: row[1], reverse=True) + total = sum(value for _, value in rows) + top_two = ", ".join(f"`{name}` {value:,.0f} ms ({value / total * 100:.0f}%)" for name, value in rows[:2]) + lines += ["", "## Where the time goes", "", f"- Dominated by {top_two} of {total:,.0f} ms across stages."] + residual, mean_total = _mean_residual_ms(suite) + if residual is not None and mean_total: + share = abs(residual) / mean_total * 100.0 + reading = ( + "stages overlap, which is the parallel dispatch doing its job" + if residual < 0 + else "time is spent outside the instrumented stages" + ) + lines.append( + f"- Mean residual {residual:+,.0f} ms of {mean_total:,.0f} ms ({share:.0f}% unaccounted): {reading}." + ) + p50, p90 = _number((suite.get("summary_metrics") or {}).get("latency_p50_ms")), _number( + (suite.get("summary_metrics") or {}).get("latency_p90_ms") + ) + if p50 and p90: + lines.append( + f"- Tail: p90 is {p90 / p50:.1f}x p50 ({p90:,.0f} ms vs {p50:,.0f} ms), so a mean latency " + "figure understates what a slow query costs." + ) + return lines + + +def _ablation_insight_lines(ablation: Dict[str, Any]) -> List[str]: + rows = ablation.get("ablations") or [] + if not rows: + return [] + + lines = ["", "## What each component contributes", ""] + queries = ablation.get("queries") or 0 + floor = _sign_test_floor(int(queries)) + subset = ablation.get("query_subset") or {} + lines.append( + f"- {ablation.get('executed')} of {len(rows)} rows executed over {queries} queries from " + f"`{subset.get('source')}`; full matrix: **{subset.get('is_full_matrix')}**." + ) + if floor is not None and floor > 0.05: + lines.append( + f"- **At n = {queries} the exact paired sign-flip test cannot return a p below {floor:.3f}.** Every " + "row here therefore reads *not significant* regardless of effect size: that is the test's floor, " + "not evidence the component does nothing. Raise the query count before reading these verdicts." + ) + + significant, contradicted, unmeasurable = [], [], [] + for row in rows: + primary = (row.get("comparisons") or {}).get(row.get("primary_metric")) or {} + name = str(row.get("label", row.get("key"))) + if not row.get("executed"): + unmeasurable.append(f"{name} (not executed: ingestion-scoped, needs its own corpus)") + continue + if primary.get("comparable") is False: + unmeasurable.append(f"{name} (not comparable: the corpus rebuild renames every gold region)") + continue + delta = _number(row.get("measured_primary_delta_pct")) + if delta is None: + unmeasurable.append(f"{name} (no denominator: the full system scores zero on this metric)") + continue + if primary.get("significant"): + direction = "worse" if delta < 0 else "better" + significant.append(f"{name}: {delta:+.1f}% on {row.get('primary_metric')} - removing it made the system {direction}") + if row.get("claim_supported") == "contradicted": + contradicted.append(f"{name}: measured {delta:+.1f}% against a published {row.get('documented_claim_pct'):+.1f}%") + + if significant: + lines += ["- Significant at alpha=0.05:"] + [f" - {item}" for item in significant] + else: + lines.append("- No row reached significance, so no component's contribution is established by this run.") + if contradicted: + lines += ["- **Claims the measurement reverses:**"] + [f" - {item}" for item in contradicted] + if unmeasurable: + lines += ["- Rows this run could not turn into a number:"] + [f" - {item}" for item in unmeasurable] + + for row in rows: + corpora = row.get("corpora") or {} + full, ablated = corpora.get("full") or {}, corpora.get("ablated") or {} + if full.get("total_regions") and ablated.get("total_regions"): + lines.append( + f"- What the {row.get('key')} corpus rebuild actually did: regions " + f"{full['total_regions']:,} -> {ablated['total_regions']:,}, serialized tables " + f"{full.get('structured_tables_serialized')} -> {ablated.get('structured_tables_serialized')}. " + "That census is the finding for this row; its recall delta is not comparable." + ) + + baseline = ablation.get("baseline") or {} + comparisons = baseline.get("full_vs_baseline") or {} + if comparisons: + favours_full = [ + metric + for metric, comparison in comparisons.items() + if (_number(comparison.get("delta")) or 0) > 0 and comparison.get("significant") + ] + favours_baseline = [ + metric + for metric, comparison in comparisons.items() + if (_number(comparison.get("delta")) or 0) < 0 and comparison.get("significant") + ] + lines += ["", f"## Against {baseline.get('system', 'the baseline')}", ""] + lines.append( + f"- Significantly better on: {', '.join(favours_full) if favours_full else 'nothing'}. " + f"Significantly worse on: {', '.join(favours_baseline) if favours_baseline else 'nothing'}." + ) + latency = comparisons.get("latency_ms") or {} + latency_delta = _number(latency.get("delta")) + if latency_delta is not None: + verdict = "significant" if latency.get("significant") else "not significant" + lines.append( + f"- Latency cost of the tri-path architecture: {latency_delta:+,.0f} ms per query ({verdict}). " + "Whether that buys anything is exactly what the quality rows above have to answer." + ) + return lines + + +def _caveat_lines(suite: Optional[Dict[str, Any]]) -> List[str]: + lines = ["", "## What this run does not establish", ""] + if suite: + counts = suite.get("degradation_counts") or {} + scored = suite.get("queries_scored") or 0 + always = [label for label, count in counts.items() if scored and count >= scored] + if always: + lines.append( + f"- These boundaries fired on **every** query: {', '.join(f'`{label}`' for label in always)}. " + "Any metric downstream of them describes the fallback, not the component it stands in for." + ) + modes = suite.get("retrieval_modes") or {} + scans = {label: count for label, count in modes.items() if "substring_scan" in label} + if scans: + lines.append( + f"- Substring-scan fallback on: {', '.join(f'`{label}` x{count}' for label, count in scans.items())}. " + "For those modalities the run measures a linear scan, not the index." + ) + if modes and not scans: + lines.append(f"- Every modality retrieved through its index: {', '.join(f'`{label}`' for label in modes)}.") + lines += [ + "- Gold labels are **derived, not annotated**: table cells know their region by construction and prose " + "anchors are phrases verified unique, which over-represents lookup and cannot express multi-hop reasoning.", + "- Attribution is lexical plus numeric consistency. `entailment_threshold` stays inert until a real NLI " + "model is loaded, so these numbers are not faithfulness measurements.", + "- Figures are regenerated from the JSON reports beside them; every number here is traceable to a file, " + "and none of it is carried over from a previous run.", + ] + return lines + + +def build_all( + eval_report: Optional[Union[str, Path]] = None, + ablation_report: Optional[Union[str, Path]] = None, + out_dir: Union[str, Path] = "artifacts/figures", + formats: Sequence[str] = DEFAULT_FORMATS, +) -> Dict[str, Any]: + """Render every figure the given reports support, and write ``INSIGHTS.md`` beside them. + + A missing report is not an error - it means those figures are skipped and the digest + says which, rather than an axis appearing with nothing on it. + """ + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + + suite = _load(eval_report) + ablation = _load(ablation_report) + if suite is None and ablation is None: + raise SystemExit("no reports to plot: pass --eval-report and/or --ablation-report") + + written: List[Path] = [] + skipped: List[str] = [] + + def _attempt(name: str, builder, payload: Optional[Dict[str, Any]]) -> None: + if payload is None: + skipped.append(f"{name} (report not supplied)") + return + paths = builder(payload, out, formats) + if paths: + written.extend(paths) + else: + skipped.append(f"{name} (inputs absent from the report)") + + _attempt("headline metrics", figure_headline_metrics, suite) + _attempt("per stratum", figure_per_stratum, suite) + _attempt("stage latency", figure_stage_latency, suite) + _attempt("latency distribution", figure_latency_distribution, suite) + _attempt("degradation", figure_degradation, suite) + _attempt("ablation effects", figure_ablation_effects, ablation) + _attempt("baseline comparison", figure_baseline_comparison, ablation) + + lines = _insight_lines(suite, ablation) + if ablation: + lines += _ablation_insight_lines(ablation) + lines += _caveat_lines(suite) + if skipped: + lines += ["", "## Figures not produced", ""] + [f"- {item}" for item in skipped] + + insights_path = out / "INSIGHTS.md" + insights_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + logger.info("Wrote %d figure file(s) and %s", len(written), insights_path) + return {"figures": [str(path) for path in written], "insights": str(insights_path), "skipped": skipped} + + +def _load(path: Optional[Union[str, Path]]) -> Optional[Dict[str, Any]]: + if not path: + return None + resolved = Path(path) + if not resolved.exists(): + logger.warning("report not found, skipping its figures: %s", resolved) + return None + return json.loads(resolved.read_text(encoding="utf-8")) + + +__all__ = [ + "CI_ALIASES", + "QUALITY_METRICS", + "SERIES", + "build_all", + "figure_ablation_effects", + "figure_baseline_comparison", + "figure_degradation", + "figure_headline_metrics", + "figure_latency_distribution", + "figure_per_stratum", + "figure_stage_latency", +] diff --git a/docureason/tripath/evaluation/gold_queries.py b/docureason/tripath/evaluation/gold_queries.py new file mode 100644 index 0000000..eb576c5 --- /dev/null +++ b/docureason/tripath/evaluation/gold_queries.py @@ -0,0 +1,495 @@ +"""gold_queries.py — build a stratified query set whose gold labels are derivable. + +The previous evaluation set was ``benchmark_dataset.py``: hand-written questions with +invented answers ("2.4M" against a table containing 100M and 125M) and gold ids naming +fixture stems that no longer resolved. Audit P0-4 and §10 both come back to the same +thing - if the labels are arbitrary, a retrieval score measures the labels. + +The approach here is different: **derive** each query from the corpus so its gold +region is known by construction, and refuse any query whose anchor is not unique. + +* ``tabular_lookup`` - a row label and a column header from a parsed table. The gold + region is the table; the answer is the cell where they intersect. +* ``numeric`` - the same, restricted to cells that parse as numbers, phrased as a + value question. Separate stratum because exact-figure retrieval is the claim the + SQL path exists to support. +* ``prose`` - a distinctive phrase from a body region, verified to appear in exactly + one region corpus-wide. +* ``figure`` - a figure caption, with the same uniqueness requirement. + +Uniqueness is the whole mechanism. A query is emitted only if its anchor identifies +one region across the entire corpus; otherwise a retriever could return a *correct* +region and be scored wrong, which measures the label format rather than retrieval. +Rejected candidates are counted by reason, so the yield is visible instead of the +generator quietly emitting whatever it managed. + +These labels are **programmatically derived, not human-annotated**, and every record +says so in ``provenance``. That is a real limitation: derivable queries over-represent +lookup and under-represent multi-hop reasoning, which no rule can generate. It is not +a substitute for annotation - it is what makes the numbers falsifiable without it. +""" + +from __future__ import annotations + +import json +import random +import re +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union + +from docureason.tripath.utils import get_logger + +logger = get_logger(__name__) + +STRATA = ("tabular_lookup", "numeric", "prose", "figure") + +#: Region types that count as prose for the prose stratum. +_PROSE_TYPES = frozenset({"body", "paragraph", "text"}) + +#: A phrase this short is unlikely to be distinctive; this long stops being a query +#: and starts being a copy of the region. +_MIN_ANCHOR_WORDS = 4 +_MAX_ANCHOR_WORDS = 9 + +#: Cell/label text this short carries no signal ("-", "1", "$"). +_MIN_LABEL_CHARS = 4 + +_NUMERIC = re.compile(r"^\(?-?[$€£]?\s?\d[\d,]*\.?\d*\s?[%)]?$") +_WORD = re.compile(r"[A-Za-z][A-Za-z'\-]+") + +#: Words too common to make a phrase distinctive, so an anchor must not be built +#: entirely from them. +_STOPWORDS = frozenset( + """the a an and or of to in for on at by with from as is are was were be been being + this that these those it its our we their his her they you your not no if then than + which who whom whose what when where why how all any both each few more most other + some such only own same so too very can will just should now""".split() +) + + +@dataclass +class GoldQuery: + """One evaluation case with a gold region derived from the corpus.""" + + id: str + stratum: str + query: str + relevant_ids: List[str] + document_id: str + source: str + derivation: str + anchor: Dict[str, Any] + ground_truth_answer: Optional[str] = None + provenance: str = "programmatic" + + def as_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "stratum": self.stratum, + "query": self.query, + "relevant_ids": list(self.relevant_ids), + "document_id": self.document_id, + "source": self.source, + "ground_truth_answer": self.ground_truth_answer, + "derivation": self.derivation, + "anchor": dict(self.anchor), + "provenance": self.provenance, + } + + +@dataclass +class GenerationReport: + """What the generator produced and, more usefully, what it threw away.""" + + queries: List[GoldQuery] = field(default_factory=list) + rejected: Counter = field(default_factory=Counter) + per_stratum: Counter = field(default_factory=Counter) + regions_scanned: int = 0 + documents: int = 0 + + def as_dict(self) -> Dict[str, Any]: + return { + "query_count": len(self.queries), + "documents": self.documents, + "regions_scanned": self.regions_scanned, + "per_stratum": dict(self.per_stratum), + "rejected": dict(self.rejected), + } + + +def _regions(corpus: Dict[str, Any]) -> List[Tuple[Dict[str, Any], Dict[str, Any]]]: + """``(document, region)`` pairs from a ``corpus.json`` payload.""" + pairs: List[Tuple[Dict[str, Any], Dict[str, Any]]] = [] + for document in corpus.get("documents") or []: + for region in document.get("regions") or []: + pairs.append((document, region)) + return pairs + + +def _region_id(region: Dict[str, Any]) -> str: + return str(region.get("id") or (region.get("metadata") or {}).get("region_id") or "") + + +def _normalise(text: str) -> str: + return " ".join(str(text).lower().split()) + + +def build_phrase_index(corpus: Dict[str, Any], anchor_words: int) -> Dict[str, List[str]]: + """Map every *anchor_words*-length word shingle to the regions containing it. + + Built once for the whole corpus: uniqueness has to be checked globally, and doing + it with a substring scan per candidate would be quadratic in corpus size. + """ + index: Dict[str, List[str]] = {} + for _, region in _regions(corpus): + region_id = _region_id(region) + if not region_id: + continue + words = _WORD.findall(_normalise(region.get("text", ""))) + seen_here = set() + for start in range(0, max(0, len(words) - anchor_words + 1)): + shingle = " ".join(words[start : start + anchor_words]) + if shingle in seen_here: + continue + seen_here.add(shingle) + index.setdefault(shingle, []).append(region_id) + return index + + +def _is_distinctive(shingle: str) -> bool: + """A phrase built only from stopwords cannot anchor a query.""" + words = shingle.split() + content = [word for word in words if word not in _STOPWORDS] + return len(content) >= max(2, len(words) // 2) + + +def _table_payload(region: Dict[str, Any]) -> Optional[Dict[str, Any]]: + raw = region.get("table_json") + if isinstance(raw, str): + try: + import json + + raw = json.loads(raw) + except Exception: + return None + return raw if isinstance(raw, dict) else None + + +def _is_numeric(value: Any) -> bool: + return bool(_NUMERIC.match(str(value).strip())) + + +def generate_queries( + corpus: Dict[str, Any], + per_stratum: int = 50, + seed: int = 0, + strata: Sequence[str] = STRATA, +) -> GenerationReport: + """Derive up to *per_stratum* queries for each stratum from *corpus*. + + Deterministic for a given corpus and seed, because an evaluation set that changes + between runs makes every comparison against it meaningless. + """ + rng = random.Random(seed) + report = GenerationReport() + pairs = _regions(corpus) + report.regions_scanned = len(pairs) + report.documents = len(corpus.get("documents") or []) + + phrase_indexes = { + words: build_phrase_index(corpus, words) for words in range(_MIN_ANCHOR_WORDS, _MAX_ANCHOR_WORDS + 1) + } + + counter = 0 + + def next_id() -> str: + nonlocal counter + counter += 1 + return f"q-{counter:04d}" + + table_pairs = [(d, r) for d, r in pairs if r.get("type") == "table" and _table_payload(r)] + prose_pairs = [(d, r) for d, r in pairs if r.get("type") in _PROSE_TYPES] + figure_pairs = [(d, r) for d, r in pairs if r.get("type") == "figure"] + rng.shuffle(table_pairs) + rng.shuffle(prose_pairs) + rng.shuffle(figure_pairs) + + if "tabular_lookup" in strata: + for query in _table_queries(table_pairs, per_stratum, rng, report, numeric_only=False, next_id=next_id): + report.queries.append(query) + report.per_stratum["tabular_lookup"] += 1 + + if "numeric" in strata: + rng.shuffle(table_pairs) + for query in _table_queries(table_pairs, per_stratum, rng, report, numeric_only=True, next_id=next_id): + report.queries.append(query) + report.per_stratum["numeric"] += 1 + + if "prose" in strata: + for query in _phrase_queries(prose_pairs, phrase_indexes, per_stratum, rng, report, "prose", next_id): + report.queries.append(query) + report.per_stratum["prose"] += 1 + + if "figure" in strata: + if not figure_pairs: + # Distinguish "no figure regions exist in this corpus" from "the generator + # found none it could use": HTML filings carry images that Docling's HTML + # pipeline does not emit as figure regions, so this stratum is empty by + # corpus composition, not by rejection. + report.rejected["figure:no_regions_in_corpus"] += 1 + for query in _phrase_queries(figure_pairs, phrase_indexes, per_stratum, rng, report, "figure", next_id): + report.queries.append(query) + report.per_stratum["figure"] += 1 + + logger.info( + "gold_queries: %d query(ies) from %d region(s) across %d document(s); rejected %s", + len(report.queries), + report.regions_scanned, + report.documents, + dict(report.rejected), + ) + return report + + +def _table_queries( + table_pairs: Sequence[Tuple[Dict[str, Any], Dict[str, Any]]], + limit: int, + rng: random.Random, + report: GenerationReport, + numeric_only: bool, + next_id: Any, +) -> Iterable[GoldQuery]: + """Row-label + column-header lookups, one per table at most. + + One per table keeps the set from being fifty questions about the same balance + sheet, which would make a per-stratum mean a statement about one table. + """ + stratum = "numeric" if numeric_only else "tabular_lookup" + produced = 0 + + for document, region in table_pairs: + if produced >= limit: + break + payload = _table_payload(region) + region_id = _region_id(region) + if not payload or not region_id: + report.rejected[f"{stratum}:no_payload"] += 1 + continue + + columns = [str(c).strip() for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + if len(columns) < 2 or not rows: + report.rejected[f"{stratum}:too_small"] += 1 + continue + + candidates: List[Tuple[str, str, str]] = [] # (row_label, column, value) + for row in rows: + if not row: + continue + label = str(row[0]).strip() + if len(label) < _MIN_LABEL_CHARS or _is_numeric(label): + continue + for position in range(1, min(len(row), len(columns))): + value = str(row[position]).strip() + column = columns[position] + if not value or len(column) < 2: + continue + if numeric_only and not _is_numeric(value): + continue + if not numeric_only and _is_numeric(value) and len(value) < _MIN_LABEL_CHARS: + continue + candidates.append((label, column, value)) + + if not candidates: + report.rejected[f"{stratum}:no_cell"] += 1 + continue + + label, column, value = rng.choice(candidates) + if numeric_only: + text = f"What is the {label} figure reported under {column}?" + else: + text = f"In the table reporting {label}, what value is shown for {column}?" + + yield GoldQuery( + id=next_id(), + stratum=stratum, + query=text, + relevant_ids=[region_id], + document_id=str(document.get("id", "")), + source=str(document.get("source", "")), + derivation="table_cell_numeric" if numeric_only else "table_cell", + anchor={"row_label": label, "column": column, "cell_value": value}, + ground_truth_answer=value, + ) + produced += 1 + + +def _phrase_queries( + pairs: Sequence[Tuple[Dict[str, Any], Dict[str, Any]]], + phrase_indexes: Dict[int, Dict[str, List[str]]], + limit: int, + rng: random.Random, + report: GenerationReport, + stratum: str, + next_id: Any, +) -> Iterable[GoldQuery]: + """Queries anchored on a phrase that occurs in exactly one region corpus-wide.""" + produced = 0 + + for document, region in pairs: + if produced >= limit: + break + region_id = _region_id(region) + text = _normalise(region.get("text", "")) + words = _WORD.findall(text) + if not region_id or len(words) < _MIN_ANCHOR_WORDS + 2: + report.rejected[f"{stratum}:too_short"] += 1 + continue + + anchor = _unique_phrase(words, region_id, phrase_indexes, rng) + if anchor is None: + # Boilerplate regions (risk-factor headers, legal footers) repeat verbatim + # across filings and have no unique phrase; they cannot be gold-labelled. + report.rejected[f"{stratum}:no_unique_phrase"] += 1 + continue + + if stratum == "figure": + query_text = f"Which figure describes {anchor}?" + else: + query_text = f"What does the filing say about {anchor}?" + + yield GoldQuery( + id=next_id(), + stratum=stratum, + query=query_text, + relevant_ids=[region_id], + document_id=str(document.get("id", "")), + source=str(document.get("source", "")), + derivation="unique_phrase", + anchor={"phrase": anchor, "phrase_words": len(anchor.split())}, + ground_truth_answer=None, + ) + produced += 1 + + +def _unique_phrase( + words: List[str], + region_id: str, + phrase_indexes: Dict[int, Dict[str, List[str]]], + rng: random.Random, +) -> Optional[str]: + """A shingle from *words* that appears only in *region_id*, or None.""" + lengths = sorted(phrase_indexes) + for length in lengths: + index = phrase_indexes[length] + starts = list(range(0, max(0, len(words) - length + 1))) + rng.shuffle(starts) + for start in starts[:40]: + shingle = " ".join(words[start : start + length]) + if not _is_distinctive(shingle): + continue + owners = index.get(shingle) or [] + if len(owners) == 1 and owners[0] == region_id: + return shingle + return None + + +def validate_queries(queries: Sequence[Dict[str, Any]], corpus: Dict[str, Any]) -> Dict[str, Any]: + """Check every gold id resolves to a region, and every anchor is still unique. + + Run after generation and again before any evaluation: a corpus rebuilt with + different chunk sizes or a different digest invalidates the gold ids, and silently + scoring against dangling labels is how a suite comes to report zero for reasons + that have nothing to do with retrieval. + + Uniqueness is checked over the **same word-token shingles** the generator used, not + by raw substring search. Those are not the same test: tokenizing drops digits and + punctuation, so ``"revenue increased compared to"`` is a valid token shingle of + ``"revenue increased 4% compared to"`` and is not a substring of it. Validating + with the stricter test reported 21 sound anchors as broken - a disagreement between + two definitions of the same property, not a defect in the data. + """ + known = {_region_id(region) for _, region in _regions(corpus) if _region_id(region)} + + dangling: List[str] = [] + non_unique: List[str] = [] + + phrase_lengths = { + len(str((record.get("anchor") or {}).get("phrase", "")).split()) + for record in queries + if (record.get("anchor") or {}).get("phrase") + } + indexes = {length: build_phrase_index(corpus, length) for length in phrase_lengths if length > 0} + + for record in queries: + for gold in record.get("relevant_ids") or []: + if gold not in known: + dangling.append(f"{record.get('id')}:{gold}") + + phrase = (record.get("anchor") or {}).get("phrase") + if not phrase: + continue + owners = indexes.get(len(str(phrase).split()), {}).get(str(phrase)) or [] + expected = set(record.get("relevant_ids") or []) + if len(owners) != 1 or set(owners) != expected: + non_unique.append(f"{record.get('id')}:{len(owners)} owner(s)") + + return { + "queries": len(queries), + "regions_in_corpus": len(known), + "dangling_gold_ids": dangling, + "non_unique_anchors": non_unique, + "valid": not dangling and not non_unique, + } + + +def load_eval_cases(path: Union[str, Path], limit: int = 0, stratified: bool = False) -> List[Dict[str, Any]]: + """Read a generated query file back as harness cases. + + Lives here rather than in a script because both the ablation sweep and the eval suite + need it and must agree: two loaders that sample differently would report metrics over + two different query sets under one corpus name. + + *stratified* samples ``limit`` evenly across strata. It is not a nicety - the + generator emits stratum by stratum, so a plain prefix ``limit`` runs the whole + evaluation against tabular lookup alone and calls the result a corpus-wide number. + """ + payload = json.loads(Path(path).read_text(encoding="utf-8")) + queries = payload.get("queries") or [] + if not queries: + raise ValueError(f"{path} contains no queries") + + if limit and stratified: + by_stratum: Dict[str, List[Dict[str, Any]]] = {} + for record in queries: + by_stratum.setdefault(str(record.get("stratum", "unknown")), []).append(record) + per = max(1, limit // max(1, len(by_stratum))) + picked: List[Dict[str, Any]] = [] + for stratum in sorted(by_stratum): + picked.extend(by_stratum[stratum][:per]) + queries = picked[:limit] + elif limit: + queries = queries[:limit] + + return [ + { + "query": record["query"], + "relevant_ids": record.get("relevant_ids", []), + "ground_truth_answer": record.get("ground_truth_answer"), + "stratum": record.get("stratum"), + } + for record in queries + ] + + +__all__ = [ + "STRATA", + "GenerationReport", + "GoldQuery", + "build_phrase_index", + "generate_queries", + "load_eval_cases", + "validate_queries", +] diff --git a/docureason/tripath/evaluation/statistics.py b/docureason/tripath/evaluation/statistics.py new file mode 100644 index 0000000..f41c837 --- /dev/null +++ b/docureason/tripath/evaluation/statistics.py @@ -0,0 +1,316 @@ +"""statistics.py — uncertainty and significance for evaluation metrics. + +Every number the harness reported was a bare mean over a handful of queries, with +no interval and no test. On a 3-case smoke suite the difference between +``recall@5 = 0.67`` and ``recall@5 = 1.00`` is one query, so a mean alone could not +distinguish a real improvement from resampling noise - and the ablation table in +``Documentations/eval_method.md`` quoted deltas to one decimal place with nothing +behind them. + +Three procedures, all non-parametric because metric distributions here are bounded, +discrete and badly skewed (recall@5 over 5 slots takes six possible values), which +rules out the normal approximation a t-interval assumes: + +* :func:`bootstrap_ci` - percentile bootstrap interval for the mean of one metric. +* :func:`paired_bootstrap` - interval for the *difference* between two systems + scored on the same queries. Paired, because the query-to-query variance dwarfs + the between-system variance: an unpaired test on the same data is far less + sensitive and would call real ablation effects insignificant. +* :func:`paired_permutation_test` - exact-when-small sign-flip test for the same + difference, giving a p-value that makes no distributional assumption at all. + +Implemented in pure Python against :mod:`random` rather than NumPy: this module is +imported by the CI metric gate, where a deterministic seed and no array dependency +matter more than speed. A 10,000-iteration bootstrap over a few hundred queries runs +in about a second. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence + +#: Default resample count. 10,000 puts the Monte-Carlo error on a 95% percentile +#: bound at roughly +/-0.005 of a metric point, an order of magnitude below the +#: 0.01 tolerance the CI regression gate uses. +DEFAULT_ITERATIONS = 10_000 + +#: Below this many observations, :func:`paired_permutation_test` enumerates all +#: 2**n sign assignments and returns an exact p-value instead of sampling. +EXACT_PERMUTATION_LIMIT = 14 + + +def mean(values: Sequence[float]) -> Optional[float]: + """Arithmetic mean, or ``None`` for an empty sample. + + ``None`` rather than ``0.0``: a metric with no observations is *unmeasured*, + and reporting it as zero is what let a disabled attribution gate publish a + perfect faithfulness score (audit P0-2). + """ + clean = [float(v) for v in values if v is not None] + if not clean: + return None + return sum(clean) / len(clean) + + +@dataclass +class ConfidenceInterval: + """A percentile bootstrap interval for the mean of one metric.""" + + point: Optional[float] + low: Optional[float] + high: Optional[float] + n: int + confidence: float = 0.95 + iterations: int = DEFAULT_ITERATIONS + + @property + def width(self) -> Optional[float]: + if self.low is None or self.high is None: + return None + return self.high - self.low + + def as_dict(self) -> Dict[str, Any]: + return { + "point": _round(self.point), + "ci_low": _round(self.low), + "ci_high": _round(self.high), + "ci_width": _round(self.width), + "n": self.n, + "confidence": self.confidence, + "bootstrap_iterations": self.iterations, + } + + +@dataclass +class PairedComparison: + """A paired comparison of two systems scored on the same queries.""" + + n: int + mean_a: Optional[float] + mean_b: Optional[float] + delta: Optional[float] + ci_low: Optional[float] + ci_high: Optional[float] + p_value: Optional[float] + alpha: float = 0.05 + iterations: int = DEFAULT_ITERATIONS + exact: bool = False + notes: List[str] = field(default_factory=list) + + @property + def significant(self) -> Optional[bool]: + """True when the paired test rejects at :attr:`alpha`. + + Reported alongside the interval, never instead of it: with n this small a + non-significant result usually means "not enough queries", not "no effect". + """ + if self.p_value is None: + return None + return self.p_value < self.alpha + + def as_dict(self) -> Dict[str, Any]: + return { + "n_paired": self.n, + "mean_a": _round(self.mean_a), + "mean_b": _round(self.mean_b), + "delta": _round(self.delta), + "delta_ci_low": _round(self.ci_low), + "delta_ci_high": _round(self.ci_high), + "p_value": _round(self.p_value, 5), + "alpha": self.alpha, + "significant": self.significant, + "test": "exact paired sign-flip" if self.exact else "sampled paired sign-flip", + "iterations": self.iterations, + "notes": list(self.notes), + } + + +def _round(value: Optional[float], digits: int = 4) -> Optional[float]: + return None if value is None else round(float(value), digits) + + +def bootstrap_ci( + values: Sequence[float], + confidence: float = 0.95, + iterations: int = DEFAULT_ITERATIONS, + seed: int = 0, +) -> ConfidenceInterval: + """Percentile bootstrap interval for the mean of *values*. + + Resamples the observations with replacement *iterations* times and takes the + empirical quantiles of the resampled means. A single observation yields a + degenerate interval (both bounds equal the point estimate) rather than an error: + that is the honest answer for n=1, and it makes the width visibly useless. + """ + clean = [float(v) for v in values if v is not None] + n = len(clean) + point = mean(clean) + if n == 0: + return ConfidenceInterval(None, None, None, 0, confidence, iterations) + if n == 1: + return ConfidenceInterval(point, point, point, 1, confidence, 0) + + rng = random.Random(seed) + means: List[float] = [] + for _ in range(iterations): + total = 0.0 + for _ in range(n): + total += clean[rng.randrange(n)] + means.append(total / n) + means.sort() + + alpha = 1.0 - confidence + return ConfidenceInterval( + point=point, + low=_quantile(means, alpha / 2.0), + high=_quantile(means, 1.0 - alpha / 2.0), + n=n, + confidence=confidence, + iterations=iterations, + ) + + +def _quantile(sorted_values: List[float], q: float) -> float: + """Linear-interpolated quantile of an already-sorted sample.""" + if not sorted_values: + raise ValueError("quantile of an empty sample") + if len(sorted_values) == 1: + return sorted_values[0] + position = q * (len(sorted_values) - 1) + lower = int(position) + upper = min(lower + 1, len(sorted_values) - 1) + weight = position - lower + return sorted_values[lower] * (1.0 - weight) + sorted_values[upper] * weight + + +def _pair(a: Sequence[float], b: Sequence[float]) -> List[tuple[float, float]]: + """Observations present in both samples, positionally aligned. + + Callers align by query id before calling; this drops any pair where either side + is missing so a metric that one system could not produce cannot silently shift + the other system's mean. + """ + pairs: List[tuple[float, float]] = [] + for left, right in zip(a, b): + if left is None or right is None: + continue + pairs.append((float(left), float(right))) + return pairs + + +def paired_bootstrap( + a: Sequence[float], + b: Sequence[float], + confidence: float = 0.95, + iterations: int = DEFAULT_ITERATIONS, + seed: int = 0, +) -> ConfidenceInterval: + """Bootstrap interval for the mean paired difference ``a - b``. + + Resamples *queries*, not scores, so each resample keeps both systems' results + for the same query together. An interval that excludes zero is the same + conclusion a significance test reaches, stated in metric units. + """ + pairs = _pair(a, b) + deltas = [left - right for left, right in pairs] + return bootstrap_ci(deltas, confidence=confidence, iterations=iterations, seed=seed) + + +def paired_permutation_test( + a: Sequence[float], + b: Sequence[float], + iterations: int = DEFAULT_ITERATIONS, + seed: int = 0, +) -> tuple[Optional[float], bool]: + """Two-sided paired sign-flip test on ``a - b``. + + Under the null hypothesis that the two systems are interchangeable, the sign of + each query's difference is arbitrary. The test enumerates (or samples) sign + assignments and reports how often ``|mean delta|`` is at least as extreme as + observed. Returns ``(p_value, exact)``; ``exact`` is True when all ``2**n`` + assignments were enumerated. + + The p-value uses the add-one correction ``(hits + 1) / (iterations + 1)`` in the + sampled case, so it can never be reported as exactly 0 - a sampled test cannot + justify that claim. + """ + pairs = _pair(a, b) + deltas = [left - right for left, right in pairs] + n = len(deltas) + if n == 0: + return None, False + + observed = abs(sum(deltas) / n) + # Ties contribute nothing under sign flipping; with every delta zero the two + # systems produced identical scores, which is p = 1 by definition. + if all(delta == 0.0 for delta in deltas): + return 1.0, True + + if n <= EXACT_PERMUTATION_LIMIT: + hits = 0 + total = 1 << n + for mask in range(total): + flipped = sum(-delta if (mask >> i) & 1 else delta for i, delta in enumerate(deltas)) + if abs(flipped / n) >= observed - 1e-12: + hits += 1 + return hits / total, True + + rng = random.Random(seed) + hits = 0 + for _ in range(iterations): + flipped = sum(delta if rng.random() < 0.5 else -delta for delta in deltas) + if abs(flipped / n) >= observed - 1e-12: + hits += 1 + return (hits + 1) / (iterations + 1), False + + +def compare_paired( + a: Sequence[float], + b: Sequence[float], + confidence: float = 0.95, + iterations: int = DEFAULT_ITERATIONS, + seed: int = 0, + alpha: float = 0.05, +) -> PairedComparison: + """Full paired comparison of *a* against *b*: delta, bootstrap CI and p-value.""" + pairs = _pair(a, b) + n = len(pairs) + notes: List[str] = [] + if n == 0: + return PairedComparison(0, None, None, None, None, None, None, alpha, iterations, False, ["no paired queries"]) + + dropped = max(len(a), len(b)) - n + if dropped > 0: + notes.append(f"{dropped} query(ies) dropped: metric missing on one side") + if n < 30: + notes.append(f"n={n} is small; a non-significant result here is weak evidence of no effect") + + interval = paired_bootstrap(a, b, confidence=confidence, iterations=iterations, seed=seed) + p_value, exact = paired_permutation_test(a, b, iterations=iterations, seed=seed) + + return PairedComparison( + n=n, + mean_a=mean([left for left, _ in pairs]), + mean_b=mean([right for _, right in pairs]), + delta=interval.point, + ci_low=interval.low, + ci_high=interval.high, + p_value=p_value, + alpha=alpha, + iterations=iterations, + exact=exact, + notes=notes, + ) + + +__all__ = [ + "ConfidenceInterval", + "PairedComparison", + "bootstrap_ci", + "compare_paired", + "mean", + "paired_bootstrap", + "paired_permutation_test", +] diff --git a/docureason/tripath/indexing/dense_index.py b/docureason/tripath/indexing/dense_index.py index 114e11c..d9a3c42 100644 --- a/docureason/tripath/indexing/dense_index.py +++ b/docureason/tripath/indexing/dense_index.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import threading from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -92,6 +93,13 @@ def __init__( self._model: Any = None # lazy-loaded SentenceTransformer #: Per-modality FAISS index cache so search() does not re-read from disk per query. self._index_cache: Dict[str, Any] = {} + #: Per-modality metadata cache. search() used to json.loads the whole sidecar on + #: every query, which on a large corpus cost more than the vector search itself. + self._meta_cache: Dict[str, List[Dict[str, Any]]] = {} + #: Guards the lazy model load and the two caches. The parallel modality + #: dispatch calls search() from three threads against this one builder, and + #: without the lock all three could load the embedding model concurrently. + self._load_lock = threading.Lock() log_pipeline_flag("dense_index_model", self.model_name, f"FAISS model (domain={self.domain})", logger) log_pipeline_flag("dense_index_type", self.index_type, f"FAISS index type (hnsw_m={self.hnsw_m})", logger) log_pipeline_flag( @@ -193,6 +201,11 @@ def build(self, records: List[Dict[str, Any]], modality: str) -> Path: faiss.write_index(index, str(index_path)) meta_path.write_text(meta_json, encoding="utf-8") + # A rebuild invalidates both caches; without this, search() in the same + # process would keep answering from the index this call just replaced. + with self._load_lock: + self._index_cache.pop(modality, None) + self._meta_cache.pop(modality, None) self.build_stats[modality] = { "records": len(records), "ntotal": int(index.ntotal), @@ -262,11 +275,20 @@ def search(self, query: str, modality: str, k: int = 5) -> List[Dict[str, Any]]: if model is None: return [] - meta = json.loads(meta_path.read_text(encoding="utf-8")) + meta = self._meta_cache.get(modality) index = self._index_cache.get(modality) - if index is None: - index = faiss.read_index(str(index_path)) - self._index_cache[modality] = index + if meta is None or index is None: + with self._load_lock: + # Re-check inside the lock: another thread may have loaded both + # while this one waited. + meta = self._meta_cache.get(modality) + if meta is None: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + self._meta_cache[modality] = meta + index = self._index_cache.get(modality) + if index is None: + index = faiss.read_index(str(index_path)) + self._index_cache[modality] = index if hasattr(index, "hnsw"): index.hnsw.efSearch = self.hnsw_ef_search @@ -292,9 +314,15 @@ def search(self, query: str, modality: str, k: int = 5) -> List[Dict[str, Any]]: return [] def _get_model(self) -> Optional[Any]: - """Lazy-load and cache the SentenceTransformer model.""" + """Lazy-load and cache the SentenceTransformer model (thread-safe).""" if self._model is not None: return self._model + with self._load_lock: + if self._model is not None: + return self._model + return self._load_model_locked() + + def _load_model_locked(self) -> Optional[Any]: try: from sentence_transformers import SentenceTransformer @@ -308,6 +336,34 @@ def _get_model(self) -> Optional[Any]: self.model_revision or "hub-default", ) except Exception as exc: - logger.warning("Failed to load SentenceTransformer (%s) — using fallback", exc) - self._model = None + # Pinning a revision makes the loader resolve that commit against the hub. + # When the hub is unreachable but the commit is already in the local cache, + # the right answer is the cached weights, not a degraded index — so retry + # offline before giving up. Without this, adding a pin turned every + # network hiccup into a silently empty dense index. + self._model = self._load_from_cache(exc) return self._model + + def _load_from_cache(self, first_error: Exception) -> Optional[Any]: + """Second attempt at the pinned weights, restricted to the local cache.""" + try: + from sentence_transformers import SentenceTransformer + + kwargs: Dict[str, Any] = {"local_files_only": True} + if self.model_revision: + kwargs["revision"] = self.model_revision + model = SentenceTransformer(self.model_name, **kwargs) + logger.warning( + "Hub unreachable (%s) — loaded %s from the local cache at revision %s", + first_error, + self.model_name, + self.model_revision or "hub-default", + ) + return model + except Exception as exc: + logger.warning( + "Failed to load SentenceTransformer (%s; cache retry: %s) — dense retrieval will degrade", + first_error, + exc, + ) + return None diff --git a/docureason/tripath/indexing/sparse_index.py b/docureason/tripath/indexing/sparse_index.py index a6a463e..2b5da0e 100644 --- a/docureason/tripath/indexing/sparse_index.py +++ b/docureason/tripath/indexing/sparse_index.py @@ -30,6 +30,7 @@ import json import logging import re +import threading from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -77,6 +78,9 @@ def __init__( self.bm25_b = idx_cfg.bm25_b if idx_cfg else 0.75 self._indices: Dict[str, Any] = {} # in-memory cache after build self._meta: Dict[str, List[Dict[str, Any]]] = {} + #: Guards the lazy search-time load; the parallel modality dispatch calls + #: search() for text and table concurrently against this one builder. + self._load_lock = threading.Lock() #: Per-modality build observations: {"records", "indexed", "degraded"}. self.build_stats: Dict[str, Dict[str, Any]] = {} @@ -190,9 +194,12 @@ def search( meta_path = index_dir / "meta.json" if not meta_path.exists(): return [] - # Cached: this file was re-read from disk on every single query before. - meta = json.loads(meta_path.read_text(encoding="utf-8")) - self._meta[modality] = meta + with self._load_lock: + meta = self._meta.get(modality) + if meta is None: + # Cached: this file was re-read from disk on every single query before. + meta = json.loads(meta_path.read_text(encoding="utf-8")) + self._meta[modality] = meta if not meta: return [] @@ -203,8 +210,11 @@ def search( import bm25s if retriever is None: - retriever = bm25s.BM25.load(str(index_dir), load_corpus=False) - self._indices[modality] = retriever + with self._load_lock: + retriever = self._indices.get(modality) + if retriever is None: + retriever = bm25s.BM25.load(str(index_dir), load_corpus=False) + self._indices[modality] = retriever query_tokens = bm25s.tokenize([query], stopwords="en") results, scores = retriever.retrieve(query_tokens, k=min(k, len(meta))) diff --git a/docureason/tripath/indexing/table_store.py b/docureason/tripath/indexing/table_store.py new file mode 100644 index 0000000..9385161 --- /dev/null +++ b/docureason/tripath/indexing/table_store.py @@ -0,0 +1,348 @@ +"""table_store.py — DuckDB tables registered once at ingestion, not per query. + +``TableSQLRetriever`` rebuilt every table it touched on **every query**: a +``CREATE TABLE`` plus one parameter-bound ``INSERT`` per row, into a fresh +``:memory:`` database, discarded when the query finished. For a filing with 40 +tables that is 40 schema creations and every row re-inserted per question asked, and +the inferred column types were re-derived each time - so the same table could type a +column as ``VARCHAR`` for one query and ``DECIMAL`` for another depending on which +rows the parser happened to produce. + +Registering at ingestion fixes both: the schema is decided once, recorded in +``tables_index.json`` next to the corpus it describes, and queries run against a +prepared read-only database. + +Layout, inside the run's output directory:: + + tables.duckdb one table per table-region + tables_index.json region_id -> {table_name, columns, column_types, row_count} + +The index is the contract. A region absent from it has no prepared table, and the +retriever falls back to the in-memory path rather than guessing a table name - so a +partially built store degrades instead of raising. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union + +from docureason.tripath.utils import get_logger + +logger = get_logger(__name__) + +DB_FILENAME = "tables.duckdb" +INDEX_FILENAME = "tables_index.json" + +#: DuckDB identifiers are quoted everywhere in this module, but a table name derived +#: from a region id is also used as a dict key and logged, so it is normalised to a +#: conservative character set rather than quoted-and-hoped. +_UNSAFE = re.compile(r"[^0-9a-zA-Z_]+") + + +def table_name_for(region_id: str) -> str: + """Deterministic SQL table name for *region_id*. + + Deterministic matters: the name is stored in ``tables_index.json`` at ingestion + and looked up at query time, in a different process. + """ + slug = _UNSAFE.sub("_", str(region_id)).strip("_").lower() + if not slug: + slug = "unnamed" + if slug[0].isdigit(): + slug = f"t_{slug}" + return slug[:180] + + +@dataclass +class RegisteredTable: + """One table-region's prepared schema.""" + + region_id: str + document_id: str + table_name: str + columns: List[str] + column_types: Dict[str, str] + row_count: int + page_no: Optional[int] = None + unit_multiplier: int = 1 + + def as_dict(self) -> Dict[str, Any]: + return { + "region_id": self.region_id, + "document_id": self.document_id, + "table_name": self.table_name, + "columns": list(self.columns), + "column_types": dict(self.column_types), + "row_count": self.row_count, + "page_no": self.page_no, + "unit_multiplier": self.unit_multiplier, + } + + @classmethod + def from_dict(cls, payload: Dict[str, Any]) -> "RegisteredTable": + return cls( + region_id=str(payload.get("region_id", "")), + document_id=str(payload.get("document_id", "")), + table_name=str(payload.get("table_name", "")), + columns=list(payload.get("columns") or []), + column_types=dict(payload.get("column_types") or {}), + row_count=int(payload.get("row_count") or 0), + page_no=payload.get("page_no"), + unit_multiplier=int(payload.get("unit_multiplier") or 1), + ) + + +@dataclass +class TableStoreStats: + """What a build actually registered, for the ingestion audit.""" + + tables_registered: int = 0 + rows_registered: int = 0 + regions_skipped: int = 0 + degraded: Optional[str] = None + skipped_reasons: Dict[str, int] = field(default_factory=dict) + + def as_dict(self) -> Dict[str, Any]: + return { + "tables_registered": self.tables_registered, + "rows_registered": self.rows_registered, + "regions_skipped": self.regions_skipped, + "skipped_reasons": dict(self.skipped_reasons), + "degraded": self.degraded, + } + + +class DuckDBTableStore: + """Build and read the per-run DuckDB table store.""" + + def __init__(self, output_dir: Union[str, Path]) -> None: + self.output_dir = Path(output_dir) + self.db_path = self.output_dir / DB_FILENAME + self.index_path = self.output_dir / INDEX_FILENAME + self._index: Optional[Dict[str, RegisteredTable]] = None + self._connection: Any = None + + # -- availability ------------------------------------------------------- + + @staticmethod + def duckdb_available() -> bool: + try: + import duckdb # noqa: F401 + + return True + except ImportError: + return False + + @property + def available(self) -> bool: + """True when a prepared store exists on disk and DuckDB can read it.""" + return self.db_path.exists() and self.index_path.exists() and self.duckdb_available() + + # -- build (ingestion time) -------------------------------------------- + + def build(self, documents: Sequence[Any]) -> TableStoreStats: + """Register every table-region in *documents* as a DuckDB table. + + Rebuilds from scratch: a stale table left over from a previous corpus in the + same directory would still be reachable by name and would answer with another + document's numbers, which is worse than having no store at all. + """ + stats = TableStoreStats() + if not self.duckdb_available(): + stats.degraded = "duckdb_unavailable" + logger.warning("DuckDBTableStore: duckdb not installed - tables will be built per query") + return stats + + import duckdb + + from ..retrieval.table_sql import infer_sql_type, quote_identifier, sanitize_cell_for_sql + + self.output_dir.mkdir(parents=True, exist_ok=True) + self.db_path.unlink(missing_ok=True) + index: Dict[str, RegisteredTable] = {} + + try: + con = duckdb.connect(str(self.db_path)) + except Exception as exc: + stats.degraded = "connect_failed" + logger.error("DuckDBTableStore: cannot open %s: %s", self.db_path, exc) + return stats + + try: + for document in documents: + document_id = str(getattr(document, "id", "") or "") + for region in getattr(document, "regions", []) or []: + if getattr(region, "type", "") != "table": + continue + metadata = getattr(region, "metadata", {}) or {} + region_id = str(metadata.get("region_id") or "") + payload = _table_payload(region) + + reason = _rejection_reason(region_id, payload) + if reason: + stats.regions_skipped += 1 + stats.skipped_reasons[reason] = stats.skipped_reasons.get(reason, 0) + 1 + continue + + columns = [str(c) for c in payload["columns"]] + rows = payload["rows"] + name = _unique_name(table_name_for(region_id), index) + + column_types: Dict[str, str] = {} + for position, column in enumerate(columns): + values = [row[position] for row in rows if position < len(row)] + column_types[column] = infer_sql_type(values) + + col_defs = ", ".join(f"{quote_identifier(c)} {column_types[c]}" for c in columns) + con.execute(f"CREATE TABLE {quote_identifier(name)} ({col_defs})") + placeholders = ", ".join(["?"] * len(columns)) + for row in rows: + padded = list(row) + [None] * (len(columns) - len(row)) + con.execute( + f"INSERT INTO {quote_identifier(name)} VALUES ({placeholders})", + [ + sanitize_cell_for_sql(value, column_types[column]) + for column, value in zip(columns, padded[: len(columns)]) + ], + ) + + index[region_id] = RegisteredTable( + region_id=region_id, + document_id=document_id, + table_name=name, + columns=columns, + column_types=column_types, + row_count=len(rows), + page_no=metadata.get("page_no"), + unit_multiplier=int(payload.get("unit_multiplier") or 1), + ) + stats.tables_registered += 1 + stats.rows_registered += len(rows) + except Exception as exc: + stats.degraded = "build_error" + logger.error("DuckDBTableStore: build failed: %s", exc) + finally: + con.close() + + self.index_path.write_text( + json.dumps({rid: entry.as_dict() for rid, entry in index.items()}, indent=2), + encoding="utf-8", + ) + self._index = index + logger.info( + "DuckDBTableStore: registered %d table(s) / %d row(s) in %s (skipped %d)", + stats.tables_registered, + stats.rows_registered, + self.db_path.name, + stats.regions_skipped, + ) + return stats + + # -- read (query time) -------------------------------------------------- + + def index(self) -> Dict[str, RegisteredTable]: + """The region_id to prepared-table map, read from disk once.""" + if self._index is None: + if not self.index_path.exists(): + self._index = {} + else: + try: + raw = json.loads(self.index_path.read_text(encoding="utf-8")) + self._index = {rid: RegisteredTable.from_dict(entry) for rid, entry in raw.items()} + except Exception as exc: + logger.warning("DuckDBTableStore: unreadable index %s: %s", self.index_path, exc) + self._index = {} + return self._index + + def lookup(self, region_id: str) -> Optional[RegisteredTable]: + return self.index().get(str(region_id)) + + def connection(self) -> Any: + """A cached read-only connection, or None when there is nothing to read. + + Read-only is deliberate: a query path able to write to the store could mutate + the corpus it is supposed to be reporting on. + """ + if self._connection is not None: + return self._connection + if not self.available: + return None + try: + import duckdb + + self._connection = duckdb.connect(str(self.db_path), read_only=True) + except Exception as exc: + logger.warning("DuckDBTableStore: cannot open %s read-only: %s", self.db_path, exc) + self._connection = None + return self._connection + + def close(self) -> None: + if self._connection is not None: + try: + self._connection.close() + finally: + self._connection = None + + +def _table_payload(region: Any) -> Optional[Dict[str, Any]]: + """The columns/rows payload for *region*, from the object or from its metadata.""" + raw = getattr(region, "table_json", None) + if isinstance(raw, str): + try: + raw = json.loads(raw) + except Exception: + raw = None + if not isinstance(raw, dict): + metadata = getattr(region, "metadata", {}) or {} + candidate = metadata.get("table_json") + if isinstance(candidate, str): + try: + candidate = json.loads(candidate) + except Exception: + candidate = None + raw = candidate if isinstance(candidate, dict) else None + return raw + + +def _rejection_reason(region_id: str, payload: Optional[Dict[str, Any]]) -> Optional[str]: + """Why this region cannot be registered, or None when it can.""" + if not region_id: + return "no_region_id" + if not payload: + return "no_table_json" + columns = payload.get("columns") or [] + rows = payload.get("rows") or [] + if not columns: + return "no_columns" + if not rows: + return "no_rows" + if len({str(c) for c in columns}) != len(columns): + # Duplicate headers cannot become distinct SQL columns. The in-memory path has + # the same problem, but it fails per query rather than poisoning the store. + return "duplicate_columns" + return None + + +def _unique_name(base: str, index: Dict[str, RegisteredTable]) -> str: + """*base*, suffixed when another region already claimed that name.""" + taken = {entry.table_name for entry in index.values()} + if base not in taken: + return base + suffix = 2 + while f"{base}_{suffix}" in taken: + suffix += 1 + return f"{base}_{suffix}" + + +__all__ = [ + "DB_FILENAME", + "INDEX_FILENAME", + "DuckDBTableStore", + "RegisteredTable", + "TableStoreStats", + "table_name_for", +] diff --git a/docureason/tripath/ingestion/figure_captioner.py b/docureason/tripath/ingestion/figure_captioner.py index cf04f7e..e27b3f2 100644 --- a/docureason/tripath/ingestion/figure_captioner.py +++ b/docureason/tripath/ingestion/figure_captioner.py @@ -32,6 +32,7 @@ from pathlib import Path from typing import Any, List, Optional, Tuple +from docureason.tripath.ingestion.rasterizer import PageRasterizer, page_index_from_region from docureason.tripath.model_pins import resolve_revision from docureason.tripath.utils import get_logger, log_pipeline_flag, trace_execution @@ -71,9 +72,14 @@ def __init__( self, use_blip2: bool = True, use_clip: bool = True, + image_rendering_dpi: int = 150, ) -> None: self.use_blip2 = use_blip2 self.use_clip = use_clip + #: Rasterization DPI for figure crops. This is the first consumer of + #: ``IngestionConfig.image_rendering_dpi``, which the startup banner had been + #: reporting for a setting that reached no code (§6, P1-12). + self.image_rendering_dpi = int(image_rendering_dpi) #: Captions produced per tier during this instance's lifetime. Recorded so a #: run where BLIP-2 and CLIP both failed and every caption came from the #: deterministic metadata fallback is distinguishable from a real vision run @@ -81,6 +87,12 @@ def __init__( self.tier_counts: dict = {"existing_text": 0, "blip2": 0, "clip": 0, "metadata": 0} log_pipeline_flag("figure_captioner_blip2", use_blip2, "BLIP-2 figure captioner flag", logger) log_pipeline_flag("figure_captioner_clip", use_clip, "CLIP chart classifier flag", logger) + log_pipeline_flag( + "image_rendering_dpi", + self.image_rendering_dpi, + "Page rasterization DPI for figure crops and OCR", + logger, + ) self._blip2_model: Any = None self._blip2_proc: Any = None @@ -117,6 +129,23 @@ def caption_figures( if not figure_indices: return regions + # One rasterizer per document: it caches rendered pages, so a page carrying six + # figures is rendered once rather than six times. + rasterizer = PageRasterizer(path, dpi=self.image_rendering_dpi) if path.suffix.lower() == ".pdf" else None + try: + return self._caption_each(regions, figure_indices, path, doc_title, rasterizer) + finally: + if rasterizer is not None: + rasterizer.close() + + def _caption_each( + self, + regions: List[Region], + figure_indices: List[int], + path: Path, + doc_title: str, + rasterizer: Optional[PageRasterizer], + ) -> List[Region]: for idx in figure_indices: region = regions[idx] existing_text = (region.text or "").strip() @@ -125,7 +154,7 @@ def caption_figures( caption = existing_text source = "existing_text" else: - generated, source = self._caption_region(region, path, doc_title) + generated, source = self._caption_region(region, path, doc_title, rasterizer) caption = f"{existing_text} - {generated}".strip(" -") if existing_text else generated self.tier_counts[source] = self.tier_counts.get(source, 0) + 1 @@ -149,9 +178,17 @@ def caption_figures( # Caption generation tiers # ------------------------------------------------------------------ - def _caption_region(self, region: Region, source_path: Path, doc_title: str) -> Tuple[str, str]: + def _caption_region( + self, + region: Region, + source_path: Path, + doc_title: str, + rasterizer: Optional[PageRasterizer] = None, + ) -> Tuple[str, str]: """Try each tier in order; return ``(caption, tier_name)`` for the first that works.""" - image_bytes = self._extract_region_image(region, source_path) + image_bytes = self._region_crop_bytes(region, rasterizer) + if image_bytes is None: + image_bytes = self._extract_region_image(region, source_path) if image_bytes: if self.use_blip2: @@ -178,6 +215,33 @@ def _caption_region(self, region: Region, source_path: Path, doc_title: str) -> # Image extraction # ------------------------------------------------------------------ + def _region_crop_bytes(self, region: Region, rasterizer: Optional[PageRasterizer]) -> Optional[bytes]: + """A real raster crop of *this* figure, or None when there is nothing to caption. + + Two behaviours the previous ``page.images[0]`` path could not have: + + * every figure gets **its own** crop, taken at its own bbox (P1-3); + * a crop with no visual variation is refused, so an empty rectangle is no + longer captioned by BLIP-2 and indexed as a figure (P1-4). Refusing routes + the region to the deterministic metadata caption, which is counted in + ``tier_counts`` and reported as ingestion degradation. + """ + if rasterizer is None or not rasterizer.available: + return None + + page_index = page_index_from_region(region.page_no, rasterizer.page_count) + crop = rasterizer.crop(page_index, region.bbox) + if crop is None: + return None + if PageRasterizer.is_blank(crop): + self.tier_counts["blank_crop_skipped"] = self.tier_counts.get("blank_crop_skipped", 0) + 1 + logger.warning( + "FigureCaptioner: the crop for a figure on page %s is blank — refusing to caption an empty region", + region.page_no, + ) + return None + return PageRasterizer.to_png_bytes(crop) + def _extract_region_image(self, region: Region, source_path: Path) -> Optional[bytes]: """Extract image bytes for the region's page / bbox. diff --git a/docureason/tripath/ingestion/ocr_fallback.py b/docureason/tripath/ingestion/ocr_fallback.py index f84f9aa..31eaaea 100644 --- a/docureason/tripath/ingestion/ocr_fallback.py +++ b/docureason/tripath/ingestion/ocr_fallback.py @@ -32,12 +32,12 @@ from __future__ import annotations -import io from pathlib import Path from typing import Any, Dict, List, Optional from docureason.tripath.utils import get_logger, log_pipeline_flag, trace_execution +from .rasterizer import PageRasterizer, page_index_from_region from .schema import Region logger = get_logger(__name__) @@ -68,10 +68,14 @@ def __init__( languages: List[str] | None = None, gpu: bool | None = None, min_chars_threshold: int = _MIN_CHARS_THRESHOLD, + image_rendering_dpi: int = 150, ) -> None: self.languages = languages or ["en"] self.gpu = gpu self.min_chars_threshold = min_chars_threshold + #: DPI for the page rasters handed to EasyOCR. OCR accuracy is sensitive to + #: resolution, and this setting previously reached nothing at all. + self.image_rendering_dpi = int(image_rendering_dpi) log_pipeline_flag( "ocr_min_chars_threshold", min_chars_threshold, "Character threshold to detect scanned PDF pages", logger ) @@ -119,18 +123,21 @@ def run(self, pdf_path: Path, existing_regions: List[Region]) -> List[Region]: return [] ocr_regions: List[Region] = [] - for page_no, page_image_bytes in scanned_pages.items(): + for page_index, page_image_bytes in scanned_pages.items(): text = self._ocr_image(reader, page_image_bytes) if text.strip(): ocr_regions.append( Region( type="body", text=text.strip(), - page_no=page_no, + # Emitted 1-based, matching Docling's convention: these regions + # are appended to Docling's, and two page-numbering conventions + # in one document is how the off-by-one above came about. + page_no=page_index + 1, layout_source="ocr", ) ) - logger.debug("OCRFallback: page %d → %d chars", page_no, len(text)) + logger.debug("OCRFallback: page index %d -> %d chars", page_index, len(text)) return ocr_regions @@ -139,14 +146,20 @@ def run(self, pdf_path: Path, existing_regions: List[Region]) -> List[Region]: # ------------------------------------------------------------------ @staticmethod - def _count_chars_per_page(regions: List[Region]) -> Dict[int, int]: - """Sum character counts per page_no from existing regions.""" + def _count_chars_per_page(regions: List[Region], page_count: int = 0) -> Dict[int, int]: + """Characters per **0-based page index**, from the regions already extracted. + + Docling reports ``page_no`` 1-based while the page loop below indexes from 0 + (audit P1-2). Counting under one convention and looking up under the other + meant page N's character count was compared against page N+1's content, so the + scanned-page detector was reliably off by one page. + """ counts: Dict[int, int] = {} for region in regions: - p = region.page_no - if p < 0: + if region.page_no is None or region.page_no < 0: continue - counts[p] = counts.get(p, 0) + len(region.text) + index = page_index_from_region(region.page_no, page_count or region.page_no + 1) + counts[index] = counts.get(index, 0) + len(region.text) return counts def _detect_scanned_pages( @@ -154,51 +167,39 @@ def _detect_scanned_pages( pdf_path: Path, page_char_counts: Dict[int, int], ) -> Dict[int, bytes]: - """Return a mapping of page_no → raw image bytes for scanned pages.""" + """Map 0-based page index -> a PNG raster of the whole page, for scanned pages. + + The page is **rendered**, not scavenged. Taking ``page.images[0]`` returned one + embedded image per page, so a scan split across two images OCR'd only the first + half (P2-13) - and when a page had no embedded image at all, the previous code + handed EasyOCR a blank white canvas, which costs seconds per page to read + nothing. A rendered page contains whatever is actually on it. + """ scanned: Dict[int, bytes] = {} - try: - import pypdf + rasterizer = PageRasterizer(pdf_path, dpi=self.image_rendering_dpi) + if not rasterizer.available: + logger.warning("OCRFallback: cannot rasterize %s — skipping OCR entirely", pdf_path.name) + return scanned - reader = pypdf.PdfReader(str(pdf_path)) - for page_no, page in enumerate(reader.pages): - if page_char_counts.get(page_no, 0) >= self.min_chars_threshold: + try: + for page_index in range(rasterizer.page_count): + if page_char_counts.get(page_index, 0) >= self.min_chars_threshold: continue # Docling already extracted enough text here - # Try to grab the first embedded raster image from the page. - image_bytes = self._extract_page_image(page, page_no) - if image_bytes: - scanned[page_no] = image_bytes + image = rasterizer.render(page_index) + if image is None: + continue + if PageRasterizer.is_blank(image): + logger.debug("OCRFallback: page %d renders blank — nothing to OCR", page_index) + continue + payload = PageRasterizer.to_png_bytes(image) + if payload: + scanned[page_index] = payload except Exception as exc: logger.warning("Could not inspect pages of %s: %s", pdf_path.name, exc) + finally: + rasterizer.close() return scanned - @staticmethod - def _extract_page_image(page: Any, page_no: int) -> Optional[bytes]: - """Extract the first raster image embedded in a pypdf page object. - - Falls back to a blank-white PIL image rendered at a fixed DPI when no - embedded images are found (so EasyOCR at least gets something). - """ - try: - images = list(page.images) - if images: - return images[0].data # raw bytes (JPEG / PNG etc.) - except Exception: - pass - - # Fallback: render page as a plain white PIL image. - try: - from PIL import Image - - width = int(float(page.mediabox.width)) - height = int(float(page.mediabox.height)) - img = Image.new("RGB", (max(width, 100), max(height, 100)), color=(255, 255, 255)) - buf = io.BytesIO() - img.save(buf, format="PNG") - return buf.getvalue() - except Exception as exc: - logger.debug("Could not render fallback image for page %d: %s", page_no, exc) - return None - def _get_reader(self) -> Optional[Any]: """Return a cached EasyOCR Reader, initialising it on first use.""" if OCRFallback._reader_cache is not None: diff --git a/docureason/tripath/ingestion/rasterizer.py b/docureason/tripath/ingestion/rasterizer.py new file mode 100644 index 0000000..cc852a3 --- /dev/null +++ b/docureason/tripath/ingestion/rasterizer.py @@ -0,0 +1,230 @@ +"""rasterizer.py — render PDF pages, so figure and OCR images are real. + +Three defects shared one root cause: nothing in the pipeline could rasterize a page. + +* **P1-3** - the figure captioner returned ``page.images[0]`` for *every* figure on a + page, so three charts on one page were captioned from whichever image happened to be + first, and two of the three captions described the wrong chart. +* **P1-4** - when there was no embedded image, it built a **blank white canvas**, + cropped it to the bbox and captioned that. BLIP-2 dutifully described an empty + rectangle and the result was indexed as a figure. +* **P2-13** - the OCR fallback also took only the first embedded image per page, so a + scanned page whose text sat in the second image OCR'd to nothing. + +``pypdfium2`` (already installed - Docling depends on it) renders a page to a bitmap, +which is what all three needed. ``IngestionConfig.image_rendering_dpi`` finally has a +consumer: it was a documented, banner-printed setting with no effect anywhere (§6). + +Coordinates +----------- +Docling reports a bbox as ``(left, top, right, bottom)`` in PDF points, in a +**bottom-left** origin, so ``top > bottom`` numerically. PIL crops in pixels from a +**top-left** origin. :meth:`PageRasterizer.crop` does that conversion in one place; +it was previously open-coded in the captioner with the coordinates named ``(x0, y0, +x1, y1)``, which is how the flip came to be applied to the wrong pair. + +Page indexing +------------- +Docling provenance ``page_no`` is **1-based**; ``pypdfium2`` and ``pypdf`` index pages +from 0 (audit P1-2). :func:`page_index_from_region` performs that single conversion, +clamped to the document, rather than each caller guessing. +""" + +from __future__ import annotations + +import io +from typing import Any, Dict, Optional, Tuple + +from docureason.tripath.utils import get_logger + +logger = get_logger(__name__) + +#: PDF user space is defined at 72 points per inch. +POINTS_PER_INCH = 72.0 + +#: A bbox narrower or shorter than this (in PDF points, before padding) cannot contain +#: a figure. Checked unpadded: with padding included, a 0.2pt-wide box would clear a +#: pixel-based threshold and yield an 8px sliver that is worse than the whole page. +MIN_CROP_POINTS = 8.0 + +#: A crop whose pixels vary less than this is treated as blank. Chosen well below the +#: variation of any real chart (axes and text alone exceed it) and above JPEG noise on +#: a genuinely empty region. +BLANK_STD_THRESHOLD = 3.0 + + +def page_index_from_region(page_no: Optional[int], page_count: int) -> int: + """Convert a 1-based Docling ``page_no`` to a 0-based page index, clamped. + + ``page_no`` of -1 (Docling gave no provenance) means "unknown", which resolves to + the first page - the same thing the callers did before, but stated once. + """ + if page_count <= 0: + return 0 + if page_no is None or page_no < 0: + return 0 + index = page_no - 1 if page_no >= 1 else 0 + return max(0, min(index, page_count - 1)) + + +class PageRasterizer: + """Render and crop pages of one PDF at a fixed DPI, caching each page. + + Caching matters: a filing page carrying six figures would otherwise be rasterized + six times, and rasterization is the expensive part of captioning. + """ + + def __init__(self, pdf_path: Any, dpi: int = 150) -> None: + self.pdf_path = pdf_path + self.dpi = max(36, int(dpi)) + self.scale = self.dpi / POINTS_PER_INCH + self._document: Any = None + self._pages: Dict[int, Any] = {} + self._page_sizes: Dict[int, Tuple[float, float]] = {} + self._unavailable = False + + # -- lifecycle ---------------------------------------------------------- + + def _open(self) -> Any: + if self._document is not None or self._unavailable: + return self._document + try: + import pypdfium2 + + self._document = pypdfium2.PdfDocument(str(self.pdf_path)) + except Exception as exc: + logger.warning("PageRasterizer: cannot open %s (%s) — rasterization unavailable", self.pdf_path, exc) + self._unavailable = True + self._document = None + return self._document + + @property + def available(self) -> bool: + return self._open() is not None + + @property + def page_count(self) -> int: + document = self._open() + try: + return len(document) if document is not None else 0 + except Exception: + return 0 + + def close(self) -> None: + self._pages.clear() + if self._document is not None: + try: + self._document.close() + except Exception: + pass + self._document = None + + # -- rendering ---------------------------------------------------------- + + def render(self, page_index: int) -> Optional[Any]: + """The rendered page as a PIL image, or None when rasterization is unavailable.""" + if page_index in self._pages: + return self._pages[page_index] + + document = self._open() + if document is None: + return None + try: + page = document[page_index] + width_pt, height_pt = page.get_size() + self._page_sizes[page_index] = (float(width_pt), float(height_pt)) + image = page.render(scale=self.scale).to_pil().convert("RGB") + except Exception as exc: + logger.warning("PageRasterizer: page %d of %s failed to render: %s", page_index, self.pdf_path, exc) + return None + + self._pages[page_index] = image + return image + + def page_size(self, page_index: int) -> Optional[Tuple[float, float]]: + """Page size in PDF points, populated as a side effect of rendering.""" + if page_index not in self._page_sizes: + self.render(page_index) + return self._page_sizes.get(page_index) + + def crop( + self, + page_index: int, + bbox: Optional[Tuple[float, float, float, float]], + pad_points: float = 2.0, + ) -> Optional[Any]: + """Crop the rendered page to *bbox*, given as Docling ``(l, t, r, b)`` points. + + A missing or degenerate bbox yields the whole page rather than nothing: a + full-page image is a worse caption input than a tight crop, but it is a real + image of real content, which a blank canvas was not. + """ + image = self.render(page_index) + if image is None: + return None + size = self._page_sizes.get(page_index) + if not bbox or size is None: + return image + + page_height = size[1] + left, top, right, bottom = (float(v) for v in bbox) + # Bottom-left origin: `top` is numerically the larger y. Normalise first so a + # producer that emits them the other way round still crops the same box. + y_high, y_low = max(top, bottom), min(top, bottom) + x_left, x_right = min(left, right), max(left, right) + + if (x_right - x_left) < MIN_CROP_POINTS or (y_high - y_low) < MIN_CROP_POINTS: + logger.debug("PageRasterizer: bbox %s on page %d is degenerate - using the full page", bbox, page_index) + return image + + px0 = int(max(0.0, (x_left - pad_points)) * self.scale) + px1 = int(min(size[0], (x_right + pad_points)) * self.scale) + py0 = int(max(0.0, (page_height - y_high - pad_points)) * self.scale) + py1 = int(min(page_height, (page_height - y_low + pad_points)) * self.scale) + + if px1 - px0 < 4 or py1 - py0 < 4: + return image + return image.crop((px0, py0, px1, py1)) + + # -- inspection --------------------------------------------------------- + + @staticmethod + def is_blank(image: Any, std_threshold: float = BLANK_STD_THRESHOLD) -> bool: + """True when *image* carries no visual content worth captioning. + + Uses the standard deviation of the greyscale histogram: a blank canvas has + zero, and anything with axes, gridlines or text is far above the threshold. + This is the check whose absence let empty rectangles be captioned and indexed + as figures (P1-4). + """ + if image is None: + return True + try: + from PIL import ImageStat + + stat = ImageStat.Stat(image.convert("L")) + return float(stat.stddev[0]) < std_threshold + except Exception: + # Cannot tell: treat as non-blank rather than silently discarding a figure. + return False + + @staticmethod + def to_png_bytes(image: Any) -> Optional[bytes]: + if image is None: + return None + try: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + except Exception as exc: + logger.debug("PageRasterizer: PNG encode failed: %s", exc) + return None + + +__all__ = [ + "BLANK_STD_THRESHOLD", + "MIN_CROP_POINTS", + "POINTS_PER_INCH", + "PageRasterizer", + "page_index_from_region", +] diff --git a/docureason/tripath/retrieval/hybrid_retriever.py b/docureason/tripath/retrieval/hybrid_retriever.py index dd5baf3..96691c5 100644 --- a/docureason/tripath/retrieval/hybrid_retriever.py +++ b/docureason/tripath/retrieval/hybrid_retriever.py @@ -17,14 +17,17 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable, Dict, List, Optional, Tuple from docureason.tripath.config import DocuReasonConfig -from docureason.tripath.utils import get_logger, trace_execution +from docureason.tripath.utils import StageTimer, get_logger, trace_execution from ..fusion.fuse import Fuser from ..indexing.dense_index import DenseIndexBuilder from ..indexing.sparse_index import BM25SIndexBuilder +from ..indexing.table_store import DuckDBTableStore from ..ingestion.schema import Document from ..router.configurable_router import ConfigurableRouter from .chart_understanding import ChartUnderstandingModule @@ -63,7 +66,11 @@ def __init__( # The pipeline builds BM25S for text-bearing modalities only, so the vision # path has a dense run and no sparse run. self.vision_retrieval = VisionRetrieval(dense=self.dense, sparse=None, config=self.config) - self.table_sql_retrieval = TableSQLRetriever() + # The prepared table store lives beside the corpus these indexes describe. + # When it is absent (an older run, or duckdb missing) the retriever falls back + # to building each table in memory per query. + self.table_store = DuckDBTableStore(self.config.output_dir) + self.table_sql_retrieval = TableSQLRetriever(config=self.config, store=self.table_store) self.chart_module = ChartUnderstandingModule() self.fuser = Fuser(rrf_k=self.config.retrieval.rrf_k, per_run_cap=self.candidate_caps()) self.ranker = Ranker(config=self.config) @@ -74,6 +81,10 @@ def __init__( #: SQL candidates from the most recent query, so the serving layer can build #: its SQL payload without running the DuckDB path a second time. self.last_sql_results: List[Dict[str, Any]] = [] + #: Per-stage timings from the most recent query, for the response's + #: ``latency_breakdown``. Owned here rather than in the serving layer because + #: this is where the stages are. + self.last_timer: StageTimer = StageTimer() def candidate_caps(self) -> Dict[str, int]: """Per-modality candidate caps read from ``RetrievalConfig``.""" @@ -86,45 +97,63 @@ def candidate_caps(self) -> Dict[str, int]: } @trace_execution(logger=logger) - def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any]]: - route_flags = self.router.route(query) - router_weights = self.router.get_route_weights(query) + def retrieve( + self, + query: str, + documents: List[Document], + timer: Optional[StageTimer] = None, + ) -> List[Dict[str, Any]]: + timer = timer or StageTimer() + self.last_timer = timer + + with timer.stage("route"): + route_flags = self.router.route(query) + router_weights = self.router.get_route_weights(query) logger.info("HybridRetriever router flags for query '%s': %s (weights: %s)", query, route_flags, router_weights) - batches: List[List[Dict[str, Any]]] = [] modality_modes: Dict[str, Any] = {} self.last_sql_results = [] - # 1. Modality retrieval, each capped at top_k_{modality}. - if route_flags.get("text", True): - batches.append(self.text_retrieval.retrieve(query, documents)) - modality_modes["text"] = self.text_retrieval.last_stats - - if route_flags.get("table", True): - batches.append(self.table_retrieval.retrieve(query, documents)) - modality_modes["table"] = self.table_retrieval.last_stats - self.last_sql_results = self.table_sql_retrieval.retrieve(query, documents) - batches.append(self.last_sql_results) + # 1. Modality retrieval, each capped at top_k_{modality}. The paths are + # independent, so they run concurrently when enabled; results are collected + # in the fixed order the tasks were declared, which keeps fusion input - + # and therefore ranking - identical either way. + tasks = self._build_tasks(route_flags, query, documents) + outcomes = self._dispatch(tasks) - if route_flags.get("vision", True): - batches.append(self.vision_retrieval.retrieve(query, documents)) - modality_modes["vision"] = self.vision_retrieval.last_stats + batches: List[List[Dict[str, Any]]] = [] + for name, batch, stats, elapsed_ms in outcomes: + batches.append(batch) + if name == "table_sql": + self.last_sql_results = batch + # The SQL path reports no sub-steps of its own, so its wall time is + # the stage. The modality paths are accounted by the dense/sparse/merge + # timings they report; whatever they spend outside those shows up in + # the timer's residual rather than being invented here. + timer.add("table_sql", elapsed_ms) + else: + modality_modes[name] = stats + self._fold_path_timings(timer, stats) pre_fusion = sum(len(batch) for batch in batches) # 2. Reciprocal Rank Fusion, re-applying the caps. - fused = self.fuser.fuse(batches, router_weights=router_weights) + with timer.stage("fuse"): + fused = self.fuser.fuse(batches, router_weights=router_weights) # 3. Chart enrichment, in place — this used to append a second copy of every # visual candidate, which is why the duplicate-rank ratio was so high. - candidates = self.chart_module.understand(query, fused) + with timer.stage("chart_enrich"): + candidates = self.chart_module.understand(query, fused) # 4. Parent-child expansion (config-gated; the flag was never checked before). if self.config.reranker.enable_parent_expansion: - candidates = self._expand_parents(candidates) + with timer.stage("parent_expand"): + candidates = self._expand_parents(candidates) # 5. Cross-encoder reranking down to final_top_k. - reranked = self.ranker.rank(query, candidates) + with timer.stage("rerank"): + reranked = self.ranker.rank(query, candidates) top_k = self.config.reranker.final_top_k final = reranked[:top_k] if top_k > 0 else reranked @@ -136,6 +165,8 @@ def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any] "returned": len(final), "charts_enriched": self.chart_module.charts_enriched, "sql_candidates": len(self.last_sql_results), + "parallel_paths": self._parallel_enabled() and len(tasks) > 1, + "paths_run": [name for name, *_ in tasks], } self.last_degradation = self._collect_degradation(modality_modes, final) if self.last_degradation: @@ -146,6 +177,95 @@ def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any] # Internals # ------------------------------------------------------------------ + def _parallel_enabled(self) -> bool: + serving = getattr(self.config, "serving", None) + return bool(getattr(serving, "enable_parallel_paths", True)) + + def _build_tasks( + self, + route_flags: Dict[str, bool], + query: str, + documents: List[Document], + ) -> List[Tuple[str, Callable[[], List[Dict[str, Any]]], Any]]: + """The retrieval calls this query needs, in a fixed, deterministic order. + + Each entry is ``(name, thunk, stats_source)``. ``stats_source`` is the + retriever whose ``last_stats`` describes the call, read only after the thunk + has returned. + """ + tasks: List[Tuple[str, Callable[[], List[Dict[str, Any]]], Any]] = [] + + if route_flags.get("text", True): + tasks.append(("text", lambda: self.text_retrieval.retrieve(query, documents), self.text_retrieval)) + + if route_flags.get("table", True): + tasks.append(("table", lambda: self.table_retrieval.retrieve(query, documents), self.table_retrieval)) + # Ablation A2 ("remove the DuckDB text-to-SQL retriever") switches the + # whole SQL path off rather than letting it run and discarding the rows, + # so the measured delta includes its latency as well as its accuracy. + if self.config.retrieval.enable_table_sql: + tasks.append(("table_sql", lambda: self.table_sql_retrieval.retrieve(query, documents), None)) + + if route_flags.get("vision", True): + tasks.append(("vision", lambda: self.vision_retrieval.retrieve(query, documents), self.vision_retrieval)) + + return tasks + + def _dispatch( + self, + tasks: List[Tuple[str, Callable[[], List[Dict[str, Any]]], Any]], + ) -> List[Tuple[str, List[Dict[str, Any]], Any, float]]: + """Run *tasks*, concurrently when enabled, returning results in task order. + + A failing path must not take the query down with it: the remaining paths still + answer, and the failure is logged and surfaces as a missing modality in + ``modality_modes`` rather than a 500. Sequential mode raises nothing extra + either, so the two dispatch modes fail the same way. + """ + if not tasks: + return [] + + def run(index: int) -> Tuple[str, List[Dict[str, Any]], Any, float]: + name, thunk, source = tasks[index] + started = time.perf_counter() + try: + batch = thunk() or [] + except Exception as exc: # one dead path, not a dead query + elapsed = (time.perf_counter() - started) * 1000.0 + logger.error("Retrieval path %r failed: %s", name, exc) + return name, [], {"mode": "failed", "error": str(exc)}, elapsed + elapsed = (time.perf_counter() - started) * 1000.0 + stats = getattr(source, "last_stats", None) if source is not None else None + return name, batch, stats, elapsed + + if not self._parallel_enabled() or len(tasks) == 1: + return [run(i) for i in range(len(tasks))] + + # max_workers is bounded by the task count: the paths are I/O- and + # torch-bound (both release the GIL), so more threads than paths buys nothing. + with ThreadPoolExecutor(max_workers=len(tasks), thread_name_prefix="docureason-path") as pool: + return list(pool.map(run, range(len(tasks)))) + + @staticmethod + def _fold_path_timings(timer: StageTimer, stats: Any) -> None: + """Add a retriever's self-reported sub-step timings to the query timer. + + Under parallel dispatch these sum to more than the wall time they occupied, + which is why :meth:`StageTimer.as_dict` reports a signed residual instead of + pretending the stages partition the request. + """ + if not isinstance(stats, dict): + return + for stat_key, stage in ( + ("dense_ms", "dense_search"), + ("sparse_ms", "sparse_search"), + ("index_merge_ms", "index_merge"), + ("corpus_scan_ms", "corpus_scan"), + ): + value = stats.get(stat_key) + if value: + timer.add(stage, float(value)) + @staticmethod def _expand_parents(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Swap in the parent text when it is strictly longer than the child text.""" @@ -183,6 +303,19 @@ def _collect_degradation( for mode in sorted(m for m in modes if m and m != "cross_encoder"): labels.append(f"reranker:{mode}") + # Ablations are deliberate, but they still change what the response means, so + # they are reported through the same channel as an accidental degradation. + if not self.config.retrieval.enable_sparse_retrieval: + labels.append("ablation:sparse_retrieval_disabled") + if not self.config.retrieval.enable_table_sql: + labels.append("ablation:table_sql_disabled") + if not self.config.reranker.enable_parent_expansion: + labels.append("ablation:parent_expansion_disabled") + if not self.config.reranker.enable_heading_penalty: + labels.append("ablation:heading_penalty_disabled") + if not self.config.retrieval.enable_index_retrieval: + labels.append("ablation:index_retrieval_disabled") + sql_modes = {item.get("mode") for item in self.last_sql_results} if "sql_simulation" in sql_modes: labels.append("table_sql:python_runner") diff --git a/docureason/tripath/retrieval/index_backed.py b/docureason/tripath/retrieval/index_backed.py index 4eed45b..919b8fa 100644 --- a/docureason/tripath/retrieval/index_backed.py +++ b/docureason/tripath/retrieval/index_backed.py @@ -25,6 +25,7 @@ from __future__ import annotations +import time from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple @@ -230,6 +231,13 @@ def index_enabled(self) -> bool: retrieval = getattr(self.config, "retrieval", None) return bool(getattr(retrieval, "enable_index_retrieval", True)) + def sparse_enabled(self) -> bool: + """False when this modality has no sparse index, or ablation A6 turned it off.""" + if self.sparse is None: + return False + retrieval = getattr(self.config, "retrieval", None) + return bool(getattr(retrieval, "enable_sparse_retrieval", True)) + def rrf_k(self) -> int: retrieval = getattr(self.config, "retrieval", None) return int(getattr(retrieval, "rrf_k", 60) or 60) @@ -254,7 +262,9 @@ def retrieve( self.last_stats = {"mode": "hybrid_index", **index_stats} return results + scan_start = time.perf_counter() results = self.scan(query, documents) + index_stats["corpus_scan_ms"] = round((time.perf_counter() - scan_start) * 1000.0, 3) # The index attempt's statistics are carried through: a stale-index signal # must not be lost just because the fallback produced the answer. self.last_stats = { @@ -277,15 +287,31 @@ def _retrieve_from_index( k = fetch_k(cap) runs: Dict[str, List[Dict[str, Any]]] = {} + # Sub-step timings are recorded on the stats dict rather than through a timer + # argument, so the retriever keeps a signature that tests and scripts can call + # without knowing about instrumentation. HybridRetriever folds them into the + # per-query StageTimer. + dense_ms = sparse_ms = 0.0 if self.dense is not None: + started = time.perf_counter() runs["dense"] = self.dense.search(query, self.modality, k=k) or [] - if self.sparse is not None: + dense_ms = (time.perf_counter() - started) * 1000.0 + if self.sparse_enabled(): + started = time.perf_counter() runs["sparse"] = self.sparse.search(query, self.modality, k=k) or [] + sparse_ms = (time.perf_counter() - started) * 1000.0 + merge_start = time.perf_counter() merged, merge_stats = merge_index_hits(runs, lookups, rrf_k=self.rrf_k()) + merge_ms = (time.perf_counter() - merge_start) * 1000.0 + stats = { "dense_hits": len(runs.get("dense", [])), "sparse_hits": len(runs.get("sparse", [])), + "sparse_enabled": self.sparse_enabled(), + "dense_ms": round(dense_ms, 3), + "sparse_ms": round(sparse_ms, 3), + "index_merge_ms": round(merge_ms, 3), **merge_stats, } if not merged: diff --git a/docureason/tripath/retrieval/ranker.py b/docureason/tripath/retrieval/ranker.py index 49dcf6f..c7eebd7 100644 --- a/docureason/tripath/retrieval/ranker.py +++ b/docureason/tripath/retrieval/ranker.py @@ -33,14 +33,25 @@ def __init__( self.heading_penalty_multiplier = r_cfg.heading_penalty_multiplier self.heading_penalty_offset = getattr(r_cfg, "heading_penalty_offset", 2.0) self.model_revision = resolve_revision(self.model_name, getattr(r_cfg, "model_revision", None)) + self.enable_cross_encoder = bool(getattr(r_cfg, "enable_cross_encoder", True)) + self.enable_heading_penalty = bool(getattr(r_cfg, "enable_heading_penalty", True)) else: self.model_name = model_name or _CROSS_ENCODER_MODEL self.short_heading_threshold = 45 self.heading_penalty_multiplier = 0.5 self.heading_penalty_offset = 2.0 self.model_revision = resolve_revision(self.model_name) + self.enable_cross_encoder = True + self.enable_heading_penalty = True def _get_cross_encoder(self) -> Optional[Any]: + # Ablation A3 ("remove cross-encoder reranker") routes through the same + # None-encoder path as a missing model, so the measured delta is the delta + # between the model and the Jaccard fallback the system already degrades to. + if not self.enable_cross_encoder: + logger.info("Cross-encoder disabled by config — using the token-similarity fallback reranker") + return None + # The cache key includes the revision so pinning a different revision does # not silently reuse weights loaded for another one. cache_key = f"{self.model_name}@{self.model_revision or 'hub-default'}" @@ -56,7 +67,28 @@ def _get_cross_encoder(self) -> Optional[Any]: logger.info("Loaded and cached process-wide CrossEncoder model: %s", cache_key) return encoder except Exception as exc: - logger.warning("CrossEncoder model unavailable (%s) — using token similarity reranker fallback", exc) + # A pinned revision is resolved against the hub; when the hub is down but + # the commit is cached locally, degrading to the Jaccard scorer would be a + # silent quality drop for no reason. Retry against the cache first. + return self._load_cross_encoder_from_cache(cache_key, exc) + + def _load_cross_encoder_from_cache(self, cache_key: str, first_error: Exception) -> Optional[Any]: + try: + from sentence_transformers import CrossEncoder + + kwargs: Dict[str, Any] = {"local_files_only": True} + if self.model_revision: + kwargs["revision"] = self.model_revision + encoder = CrossEncoder(self.model_name, **kwargs) + _GLOBAL_CROSS_ENCODER_CACHE[cache_key] = encoder + logger.warning("Hub unreachable (%s) — loaded %s from the local cache", first_error, cache_key) + return encoder + except Exception as exc: + logger.warning( + "CrossEncoder unavailable (%s; cache retry: %s) — using the token-similarity fallback", + first_error, + exc, + ) return None @trace_execution(logger=logger) @@ -94,7 +126,9 @@ def rank( body_text = body_text.split("]", 1)[-1].strip() is_short_heading = ( - len(body_text) < self.short_heading_threshold and item.get("modality") == "text" + self.enable_heading_penalty + and len(body_text) < self.short_heading_threshold + and item.get("modality") == "text" ) if is_short_heading: # Cross-encoder scores are unbounded signed logits, so the @@ -133,7 +167,11 @@ def rank( body_text = item.get("text", "") if "[Context:" in body_text: body_text = body_text.split("]", 1)[-1].strip() - is_short_heading = len(body_text) < self.short_heading_threshold and item.get("modality") == "text" + is_short_heading = ( + self.enable_heading_penalty + and len(body_text) < self.short_heading_threshold + and item.get("modality") == "text" + ) # The fallback score is non-negative by construction (RRF score and # Jaccard overlap are both >= 0), so a multiplicative penalty is # correctly monotone decreasing here — unlike the signed-logit path above. @@ -143,7 +181,11 @@ def rank( ranked_item = dict(item) ranked_item["rank_score"] = round(fused_rank_score, 3) - ranked_item["reranker_mode"] = "token_similarity_fallback" + # A deliberately ablated reranker and a missing model both land here, but + # they mean different things to a reader of the degradation labels. + ranked_item["reranker_mode"] = ( + "token_similarity_ablated" if not self.enable_cross_encoder else "token_similarity_fallback" + ) ranked_item["heading_penalty_applied"] = bool(is_short_heading) ranked.append(ranked_item) diff --git a/docureason/tripath/retrieval/table_sql.py b/docureason/tripath/retrieval/table_sql.py index 126e4f9..fc518be 100644 --- a/docureason/tripath/retrieval/table_sql.py +++ b/docureason/tripath/retrieval/table_sql.py @@ -108,9 +108,16 @@ def log_sql_failure(query: str, generated_sql: str, error: str, table_data: Dict class TableSQLRetriever: """Advanced DuckDB SQL execution engine with typed schemas and self-consistency.""" - def __init__(self, config: Optional[Any] = None) -> None: + def __init__(self, config: Optional[Any] = None, store: Optional[Any] = None) -> None: self.config = config self._duckdb_available: Optional[bool] = None + #: Optional :class:`~docureason.tripath.indexing.table_store.DuckDBTableStore`. + #: When a region is registered there, its table already exists with types + #: decided at ingestion, so the query does not re-create and re-insert it. + self.store = store + #: Counters for the most recent retrieve(), so the response can say which path + #: served each table rather than leaving the two indistinguishable. + self.last_stats: Dict[str, int] = {"prepared_tables": 0, "in_memory_tables": 0} def _has_duckdb(self) -> bool: if self._duckdb_available is None: @@ -127,6 +134,7 @@ def _has_duckdb(self) -> bool: def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any]]: results: List[Dict[str, Any]] = [] lowered_query = query.lower() + self.last_stats = {"prepared_tables": 0, "in_memory_tables": 0} for document in documents: for idx, region in enumerate(document.regions): @@ -137,17 +145,30 @@ def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any] if not table_data or not table_data.get("columns"): continue - sql_query, sql_result, score = self._execute_sql_query(lowered_query, table_data) + region_id = canonical_region_id(document.id, region, idx) + prepared = self._prepared_table(region_id) + if prepared is not None: + self.last_stats["prepared_tables"] += 1 + sql_query, sql_result, score = self._execute_sql_query( + lowered_query, + table_data, + table_name=prepared.table_name, + connection=self.store.connection(), + column_types=prepared.column_types, + ) + else: + self.last_stats["in_memory_tables"] += 1 + sql_query, sql_result, score = self._execute_sql_query(lowered_query, table_data) if score > 0.0: results.append( { "document_id": document.id, - "region_id": canonical_region_id(document.id, region, idx), + "region_id": region_id, "score": round(score, 3), "text": region.table_markdown or region.text, "modality": "table", - "mode": "typed_duckdb_sql" if self._has_duckdb() else "sql_simulation", + "mode": self._mode_label(prepared is not None), "sql_query": sql_query, "sql_result": sql_result, "table_schema": table_data.get("columns", []), @@ -159,6 +180,20 @@ def retrieve(self, query: str, documents: List[Document]) -> List[Dict[str, Any] return sorted(results, key=lambda item: item["score"], reverse=True) + def _prepared_table(self, region_id: str) -> Optional[Any]: + """The ingestion-registered table for *region_id*, when the store has one.""" + if self.store is None or not getattr(self.store, "available", False): + return None + entry = self.store.lookup(region_id) + if entry is None or self.store.connection() is None: + return None + return entry + + def _mode_label(self, prepared: bool) -> str: + if prepared: + return "prepared_duckdb_sql" + return "typed_duckdb_sql" if self._has_duckdb() else "sql_simulation" + def _extract_table_data(self, region: Any) -> Optional[Dict[str, Any]]: if getattr(region, "table_json", None): if isinstance(region.table_json, dict): @@ -201,20 +236,36 @@ def _parse_markdown_to_json(self, text: str) -> Optional[Dict[str, Any]]: return {"columns": headers, "rows": data_rows} return None - def _execute_sql_query(self, query: str, table_data: Dict[str, Any]) -> Tuple[str, List[Dict[str, Any]], float]: + def _execute_sql_query( + self, + query: str, + table_data: Dict[str, Any], + table_name: str = "doc_table", + connection: Optional[Any] = None, + column_types: Optional[Dict[str, str]] = None, + ) -> Tuple[str, List[Dict[str, Any]], float]: + """Generate one SQL statement for *query* and execute it. + + Two execution modes. With *connection* the table already exists - registered + at ingestion by :class:`DuckDBTableStore` with types decided once - and this + only runs the SELECT. Without it, an in-memory table is created and populated + per call, which is what happened for every query before the store existed. + """ columns = table_data.get("columns", []) rows = table_data.get("rows", []) if not columns or not rows: return "", [], 0.0 - # Lever A: Infer column SQL types - column_types = {} - for col_idx, col in enumerate(columns): - col_values = [row[col_idx] for row in rows if col_idx < len(row)] - column_types[col] = infer_sql_type(col_values) + # Lever A: Infer column SQL types. Types registered at ingestion win, so the + # same column cannot type differently from one query to the next. + if column_types is None: + column_types = {} + for col_idx, col in enumerate(columns): + col_values = [row[col_idx] for row in rows if col_idx < len(row)] + column_types[col] = infer_sql_type(col_values) table_data["inferred_types"] = column_types - table_name = "doc_table" + safe_table = quote_identifier(table_name) is_agg = any(agg in query for agg in ["sum", "average", "avg", "max", "min", "total sum"]) # Check for year match in columns @@ -248,30 +299,45 @@ def _execute_sql_query(self, query: str, table_data: Dict[str, Any]) -> Tuple[st concept_keyword = target_concept.split()[0] if year_match: generated_sql = ( - f"SELECT {quote_identifier(col_0)}, {quote_identifier(year_match)} FROM {table_name} " + f"SELECT {quote_identifier(col_0)}, {quote_identifier(year_match)} FROM {safe_table} " f"WHERE LOWER({quote_identifier(col_0)}) LIKE '%{concept_keyword}%'" ) else: generated_sql = ( - f"SELECT * FROM {table_name} WHERE LOWER({quote_identifier(col_0)}) LIKE '%{concept_keyword}%'" + f"SELECT * FROM {safe_table} WHERE LOWER({quote_identifier(col_0)}) LIKE '%{concept_keyword}%'" ) elif is_agg and numeric_cols: - generated_sql = f"SELECT SUM({quote_identifier(numeric_cols[0])}) AS total_sum FROM {table_name}" + generated_sql = f"SELECT SUM({quote_identifier(numeric_cols[0])}) AS total_sum FROM {safe_table}" elif year_match: - generated_sql = f"SELECT {quote_identifier(col_0)}, {quote_identifier(year_match)} FROM {table_name}" + generated_sql = f"SELECT {quote_identifier(col_0)}, {quote_identifier(year_match)} FROM {safe_table}" else: - generated_sql = f"SELECT * FROM {table_name}" + generated_sql = f"SELECT * FROM {safe_table}" # Execute the single generated statement in an in-memory DuckDB session. # NOTE: there is no self-consistency / majority-vote step here — one # template produces one statement and it is executed once. - if self._has_duckdb(): + if connection is not None: + try: + # The prepared store is read-only, so a SELECT is all that can run here. + rel = connection.execute(generated_sql) + fetched_rows = rel.fetchall() + col_names = [desc[0] for desc in rel.description] + result_dicts = [dict(zip(col_names, r)) for r in fetched_rows] + + col_matches = sum(1 for col in columns if str(col).lower() in query) + score = 0.6 + (0.3 if col_matches > 0 else 0.0) + return generated_sql, result_dicts, min(1.0, score) + except Exception as exc: + logger.warning("Prepared-table execution error: %s - logging failure", exc) + log_sql_failure(query, generated_sql, str(exc), table_data) + + elif self._has_duckdb(): try: import duckdb con = duckdb.connect(database=":memory:") col_defs = ", ".join(f"{quote_identifier(col)} {column_types.get(col, 'VARCHAR')}" for col in columns) - con.execute(f"CREATE TABLE {table_name} ({col_defs})") + con.execute(f"CREATE TABLE {safe_table} ({col_defs})") for row in rows: placeholders = ", ".join(["?"] * len(columns)) @@ -281,7 +347,7 @@ def _execute_sql_query(self, query: str, table_data: Dict[str, Any]) -> Tuple[st sanitize_cell_for_sql(val, column_types.get(col, "VARCHAR")) for col, val in zip(columns, row_padded[: len(columns)]) ] - con.execute(f"INSERT INTO {table_name} VALUES ({placeholders})", sanitized_row) + con.execute(f"INSERT INTO {safe_table} VALUES ({placeholders})", sanitized_row) rel = con.execute(generated_sql) fetched_rows = rel.fetchall() @@ -289,11 +355,11 @@ def _execute_sql_query(self, query: str, table_data: Dict[str, Any]) -> Tuple[st result_dicts = [dict(zip(col_names, r)) for r in fetched_rows] con.close() - col_matches = sum(1 for col in columns if col.lower() in query) + col_matches = sum(1 for col in columns if str(col).lower() in query) score = 0.6 + (0.3 if col_matches > 0 else 0.0) return generated_sql, result_dicts, min(1.0, score) except Exception as exc: - logger.warning("DuckDB execution error: %s — logging failure", exc) + logger.warning("DuckDB execution error: %s - logging failure", exc) log_sql_failure(query, generated_sql, str(exc), table_data) # In-memory Python runner fallback diff --git a/docureason/tripath/serving/query_service.py b/docureason/tripath/serving/query_service.py index d3f16a4..47515bd 100644 --- a/docureason/tripath/serving/query_service.py +++ b/docureason/tripath/serving/query_service.py @@ -29,18 +29,25 @@ "generation": {...}, # full generator payload "attribution": {...}, "retrieval_stats": {...}, + "latency_breakdown": {...}, # per-stage ms, plus total and signed residual "degradation": [str], # every fallback that fired, [] on a clean run + "cached": bool, # True when served from the response cache "preset": str, } """ from __future__ import annotations +import hashlib +import json +import threading +from collections import OrderedDict +from copy import deepcopy from pathlib import Path from typing import Any, Dict, List, Optional from docureason.tripath.config import DocuReasonConfig -from docureason.tripath.utils import get_logger, trace_execution +from docureason.tripath.utils import StageTimer, get_logger, trace_execution from ..attribution.claim_attributor import ClaimSupportAttributor from ..generation.generate import GenerationModule @@ -86,6 +93,15 @@ def __init__( self.generator = GenerationModule(config=self.config) self.attributor = ClaimSupportAttributor(config=self.config) + # Response cache. Keyed on the query *and* a fingerprint of the effective + # config, so a per-query override can never be served a response produced + # under different parameters. Bounded LRU; the corpus is fixed for the life + # of this service, and /api/ingest drops the service outright, so a stale + # corpus cannot be served from here. + self._cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + self._cache_lock = threading.Lock() + self.cache_stats: Dict[str, int] = {"hits": 0, "misses": 0, "evictions": 0} + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -102,8 +118,14 @@ def ranker(self) -> Any: @trace_execution(logger=logger) def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + timer = StageTimer() effective_config = self.config.apply_query_overrides(overrides or {}) + cache_key = self._cache_key(text, effective_config) + cached = self._cache_get(cache_key, timer) + if cached is not None: + return cached + if overrides: # A per-query override changes retrieval and generation parameters, so # those components are rebuilt; the index handles are reused. @@ -118,16 +140,21 @@ def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[s retriever, generator, attributor = self.retriever, self.generator, self.attributor router = retriever.router - route_flags = router.route(text) - probabilities = router.route_probabilities(text) - weights = router.get_route_weights(text) + with timer.stage("route"): + route_flags = router.route(text) + probabilities = router.route_probabilities(text) + weights = router.get_route_weights(text) - results = retriever.retrieve(text, self.documents) if self.documents else [] + results = retriever.retrieve(text, self.documents, timer=timer) if self.documents else [] + # The SQL path already ran inside retrieve(), which timed it into the shared + # timer as the "table_sql" stage; this only shapes the payload. sql_payload = self._build_sql_payload(retriever, route_flags, probabilities) - generation = generator.generate(text, results, sql_payload) + with timer.stage("generate"): + generation = generator.generate(text, results, sql_payload) answer_text = generation.get("answer", "") - attribution = attributor.attribute(answer_text, results) + with timer.stage("attribute"): + attribution = attributor.attribute(answer_text, results) degradation = list(retriever.last_degradation) degradation.extend(self._generation_degradation(generation)) @@ -135,7 +162,7 @@ def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[s if not self.documents: degradation.append("corpus:empty") - return { + response = { "query": text, "route": route_flags, "router": { @@ -153,9 +180,82 @@ def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[s "generation": generation, "attribution": attribution, "retrieval_stats": dict(retriever.last_stats), + "latency_breakdown": timer.as_dict(), "degradation": degradation, + "cached": False, "preset": effective_config.preset, } + self._cache_put(cache_key, response, effective_config) + return response + + # ------------------------------------------------------------------ + # Response cache + # ------------------------------------------------------------------ + + @staticmethod + def _cache_key(text: str, config: DocuReasonConfig) -> str: + """Digest of the query and everything about the config that changes the answer. + + ``output_dir`` and ``source_map`` are excluded: they record *where* artifacts + went and *why* a value was chosen, neither of which affects the response. + Every other field is included, so an ablation switch or a ``final_top_k`` + override can never be served a response computed under the other setting. + """ + payload = config.summary_dict() + payload.pop("source_map", None) + payload.pop("output_dir", None) + blob = json.dumps({"query": text.strip(), "config": payload}, sort_keys=True, default=str) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + def _cache_capacity(self, config: Optional[DocuReasonConfig] = None) -> int: + serving = getattr(config or self.config, "serving", None) + return max(0, int(getattr(serving, "response_cache_size", 0) or 0)) + + def _cache_get(self, key: str, timer: StageTimer) -> Optional[Dict[str, Any]]: + """A deep copy of the cached response, or None. + + The copy is not defensive pedantry: callers mutate responses (the eval harness + stamps metrics onto them), and a shared dict would let one query's annotations + leak into the next caller's answer. + + The returned ``latency_breakdown`` describes *this* call - a cache lookup - + with the original run's breakdown preserved under + ``latency_breakdown_first_run``. Replaying the original timings would quietly + credit the pipeline with work it did not do and drag every reported percentile + toward the cache's. + """ + if self._cache_capacity() == 0: + return None + + with self._cache_lock: + stored = self._cache.get(key) + if stored is None: + self.cache_stats["misses"] += 1 + return None + self._cache.move_to_end(key) + self.cache_stats["hits"] += 1 + response = deepcopy(stored) + + response["cached"] = True + response["latency_breakdown_first_run"] = stored.get("latency_breakdown") + response["latency_breakdown"] = timer.as_dict() + return response + + def _cache_put(self, key: str, response: Dict[str, Any], config: DocuReasonConfig) -> None: + capacity = self._cache_capacity(config) + if capacity == 0: + return + with self._cache_lock: + self._cache[key] = deepcopy(response) + self._cache.move_to_end(key) + while len(self._cache) > capacity: + self._cache.popitem(last=False) + self.cache_stats["evictions"] += 1 + + def clear_cache(self) -> None: + """Drop every cached response. Call after the corpus or indexes change.""" + with self._cache_lock: + self._cache.clear() # ------------------------------------------------------------------ # Internals diff --git a/docureason/tripath/utils/__init__.py b/docureason/tripath/utils/__init__.py index 32f687f..b9ceec1 100644 --- a/docureason/tripath/utils/__init__.py +++ b/docureason/tripath/utils/__init__.py @@ -1,4 +1,5 @@ from .logger import get_logger, log_pipeline_flag, setup_logger +from .timing import STAGE_ORDER, StageTimer from .tracing import trace_execution, trace_pipeline_stage __all__ = [ @@ -7,4 +8,6 @@ "log_pipeline_flag", "trace_execution", "trace_pipeline_stage", + "StageTimer", + "STAGE_ORDER", ] diff --git a/docureason/tripath/utils/logger.py b/docureason/tripath/utils/logger.py index 0a40baf..5cf631e 100644 --- a/docureason/tripath/utils/logger.py +++ b/docureason/tripath/utils/logger.py @@ -76,7 +76,20 @@ def setup_logger( if not any( isinstance(h, logging.StreamHandler) and not isinstance(h, logging.FileHandler) for h in logger.handlers ): - console_handler = logging.StreamHandler(sys.stdout) + # Log messages contain arrows and em dashes. On a Windows console + # (cp1252 by default) every such line raised UnicodeEncodeError inside + # logging and printed a traceback instead of the message, so ingestion logs + # were unreadable exactly where they were most needed. Reconfiguring the + # stream is preferable to policing the character set of every log string. + stream = sys.stdout + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + try: + reconfigure(encoding="utf-8", errors="replace") + except (ValueError, OSError): + # Detached or non-text stream: fall through and accept the codepage. + pass + console_handler = logging.StreamHandler(stream) console_handler.setLevel(level) console_handler.setFormatter(formatter) logger.addHandler(console_handler) diff --git a/docureason/tripath/utils/timing.py b/docureason/tripath/utils/timing.py new file mode 100644 index 0000000..1c4d5ed --- /dev/null +++ b/docureason/tripath/utils/timing.py @@ -0,0 +1,108 @@ +"""timing.py — per-stage latency accounting for a single query. + +The system reported one number per query: total wall time, measured *around* the +call. Audit finding P0-5 is that this measured the harness rather than the +pipeline, and that the per-stage decomposition the evaluation spec defines +(``T_vec_search``, ``T_dense_topk``, ``T_dedup``, ``T_rerank_query`` …) existed only +as equations. With one number, "the p95 regressed" cannot be attributed: an +embedding model reload, a cross-encoder cold start and a DuckDB failure retry are +indistinguishable. + +:class:`StageTimer` accumulates elapsed milliseconds per named stage and, crucially, +reports a **residual**: total measured wall time minus the sum of the stages. A +decomposition that does not add up is a decomposition with a hole in it, and the +residual is what makes that visible instead of letting the stages quietly describe +60% of the request. + +Stage names repeat freely - three modality retrievals all contribute to +``dense_search`` - because accumulation is what makes the sum meaningful. +""" + +from __future__ import annotations + +import time +from contextlib import contextmanager +from typing import Any, Dict, Iterator, Optional + +#: Stages are reported in this order when present, so two runs' breakdowns line up +#: visually. Anything not listed is appended alphabetically. +STAGE_ORDER = ( + "route", + "dense_search", + "sparse_search", + "index_merge", + "corpus_scan", + "table_sql", + "fuse", + "chart_enrich", + "parent_expand", + "rerank", + "generate", + "attribute", +) + + +class StageTimer: + """Accumulate per-stage elapsed time for one query. + + Not thread-safe by design for the *same* stage name: the parallel modality + dispatch gives each worker its own timer and merges the results, which keeps the + accounting correct without a lock on the hot path. + """ + + __slots__ = ("stages", "counts", "_started") + + def __init__(self) -> None: + self.stages: Dict[str, float] = {} + self.counts: Dict[str, int] = {} + self._started = time.perf_counter() + + @contextmanager + def stage(self, name: str) -> Iterator[None]: + """Time a block and add its duration to *name*.""" + start = time.perf_counter() + try: + yield + finally: + self.add(name, (time.perf_counter() - start) * 1000.0) + + def add(self, name: str, elapsed_ms: float) -> None: + self.stages[name] = self.stages.get(name, 0.0) + float(elapsed_ms) + self.counts[name] = self.counts.get(name, 0) + 1 + + def merge(self, other: "StageTimer") -> None: + """Fold another timer's stages into this one (used by the parallel dispatch).""" + for name, elapsed in other.stages.items(): + self.stages[name] = self.stages.get(name, 0.0) + elapsed + self.counts[name] = self.counts.get(name, 0) + other.counts.get(name, 1) + + @property + def elapsed_ms(self) -> float: + """Wall time since this timer was created.""" + return (time.perf_counter() - self._started) * 1000.0 + + def as_dict(self, total_ms: Optional[float] = None) -> Dict[str, Any]: + """The breakdown, plus the total and the unaccounted-for residual. + + ``residual_ms`` can be negative when stages overlap - which is exactly what + happens under the parallel modality dispatch, where three retrievals run + concurrently and their summed CPU time legitimately exceeds wall time. The + sign is the signal, so it is reported rather than clamped. + """ + total = self.elapsed_ms if total_ms is None else float(total_ms) + ordered = [name for name in STAGE_ORDER if name in self.stages] + ordered += sorted(name for name in self.stages if name not in STAGE_ORDER) + + breakdown = {name: round(self.stages[name], 3) for name in ordered} + accounted = sum(self.stages.values()) + return { + "stages_ms": breakdown, + "stage_calls": {name: self.counts[name] for name in ordered}, + "total_ms": round(total, 3), + "accounted_ms": round(accounted, 3), + "residual_ms": round(total - accounted, 3), + "concurrent": accounted > total + 1.0, + } + + +__all__ = ["StageTimer", "STAGE_ORDER"] diff --git a/pyproject.toml b/pyproject.toml index 8426914..056ca2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ "easyocr>=1.7.0,<2.0.0", "faiss-cpu>=1.8.0,<2.0.0", "pypdf>=5.0.0,<7.0.0", + # Page rasterization for figure crops and OCR (docureason/tripath/ingestion/rasterizer.py). + # Arrived transitively via docling; declared now that it is imported directly. + "pypdfium2>=4.0.0,<6.0.0", "pyyaml>=6.0.1,<7.0.0", "uvicorn>=0.20.0,<1.0.0" ] @@ -43,7 +46,15 @@ test = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "pytest-asyncio>=0.21.0", - "httpx>=0.24.0" + "httpx>=0.24.0", + # The figure builders are code, so they are tested. Here rather than only in + # [figures] so the CI gate covers them instead of skipping them. + "matplotlib>=3.7.0,<4.0.0" +] +figures = [ + # Reporting only: docureason/tripath/evaluation/figures.py imports it lazily, so a + # serving install carries no plotting stack. + "matplotlib>=3.7.0,<4.0.0" ] lint = [ "ruff>=0.1.0", @@ -56,7 +67,7 @@ build = [ "wheel>=0.41.0" ] dev = [ - "docureason-framework[test,lint,build]" + "docureason-framework[test,lint,build,figures]" ] [project.scripts] diff --git a/requirements.lock b/requirements.lock index 37d446b..c1f2130 100644 --- a/requirements.lock +++ b/requirements.lock @@ -70,6 +70,7 @@ pydantic-core==2.46.4 pydantic-settings==2.14.2 pygments==2.20.0 pypdf==6.14.2 +pypdfium2==5.12.1 python-bidi==0.6.11 python-dateutil==2.9.0.post0 python-dotenv==1.2.2 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..c114d5f --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,7 @@ +"""Repository scripts, importable so their logic can be tested. + +The metric-regression gate lives in ``check_metric_regression.py`` and is exercised by +``tests/tripath/test_metric_regression_gate.py``; a gate with no test cannot be trusted +to fail. This package is not part of the distribution - ``pyproject.toml`` ships +``docureason*`` only. +""" diff --git a/scripts/build_eval_corpus.py b/scripts/build_eval_corpus.py new file mode 100644 index 0000000..6132309 --- /dev/null +++ b/scripts/build_eval_corpus.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Build the evaluation corpus: fetch real filings, ingest them, derive gold queries. + + export DOCUREASON_SEC_USER_AGENT="Your Org corpus-builder (you@example.com)" + python scripts/build_eval_corpus.py --filings 24 --per-stratum 60 + +Four steps, each idempotent, so a failure part-way can be resumed: + +1. **fetch** SEC EDGAR 10-K primary documents -> ``data/eval/filings`` (cached on disk) +2. **ingest** the full DocuReason pipeline -> ``data/eval/corpus`` +3. **derive** stratified queries with gold region ids -> ``data/eval/queries.json`` +4. **validate** every gold id resolves; every phrase anchor is still unique + +Step 2 is the expensive one: layout parsing a 10-K on CPU takes minutes per filing, and +gold region ids only exist after it, so the query set cannot be built without it. + +The output is reproducible from ``provenance.json`` (filing URLs plus SHA-256) and the +generator seed. That is the difference between "we evaluated on 24 filings" as a claim +and as a checkable statement. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Any, Dict, List + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from docureason.pipeline import DocuReasonPipeline # noqa: E402 +from docureason.tripath.config import DocuReasonConfig # noqa: E402 +from docureason.tripath.evaluation.edgar import ( # noqa: E402 + DEFAULT_COMPANIES, + EdgarAccessError, + fetch_filings, +) +from docureason.tripath.evaluation.gold_queries import generate_queries, validate_queries # noqa: E402 +from docureason.tripath.utils import get_logger # noqa: E402 + +logger = get_logger("build_eval_corpus") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--out", default="data/eval", help="output root (default: data/eval)") + parser.add_argument("--filings", type=int, default=24, help="how many filings to fetch (default: 24)") + parser.add_argument("--form", default="10-K", help="EDGAR form type (default: 10-K)") + parser.add_argument("--per-company", type=int, default=2, help="filings per company (default: 2)") + parser.add_argument("--per-stratum", type=int, default=60, help="target queries per stratum (default: 60)") + parser.add_argument("--seed", type=int, default=0, help="generator seed (default: 0)") + parser.add_argument("--skip-fetch", action="store_true", help="reuse whatever is already in filings/") + parser.add_argument("--skip-ingest", action="store_true", help="reuse an existing corpus.json") + parser.add_argument("--corpus-dir", default=None, help="ingest into this directory (default: /corpus)") + parser.add_argument( + "--no-layout-parser", + action="store_true", + help="ingest with ingestion.enable_layout_parser=false: the corpus the no_layout_parser ablation row needs", + ) + parser.add_argument( + "--skip-derive", + action="store_true", + help="build the corpus only, leaving queries.json alone (use with --no-layout-parser)", + ) + return parser.parse_args() + + +def step_fetch(out: Path, args: argparse.Namespace) -> List[Dict[str, Any]]: + filings_dir = out / "filings" + if args.skip_fetch: + existing = sorted(p for p in filings_dir.glob("*.htm*")) + logger.info("Skipping fetch; %d filing(s) already on disk", len(existing)) + return [{"path": str(p)} for p in existing] + + try: + fetched = fetch_filings( + filings_dir, + companies=DEFAULT_COMPANIES, + form=args.form, + per_company=args.per_company, + limit=args.filings, + ) + except EdgarAccessError as exc: + raise SystemExit(f"EDGAR access failed: {exc}") from exc + + if not fetched: + raise SystemExit("no filings fetched; refusing to build a corpus from nothing") + return [item.as_dict() for item in fetched] + + +def step_ingest(out: Path, args: argparse.Namespace) -> Dict[str, Any]: + corpus_dir = Path(args.corpus_dir) if args.corpus_dir else out / "corpus" + corpus_path = corpus_dir / "corpus.json" + + if args.skip_ingest and corpus_path.exists(): + logger.info("Skipping ingest; reusing %s", corpus_path) + return json.loads(corpus_path.read_text(encoding="utf-8")) + + config = None + if args.no_layout_parser: + # The corpus the no_layout_parser ablation row answers from. It goes in its own + # directory: the gold ids belong to the layout-parsed corpus, and overwriting that + # corpus would relabel every one of them. This is a second observation, not a + # replacement. + config = DocuReasonConfig.load_from_yaml("configs/config.yaml") + config.ingestion.enable_layout_parser = False + logger.warning("Layout parser disabled; ingesting the ablation corpus into %s", corpus_dir) + + started = time.perf_counter() + report = DocuReasonPipeline( + input_dir=out / "filings", output_dir=corpus_dir, config=config, verbose=False + ).run() + logger.info( + "Ingested %s document(s) / %s chunk(s) in %.1f min", + report.get("document_count"), + report.get("chunk_count"), + (time.perf_counter() - started) / 60.0, + ) + if report.get("degradation"): + logger.warning("Ingestion degradation: %s", report["degradation"]) + return json.loads(corpus_path.read_text(encoding="utf-8")) + + +def step_derive(out: Path, corpus: Dict[str, Any], filings: List[Dict[str, Any]], args: argparse.Namespace) -> Path: + report = generate_queries(corpus, per_stratum=args.per_stratum, seed=args.seed) + records = [query.as_dict() for query in report.queries] + validation = validate_queries(records, corpus) + + payload = { + "dataset": "docureason-edgar-eval", + "built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "seed": args.seed, + "form": args.form, + "filing_count": len(filings), + "provenance": "Gold regions are derived programmatically from the ingested corpus, " + "not human-annotated: table cells give a known row/column intersection, and prose " + "and figure anchors are phrases verified unique across the corpus. Derivable queries " + "over-represent lookup and cannot express multi-hop reasoning.", + "generation_report": report.as_dict(), + "validation": validation, + "queries": records, + } + + path = out / "queries.json" + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + logger.info("Wrote %d query(ies) to %s", len(records), path) + + if not validation["valid"]: + logger.error( + "Validation failed: %d dangling gold id(s), %d non-unique anchor(s)", + len(validation["dangling_gold_ids"]), + len(validation["non_unique_anchors"]), + ) + return path + + +def main() -> int: + args = parse_args() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + filings = step_fetch(out, args) + corpus = step_ingest(out, args) + + if args.skip_derive: + # Deriving here would relabel the gold ids against whatever corpus this run built. + corpus_dir = Path(args.corpus_dir) if args.corpus_dir else out / "corpus" + print("") + print("Corpus built; query derivation skipped") + print(f" filings : {len(filings)}") + print(f" documents : {len(corpus.get('documents', []))}") + print(f" layout parser : {'off' if args.no_layout_parser else 'on'}") + print(f" written to : {corpus_dir}") + return 0 + + path = step_derive(out, corpus, filings, args) + + payload = json.loads(path.read_text(encoding="utf-8")) + report = payload["generation_report"] + validation = payload["validation"] + + print("") + print("Evaluation corpus") + print(f" filings : {len(filings)}") + print(f" documents : {report['documents']}") + print(f" regions scanned : {report['regions_scanned']}") + print(f" queries : {report['query_count']}") + for stratum, count in sorted(report["per_stratum"].items()): + print(f" {stratum:16s}: {count}") + print(f" rejected : {report['rejected']}") + print(f" validation : {'PASS' if validation['valid'] else 'FAIL'}") + print(f" written to : {path}") + return 0 if validation["valid"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_kaggle_eval_notebook.py b/scripts/build_kaggle_eval_notebook.py new file mode 100644 index 0000000..2ce20b0 --- /dev/null +++ b/scripts/build_kaggle_eval_notebook.py @@ -0,0 +1,882 @@ +#!/usr/bin/env python3 +"""scripts/build_kaggle_eval_notebook.py - emit the full-pipeline Kaggle notebook. + + python scripts/build_kaggle_eval_notebook.py + +Writes ``notebooks/docureason_full_eval_kaggle.ipynb``: clone this repository at a ref, +install it, build the SEC EDGAR evaluation corpus, score the full system, execute the +seven-row ablation matrix against the naive-RAG baseline, run the metric-regression gate, +and export every artifact - corpus, FAISS/BM25 indexes, DuckDB table store, model +weights cache, reports - as a reusable Kaggle dataset. + +Why a builder instead of a committed ``.ipynb``: ``notebooks/`` is gitignored, notebook +JSON reviews badly, and the cells are mostly thin calls into ``scripts/`` so the repo +stays the single source of truth for what the pipeline does. Edit here, regenerate. + +The notebook is deliberately **resumable**. A first run on a T4 spends most of its time +in ingestion and embedding; attaching the previous run's output as a dataset skips both, +so a second run measures instead of rebuilding. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Dict, List, Tuple + +DEFAULT_OUTPUT = Path("notebooks/docureason_full_eval_kaggle.ipynb") +REPO_URL = "https://github.com/arpitkumar2004/DocuReason.git" + +CELLS: List[Tuple[str, str]] = [] + + +def md(source: str) -> None: + CELLS.append(("markdown", source.strip("\n"))) + + +def code(source: str) -> None: + CELLS.append(("code", source.strip("\n"))) + + +md( + r""" +# DocuReason — full pipeline, measured end to end + +Runs the **whole** DocuReason evaluation chain from the GitHub source: real SEC EDGAR +filings in, a scored ablation matrix out. Nothing here is a fixture and nothing is +asserted — every number this notebook prints is computed in this session. + +| Stage | What runs | First run (T4) | +| :-- | :--- | :-- | +| 1–4 | Config, environment audit, clone + install, import audit | ~4 min | +| 5 | Artifact-cache restore (skipped on a first run) | seconds | +| 6 | **Smoke gate** — one query end to end over `samples/` | ~2 min | +| 7 | EDGAR fetch → ingest → derive gold queries → validate | ~45–90 min | +| 8 | Second corpus, layout parser off (the ingestion-scoped ablation row) | ~15 min | +| 9 | Full system scored over every gold query | ~15 min | +| 10 | Seven ablations + naive-RAG baseline, paired with CIs | ~60–120 min | +| 11 | Metric-regression gate | seconds | +| 12 | Results tables | seconds | +| 13 | Seven figures (PNG + PDF) and a computed insight digest | ~1 min | +| 14 | Export corpus, indexes, weights cache and reports as a reusable dataset | ~3 min | + +**Before you run:** + +1. **Settings → Accelerator → GPU** (T4 or P100). CPU works, but TableFormer table + structure and the NLI path are CUDA-gated, and ingestion is roughly 4× slower. +2. **Settings → Internet → On.** Required for the clone, PyPI, the HuggingFace hub and + EDGAR. +3. **Edit `SEC_USER_AGENT` in the next cell.** The SEC requires a real contact string in + the `User-Agent` header, and `edgar.py` refuses to send a request without one rather + than inventing an identity. Requests are rate-limited well under the 10 req/s + fair-access limit. + +**To make the second run cheap:** when this finishes, *Save Version → Save & Run All*, +then in a new session attach this notebook's output under +`/kaggle/input/docureason-artifacts`. Cell 5 restores the corpus, the indexes, the +DuckDB table store and the model-weight cache, and cells 7–8 become no-ops. +""" +) + +code( + r""" +# ============================================================================ +# CELL 1 - CONFIGURATION. This is the only cell you need to edit. +# ============================================================================ +import os +from pathlib import Path + +# --- required --------------------------------------------------------------- +# The SEC requires a real identity. Format: "Org Name contact@example.com". +SEC_USER_AGENT = "DocuReason Research your-email@example.com" + +# --- source ---------------------------------------------------------------- +REPO_URL = "https://github.com/arpitkumar2004/DocuReason.git" +REPO_REF = "main" # branch, tag or commit sha + +# --- evaluation scale ------------------------------------------------------ +FILINGS = 24 # 10-K filings to fetch (roadmap asks for 20-50) +PER_COMPANY = 2 +PER_STRATUM = 60 # target gold queries per stratum +SEED = 0 +BOOTSTRAP_ITERATIONS = 2000 + +EVAL_LIMIT = 0 # queries for the full-system suite (0 = all) +ABLATION_LIMIT = 0 # queries per ablation row (0 = all; this is the long pole) +ABLATION_STRATIFIED = True # if you set a limit, sample it evenly across strata + +RUN_NO_LAYOUT_CORPUS = True # build corpus #2 so the 7th ablation row can execute +RUN_ABLATIONS = True +RUN_REGRESSION_GATE = True + +# --- caching --------------------------------------------------------------- +# Attach a previous run's output here to skip fetch, ingestion and embedding. +CACHE_INPUT = Path("/kaggle/input/docureason-artifacts") + +WORK = Path("/kaggle/working") +REPO_DIR = WORK / "DocuReason" +EXPORT_DIR = WORK / "docureason_artifacts" +HF_CACHE = WORK / "hf_cache" + +# Set before anything imports transformers: the weights must land somewhere that +# survives the session, or every rerun re-downloads them. +HF_CACHE.mkdir(parents=True, exist_ok=True) +os.environ["HF_HOME"] = str(HF_CACHE) +os.environ["HF_HUB_CACHE"] = str(HF_CACHE / "hub") +os.environ["SENTENCE_TRANSFORMERS_HOME"] = str(HF_CACHE / "sentence-transformers") +os.environ["DOCUREASON_SEC_USER_AGENT"] = SEC_USER_AGENT +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +assert "your-email@example.com" not in SEC_USER_AGENT, ( + "Set SEC_USER_AGENT to a real contact string before running: the SEC's fair-access " + "policy requires it, and edgar.py refuses to send a request without one." +) +print("Configuration accepted.") +print(f" repo : {REPO_URL} @ {REPO_REF}") +print(f" filings : {FILINGS} per stratum: {PER_STRATUM} seed: {SEED}") +print(f" cache input : {CACHE_INPUT} (exists: {CACHE_INPUT.exists()})") +print(f" export to : {EXPORT_DIR}") +""" +) + + +code( + r""" +# ============================================================================ +# CELL 2 - ENVIRONMENT AUDIT + the command runner every later cell uses +# ============================================================================ +import json +import shutil +import subprocess +import sys +import time + + +def run(cmd, cwd=None, check=True, label=None): + # Streams output as it arrives: an ingestion step that prints nothing for 40 minutes + # is indistinguishable from a hung one. + printable = " ".join(str(part) for part in cmd) + print(f"$ {printable}\n") + started = time.perf_counter() + process = subprocess.Popen( + [str(part) for part in cmd], + cwd=str(cwd) if cwd else None, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + errors="replace", + ) + captured = [] + for line in process.stdout: + captured.append(line) + print(line.rstrip()) + process.wait() + minutes = (time.perf_counter() - started) / 60.0 + print(f"\n[exit {process.returncode} after {minutes:.1f} min]") + if check and process.returncode != 0: + raise RuntimeError(f"{label or printable} failed with exit code {process.returncode}") + return "".join(captured) + + +print("=" * 78) +print("ENVIRONMENT") +print("=" * 78) +print(f"python : {sys.version.split()[0]}") + +try: + import torch + + cuda = torch.cuda.is_available() + print(f"torch : {torch.__version__} cuda={cuda}") + if cuda: + properties = torch.cuda.get_device_properties(0) + print(f"gpu : {properties.name} {properties.total_memory / 1024 ** 3:.1f} GB") + else: + print("gpu : none - TableFormer structure recognition and the NLI path are") + print(" CUDA-gated, so this run will report them as degraded, honestly.") +except ImportError: + print("torch : not importable yet") + +try: + import psutil + + print(f"ram : {psutil.virtual_memory().total / 1024 ** 3:.1f} GB") +except ImportError: + pass + +usage = shutil.disk_usage("/kaggle/working") +print(f"disk (work) : {usage.free / 1024 ** 3:.1f} GB free of {usage.total / 1024 ** 3:.1f} GB") + +# Internet is needed for the clone, PyPI, the HF hub and EDGAR. Fail here, not in step 7. +probe = subprocess.run( + [sys.executable, "-c", "import urllib.request;urllib.request.urlopen('https://pypi.org', timeout=15)"], + capture_output=True, +) +print(f"internet : {'reachable' if probe.returncode == 0 else 'UNREACHABLE - enable Settings > Internet'}") +if probe.returncode != 0: + raise SystemExit("Internet is off. Every remaining cell needs it.") +print("=" * 78) +""" +) + +code( + r""" +# ============================================================================ +# CELL 3 - CLONE THE REPOSITORY AND INSTALL IT +# ============================================================================ +# The framework on PyPI is 1.1.3; the evaluation machinery this notebook drives lives in +# the repository. So: clone a ref, install from source, and print the sha that produced +# every number below. +if REPO_DIR.exists(): + print(f"{REPO_DIR} already present - reusing it (delete it to re-clone).") +else: + run(["git", "clone", "--depth", "1", "--branch", REPO_REF, REPO_URL, str(REPO_DIR)], label="git clone") + +commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=str(REPO_DIR), capture_output=True, text=True +).stdout.strip() +print(f"\nrepository sha: {commit}") + +os.chdir(REPO_DIR) # configs/config.yaml is resolved relative to the working directory +print(f"cwd: {Path.cwd()}") + +# Pin whatever torch stack Kaggle already has, so resolving docling's dependency tree +# cannot swap the CUDA build for a CPU wheel and silently cost us the GPU. +constraints = WORK / "constraints.txt" +pins = [] +for module in ("torch", "torchvision", "torchaudio", "transformers"): + try: + version = __import__("importlib.metadata", fromlist=["version"]).version(module) + pins.append(f"{module}=={version}") + except Exception: + continue +constraints.write_text("\n".join(pins) + "\n", encoding="utf-8") +print("constraints:", pins) + +run([sys.executable, "-m", "pip", "install", "-q", "-e", ".", "--no-deps"], label="pip install -e .") +run( + [ + sys.executable, "-m", "pip", "install", "-q", "-c", str(constraints), + "duckdb", "bm25s", "docling", "easyocr", "faiss-cpu", "pypdf", "pypdfium2", + "sentence-transformers", "pyyaml", "fastapi", "uvicorn", "pydantic", + ], + label="pip install runtime dependencies", +) + +import importlib + +import torch + +importlib.invalidate_caches() +print(f"\ntorch after install: {torch.__version__} cuda={torch.cuda.is_available()}") +if not torch.cuda.is_available(): + print("NOTE: no CUDA now. If it was available in Cell 2, the install replaced the") + print(" torch build - restart the session and pin torch in the constraints list.") +""" +) + + +code( + r""" +# ============================================================================ +# CELL 4 - IMPORT AUDIT: what is available decides what the run can claim +# ============================================================================ +import docureason + +print(f"docureason {docureason.__version__} (sha {commit[:10]})\n") + +MODULES = [ + ("docling", "layout parsing, table structure, reading order"), + ("duckdb", "the SQL path over serialized tables"), + ("bm25s", "the sparse retrieval path"), + ("faiss", "the dense index"), + ("sentence_transformers", "the embedding model and the cross-encoder"), + ("pypdfium2", "page rasterization for figure crops and OCR"), + ("pypdf", "PDF text and page counts"), + ("easyocr", "OCR fallback for scanned pages"), +] +available = {} +for name, purpose in MODULES: + try: + __import__(name) + available[name] = True + print(f" OK {name:24s} {purpose}") + except Exception as exc: + available[name] = False + print(f" MISSING {name:24s} {purpose} ({type(exc).__name__})") + +missing_core = [name for name in ("docling", "duckdb", "bm25s", "faiss") if not available[name]] +if missing_core: + raise SystemExit(f"Core dependencies missing: {missing_core}. Re-run Cell 3.") +print("\nEvery capability this run reports will be an observation, not a claim:") +print("quality_audit.json records what actually ran, and each response carries a") +print("degradation list naming any fallback that fired.") +""" +) + +code( + r""" +# ============================================================================ +# CELL 5 - RESTORE FROM A PREVIOUS RUN'S OUTPUT (skip on a first run) +# ============================================================================ +# What is worth carrying between sessions: the fetched filings, the ingested corpus with +# its FAISS/BM25 indexes and DuckDB store, the derived gold queries, and the model +# weights. Ingestion and embedding are the expensive parts; measuring is cheap. +RESTORED = {"filings": False, "corpus": False, "queries": False, "corpus_no_layout": False, "hf_cache": False} + + +def _restore(source: Path, destination: Path, key: str) -> None: + if not source.exists(): + return + destination.parent.mkdir(parents=True, exist_ok=True) + already = any(destination.iterdir()) if destination.is_dir() else destination.exists() + if already: + # A rerun of this cell must not overwrite work the session has already done. + print(f" keeping existing {destination}") + RESTORED[key] = True + return + if source.is_dir(): + shutil.copytree(source, destination, dirs_exist_ok=True) + else: + shutil.copy2(source, destination) + RESTORED[key] = True + print(f" restored {key:18s} <- {source}") + + +if CACHE_INPUT.exists(): + print(f"Restoring from {CACHE_INPUT}") + _restore(CACHE_INPUT / "data" / "eval" / "filings", REPO_DIR / "data" / "eval" / "filings", "filings") + _restore(CACHE_INPUT / "data" / "eval" / "corpus", REPO_DIR / "data" / "eval" / "corpus", "corpus") + _restore(CACHE_INPUT / "data" / "eval" / "queries.json", REPO_DIR / "data" / "eval" / "queries.json", "queries") + _restore( + CACHE_INPUT / "data" / "eval" / "corpus_no_layout", + REPO_DIR / "data" / "eval" / "corpus_no_layout", + "corpus_no_layout", + ) + if (CACHE_INPUT / "hf_cache").exists() and not any(HF_CACHE.iterdir()): + shutil.copytree(CACHE_INPUT / "hf_cache", HF_CACHE, dirs_exist_ok=True) + RESTORED["hf_cache"] = True + print(" restored hf_cache <- model weights, no re-download") +else: + print(f"No cache attached at {CACHE_INPUT} - this is a full first run.") + +print("\nrestored:", {key: value for key, value in RESTORED.items() if value} or "nothing") +""" +) + +code( + r""" +# ============================================================================ +# CELL 6 - SMOKE GATE: one query, end to end, before spending an hour on EDGAR +# ============================================================================ +# Ingests the repository's own samples/ into a throwaway corpus and asks one question. +# If the wiring is broken, it breaks here in two minutes rather than after step 7. +from docureason.pipeline import DocuReasonPipeline +from docureason.tripath.serving.query_service import QueryService + +SMOKE_CORPUS = WORK / "smoke_corpus" +smoke_report = DocuReasonPipeline(input_dir="samples", output_dir=SMOKE_CORPUS, verbose=False).run() +print(f"ingested {smoke_report.get('document_count')} document(s) / {smoke_report.get('chunk_count')} chunk(s)") +print(f"degradation: {smoke_report.get('degradation') or 'none'}") + +smoke_service = QueryService(input_dir="samples", output_dir=SMOKE_CORPUS, verbose=False) +smoke_response = smoke_service.query("What revenue figure is reported in the table?") + +REQUIRED_KEYS = ["results", "answer", "retrieved_evidence", "latency_breakdown", "degradation", "cached"] +missing = [key for key in REQUIRED_KEYS if key not in smoke_response] +assert not missing, f"response is missing {missing} - the query path is not wired as documented" + +breakdown = smoke_response["latency_breakdown"] +print(f"\nresults : {len(smoke_response['results'])}") +print(f"answer : {str(smoke_response['answer'])[:160]}") +print(f"retrieval mode : {[(m, s.get('mode')) for m, s in (smoke_response.get('retrieval_stats') or {}).items()]}") +print(f"total_ms : {breakdown.get('total_ms')} residual_ms: {breakdown.get('residual_ms')}") +print(f"stages : {sorted((breakdown.get('stages_ms') or {}).keys())}") +print(f"degradation : {smoke_response['degradation'] or 'none'}") + +# The cache must return the same answer without re-running the pipeline. +again = smoke_service.query("What revenue figure is reported in the table?") +print(f"\nsecond call cached: {again.get('cached')} (first-run timings kept: {'latency_breakdown_first_run' in again})") +assert again["answer"] == smoke_response["answer"], "cache returned a different answer" +print("\nSMOKE GATE PASSED") +""" +) + + +code( + r""" +# ============================================================================ +# CELL 7 - BUILD THE EVALUATION CORPUS: fetch, ingest, derive gold queries, validate +# ============================================================================ +# Four idempotent steps. The expensive one is ingestion: layout parsing a 10-K takes +# minutes, and gold region ids only exist after it, so the corpus is the schedule. +EVAL_ROOT = REPO_DIR / "data" / "eval" +CORPUS_DIR = EVAL_ROOT / "corpus" +QUERIES = EVAL_ROOT / "queries.json" +NO_LAYOUT_DIR = EVAL_ROOT / "corpus_no_layout" + +build_cmd = [ + sys.executable, "scripts/build_eval_corpus.py", + "--out", str(EVAL_ROOT), + "--filings", str(FILINGS), + "--per-company", str(PER_COMPANY), + "--per-stratum", str(PER_STRATUM), + "--seed", str(SEED), +] +if RESTORED["filings"]: + build_cmd.append("--skip-fetch") +if RESTORED["corpus"] and (CORPUS_DIR / "corpus.json").exists(): + build_cmd.append("--skip-ingest") + +if RESTORED["queries"] and QUERIES.exists() and (CORPUS_DIR / "corpus.json").exists(): + print(f"Reusing the restored corpus and query set at {EVAL_ROOT}") +else: + run(build_cmd, label="build_eval_corpus") + +payload = json.loads(QUERIES.read_text(encoding="utf-8")) +report, validation = payload["generation_report"], payload["validation"] +print("\nCorpus") +print(f" filings : {payload['filing_count']}") +print(f" documents : {report['documents']}") +print(f" regions : {report['regions_scanned']}") +print(f" gold queries : {report['query_count']} {report['per_stratum']}") +print(f" rejected : {report['rejected']}") +print(f" validation : {'PASS' if validation['valid'] else 'FAIL'}") +if not validation["valid"]: + raise SystemExit("gold ids do not resolve against the corpus - refusing to measure against broken labels") + +audit = json.loads((CORPUS_DIR / "quality_audit.json").read_text(encoding="utf-8")) +capabilities = audit.get("capabilities_used", {}) +print("\nWhat actually ran (observed, not claimed)") +print(f" regions by layout source : {capabilities.get('regions_by_layout_source')}") +print(f" tables serialized : {capabilities.get('structured_tables_serialized')}") +print(f" table structure enabled : {capabilities.get('table_structure_enabled')}") +print(f" duckdb table store : {capabilities.get('table_store')}") +print(f" embedding model : {capabilities.get('dense_index_model')} @ {capabilities.get('dense_index_revision')}") +print(f" index stats : {capabilities.get('index_stats')}") +print(f" degradation : {audit.get('degradation') or 'none'}") +""" +) + +code( + r""" +# ============================================================================ +# CELL 8 - CORPUS #2: the same filings, layout parser off +# ============================================================================ +# The layout-parser ablation is ingestion-scoped: the parser runs once per corpus, so it +# cannot be switched off per query. Without this second corpus the seventh matrix row can +# only be reported as "not executed" - which the runner does, rather than printing a zero. +# +# Its gold ids are NOT comparable with the main corpus's: re-ingesting renames every +# region, so the runner reports the gold-id metrics for that row as not comparable and +# carries it on answer quality plus a region/table census instead. +if not RUN_NO_LAYOUT_CORPUS: + print("Skipped by configuration; the no_layout_parser row will report as not executed.") +elif (NO_LAYOUT_DIR / "corpus.json").exists(): + print(f"Reusing {NO_LAYOUT_DIR}") +else: + run( + [ + sys.executable, "scripts/build_eval_corpus.py", + "--out", str(EVAL_ROOT), + "--corpus-dir", str(NO_LAYOUT_DIR), + "--no-layout-parser", + "--skip-derive", # deriving here would relabel every gold id against this corpus + "--skip-fetch", + ], + label="build_eval_corpus --no-layout-parser", + ) + +if (NO_LAYOUT_DIR / "quality_audit.json").exists(): + ablated_audit = json.loads((NO_LAYOUT_DIR / "quality_audit.json").read_text(encoding="utf-8")) + ablated_caps = ablated_audit.get("capabilities_used", {}) + print("\nlayout parser on vs off") + print(f" regions : {audit.get('total_regions')} vs {ablated_audit.get('total_regions')}") + print(f" chunks : {audit.get('total_chunks')} vs {ablated_audit.get('total_chunks')}") + print( + f" tables : {capabilities.get('structured_tables_serialized')}" + f" vs {ablated_caps.get('structured_tables_serialized')}" + ) + print(f" sources : {capabilities.get('regions_by_layout_source')} vs {ablated_caps.get('regions_by_layout_source')}") +""" +) + +code( + r""" +# ============================================================================ +# CELL 9 - SCORE THE FULL SYSTEM OVER EVERY GOLD QUERY +# ============================================================================ +# Bootstrap intervals, a per-stratum breakdown, the per-stage latency decomposition and a +# census of every degradation label the responses carried. +eval_cmd = [ + sys.executable, "scripts/run_eval_suite.py", + "--queries", str(QUERIES), + "--filings", str(EVAL_ROOT / "filings"), + "--corpus", str(CORPUS_DIR), + "--out", "artifacts/evaluation", + "--iterations", str(BOOTSTRAP_ITERATIONS), + "--run-name", "eval_suite", +] +if EVAL_LIMIT: + eval_cmd += ["--limit", str(EVAL_LIMIT), "--stratified"] +run(eval_cmd, label="run_eval_suite") + +suite = json.loads((REPO_DIR / "artifacts" / "evaluation" / "eval_suite.json").read_text(encoding="utf-8")) +print(f"\nscored {suite['queries_scored']} queries in {suite['wall_clock_sec']}s") +""" +) + + +code( + r""" +# ============================================================================ +# CELL 10 - THE SEVEN-ROW ABLATION MATRIX, AND THE NAIVE-RAG BASELINE +# ============================================================================ +# Every row re-runs every query with one component disabled and reports a *paired* +# comparison: mean delta, bootstrap CI, sign-flip p-value. Each row prints its measured +# delta beside the delta Documentations/eval_method.md publishes, with a verdict, so a +# claim the measurement does not support is visible instead of inherited. +# +# This is the long pole: (1 + rows + baseline) x queries full pipeline executions. +ablation_cmd = [ + sys.executable, "scripts/run_ablations.py", + "--queries", str(QUERIES), + "--filings", str(EVAL_ROOT / "filings"), + "--corpus", str(CORPUS_DIR), + "--out", "artifacts/ablations", + "--iterations", str(BOOTSTRAP_ITERATIONS), + "--seed", str(SEED), +] +if ABLATION_LIMIT: + ablation_cmd += ["--limit", str(ABLATION_LIMIT)] + if ABLATION_STRATIFIED: + ablation_cmd.append("--stratified") +if RUN_NO_LAYOUT_CORPUS and (NO_LAYOUT_DIR / "corpus.json").exists(): + ablation_cmd += ["--ingestion-corpus", f"no_layout_parser={NO_LAYOUT_DIR}"] + +if RUN_ABLATIONS: + run(ablation_cmd, label="run_ablations") +else: + print("Skipped by configuration.") +""" +) + +code( + r""" +# ============================================================================ +# CELL 11 - THE METRIC REGRESSION GATE +# ============================================================================ +# Scores a frozen fixture - 20 cases across four strata, fixed retrieval output, no +# models, no network - against a committed baseline of 99 values. It runs in CI as a +# blocking step; running it here proves this environment reproduces those numbers before +# any of the numbers above are trusted. +if RUN_REGRESSION_GATE: + run([sys.executable, "scripts/check_metric_regression.py", "--tolerance", "0.01"], label="metric regression gate") +else: + print("Skipped by configuration.") +""" +) + +code( + r""" +# ============================================================================ +# CELL 12 - RESULTS +# ============================================================================ +from IPython.display import Image, Markdown, display + +ABL_DIR = REPO_DIR / "artifacts" / "ablations" +lines = ["## Full system", ""] + +summary = suite["summary_metrics"] +intervals = suite.get("confidence_intervals") or {} +alias = {"average_recall_at_5": "recall_at_5", "average_ndcg_at_5": "ndcg_at_5", "mean_latency_ms": "latency_ms"} +lines += ["| metric | value | 95% CI |", "| :--- | ---: | :--- |"] +for key in ( + "average_recall_at_5", "average_ndcg_at_5", "mrr", "map_at_10", + "attribution_precision", "hallucination_rate", "token_f1", "exact_match", + "sql_execution_success_rate", "latency_p50_ms", "latency_p90_ms", +): + if key not in summary: + continue + interval = intervals.get(alias.get(key, key)) or {} + low, high = interval.get("ci_low"), interval.get("ci_high") + span = "—" if low is None else f"[{low:.4f}, {high:.4f}]" + lines.append(f"| `{key}` | {summary[key]:.4f} | {span} |") + +lines += ["", "### Per stratum", "", "| stratum | n | recall@5 | ndcg@5 | token_f1 |", "| :--- | ---: | ---: | ---: | ---: |"] +def cell(metrics, key): + value = (metrics or {}).get(key) + # "not measured" and "measured zero" are different findings; an em dash says which. + return "—" if not isinstance(value, (int, float)) else f"{value:.4f}" + + +for stratum, block in sorted((suite.get("per_stratum") or {}).items()): + metrics = block.get("summary_metrics") or {} + lines.append( + f"| {stratum} | {block.get('n')} | {cell(metrics, 'average_recall_at_5')} " + f"| {cell(metrics, 'average_ndcg_at_5')} | {cell(metrics, 'token_f1')} |" + ) + +stages = suite.get("stage_latency_ms") or {} +stage_rows = [(name, block) for name, block in stages.items() if isinstance(block, dict)] +if stage_rows: + lines += ["", "### Per-stage latency (ms)", "", "| stage | mean | p95 |", "| :--- | ---: | ---: |"] + for name, block in sorted(stage_rows, key=lambda row: -row[1].get("mean_ms", 0)): + lines.append(f"| `{name}` | {block.get('mean_ms')} | {block.get('p95_ms')} |") + +lines += ["", "### Degradation", ""] +lines.append( + "\n".join(f"- `{label}` on {count} query(ies)" for label, count in (suite.get("degradation_counts") or {}).items()) + or "- none reported on any query" +) + +matrix_path = ABL_DIR / "ablation_matrix.md" +if matrix_path.exists(): + lines += ["", "## Ablation matrix", "", matrix_path.read_text(encoding="utf-8").split("\n", 1)[1].strip()] + +ablation_report = {} +if (ABL_DIR / "ablation_report.json").exists(): + ablation_report = json.loads((ABL_DIR / "ablation_report.json").read_text(encoding="utf-8")) + baseline = ablation_report.get("baseline") or {} + if baseline: + lines += [ + "", f"## Full system vs {baseline.get('system')}", "", + "Sign convention: positive means the full system scores higher.", "", + "| metric | Δ | 95% CI | p | significant |", "| :--- | ---: | :--- | ---: | :-- |", + ] + for metric, comparison in baseline.get("full_vs_baseline", {}).items(): + delta = comparison.get("delta") + if delta is None: + continue + low, high = comparison.get("delta_ci_low"), comparison.get("delta_ci_high") + p_value = comparison.get("p_value") + lines.append( + f"| `{metric}` | {delta:+.4f} | [{low:+.4f}, {high:+.4f}] " + f"| {'—' if p_value is None else f'{p_value:.4f}'} | {'yes' if comparison.get('significant') else 'no'} |" + ) + subset = ablation_report.get("query_subset") or {} + lines += [ + "", "### Scope of this run", "", + f"- queries used: **{subset.get('queries_used')}** (full matrix: **{subset.get('is_full_matrix')}**)", + f"- bootstrap iterations: {ablation_report.get('bootstrap_iterations')}, seed {ablation_report.get('seed')}", + f"- rows executed: {ablation_report.get('executed')} of {len(ablation_report.get('ablations') or [])}", + f"- not executed: {ablation_report.get('not_executed') or 'none'}", + ] + +RESULTS_MD = "\n".join(lines) +display(Markdown(RESULTS_MD)) +""" +) + + +code( + r""" +# ============================================================================ +# CELL 13 - SCIENTIFIC FIGURES AND THE INSIGHT DIGEST +# ============================================================================ +# Seven figures plus INSIGHTS.md, every number in them read out of the two JSON reports: +# point estimates with their bootstrap intervals, quality per stratum, the latency +# decomposition and its full distribution, the degradation census, each ablation's measured +# effect beside the claim it is meant to support, and the paired comparison against naive +# RAG. PDF alongside PNG, so the same figure goes into a paper without restyling. +run( + [sys.executable, "-m", "pip", "install", "-q", "-c", str(constraints), "matplotlib"], + label="pip install matplotlib", +) + +FIG_DIR = REPO_DIR / "artifacts" / "figures" +run( + [ + sys.executable, "scripts/plot_results.py", + "--eval-report", "artifacts/evaluation/eval_suite.json", + "--ablation-report", "artifacts/ablations/ablation_report.json", + "--out", str(FIG_DIR), + "--formats", "png,pdf", + ], + label="plot_results", +) + +INSIGHTS_MD = (FIG_DIR / "INSIGHTS.md").read_text(encoding="utf-8") +display(Markdown(INSIGHTS_MD)) +for png in sorted(FIG_DIR.glob("fig*.png")): + display(Image(filename=str(png))) +""" +) + + +code( + r""" +# ============================================================================ +# CELL 14 - EXPORT EVERYTHING AS A REUSABLE DATASET +# ============================================================================ +# Anything expensive to recompute goes here: the fetched filings and their provenance, +# the ingested corpus with its FAISS/BM25 indexes and DuckDB table store (these *are* the +# cached embeddings), the derived gold queries, the second corpus, every report, and the +# model-weight cache. Attach this output under /kaggle/input/docureason-artifacts next +# session and Cell 5 skips straight to measurement. +EXPORT_MODEL_CACHE = True + +if EXPORT_DIR.exists(): + shutil.rmtree(EXPORT_DIR) +EXPORT_DIR.mkdir(parents=True, exist_ok=True) + + +def _copy(source: Path, relative: str) -> int: + if not source.exists(): + return 0 + destination = EXPORT_DIR / relative + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, destination, dirs_exist_ok=True) + else: + shutil.copy2(source, destination) + return sum(path.stat().st_size for path in destination.rglob("*") if path.is_file()) if destination.is_dir() else destination.stat().st_size + + +sizes = { + "data/eval/filings": _copy(EVAL_ROOT / "filings", "data/eval/filings"), + "data/eval/queries.json": _copy(QUERIES, "data/eval/queries.json"), + "data/eval/corpus": _copy(CORPUS_DIR, "data/eval/corpus"), + "data/eval/corpus_no_layout": _copy(NO_LAYOUT_DIR, "data/eval/corpus_no_layout"), + "artifacts/ablations": _copy(REPO_DIR / "artifacts" / "ablations", "artifacts/ablations"), + "artifacts/evaluation": _copy(REPO_DIR / "artifacts" / "evaluation", "artifacts/evaluation"), + "artifacts/figures": _copy(FIG_DIR, "artifacts/figures"), + "artifacts/logs": _copy(REPO_DIR / "artifacts" / "logs", "artifacts/logs"), +} +if EXPORT_MODEL_CACHE: + sizes["hf_cache"] = _copy(HF_CACHE, "hf_cache") + +(EXPORT_DIR / "RESULTS.md").write_text(RESULTS_MD, encoding="utf-8") + +manifest = { + "produced_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "repository": {"url": REPO_URL, "ref": REPO_REF, "sha": commit}, + "framework_version": docureason.__version__, + "environment": { + "python": sys.version.split()[0], + "torch": torch.__version__, + "cuda": torch.cuda.is_available(), + "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None, + "modules_available": available, + }, + "configuration": { + "filings": FILINGS, + "per_company": PER_COMPANY, + "per_stratum": PER_STRATUM, + "seed": SEED, + "bootstrap_iterations": BOOTSTRAP_ITERATIONS, + "eval_limit": EVAL_LIMIT, + "ablation_limit": ABLATION_LIMIT, + "no_layout_corpus": RUN_NO_LAYOUT_CORPUS, + }, + "corpus": { + "filings": payload["filing_count"], + "documents": report["documents"], + "regions": report["regions_scanned"], + "gold_queries": report["query_count"], + "per_stratum": report["per_stratum"], + "rejected": report["rejected"], + "validation_passed": validation["valid"], + "capabilities_used": capabilities, + }, + "results": { + "summary_metrics": suite["summary_metrics"], + "queries_scored": suite["queries_scored"], + "degradation_counts": suite.get("degradation_counts"), + "ablation_rows_executed": ablation_report.get("executed"), + "ablation_rows_not_executed": ablation_report.get("not_executed"), + "is_full_matrix": (ablation_report.get("query_subset") or {}).get("is_full_matrix"), + }, + "restored_from_cache": RESTORED, + "bytes": sizes, +} +(EXPORT_DIR / "MANIFEST.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + +total = sum(sizes.values()) +print(f"Exported to {EXPORT_DIR}\n") +for name, size in sorted(sizes.items(), key=lambda item: -item[1]): + print(f" {name:34s} {size / 1024 ** 2:9.1f} MB") +print(f" {'TOTAL':34s} {total / 1024 ** 2:9.1f} MB") +print("\nTo reuse: Save Version -> Save & Run All, then in the next session") +print("Add Data -> Your Work -> this notebook's output, mounted at") +print("/kaggle/input/docureason-artifacts. Cell 5 will restore the corpus, the indexes,") +print("the DuckDB store and the model weights, and cells 7-8 become no-ops.") +""" +) + +md( + r""" +## What this run does and does not establish + +**Established, in this session, from these filings:** every metric above, with the +interval around it and the stratum it came from; the per-stage latency decomposition +including its signed residual; each ablation's paired delta against the full system, with +the published claim printed beside it; the naive-RAG comparison, scored through the same +harness with the same generator and the same embedding model. + +**Not established:** + +- **Gold labels are derived, not annotated.** Table cells know their region by + construction and prose anchors are phrases verified unique across the corpus, but + derivable queries over-represent lookup and cannot express multi-hop reasoning. +- **The `figure` stratum is usually empty.** Docling's HTML pipeline emits no figure + regions for these filings, so the rasterization work is exercised by tests, not by this + measurement. A PDF corpus would change that. +- **The `no_layout_parser` row cannot test its own recall claim.** Its corpus renames + every gold region, so the runner reports the gold-id metrics as not comparable and + carries the row on answer quality plus a region/table census. +- **Attribution is lexical plus numeric consistency, not entailment.** `entailment_threshold` + is inert until a real NLI model is loaded, and `attribution_precision` should be read as + claim-support coverage rather than faithfulness. + +`Documentations/audit.md` is the plan of record; §13 lists what remains. +""" +) + + +def build_notebook() -> Dict[str, Any]: + cells: List[Dict[str, Any]] = [] + for kind, source in CELLS: + cell: Dict[str, Any] = { + "cell_type": kind, + "metadata": {}, + # nbformat wants a list of lines, each keeping its newline except the last. + "source": source.splitlines(keepends=True), + } + if kind == "code": + cell["execution_count"] = None + cell["outputs"] = [] + cells.append(cell) + + return { + "cells": cells, + "metadata": { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python", "version": "3.11"}, + "kaggle": {"accelerator": "gpu", "isInternetEnabled": True, "language": "python"}, + }, + "nbformat": 4, + "nbformat_minor": 5, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--out", type=Path, default=DEFAULT_OUTPUT, help=f"output path (default: {DEFAULT_OUTPUT})") + args = parser.parse_args() + + notebook = build_notebook() + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(notebook, indent=1), encoding="utf-8") + + code_cells = sum(1 for kind, _ in CELLS if kind == "code") + print(f"wrote {args.out}") + print(f" {len(CELLS)} cells ({code_cells} code, {len(CELLS) - code_cells} markdown)") + print(f" repository: {REPO_URL}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_metric_regression.py b/scripts/check_metric_regression.py new file mode 100644 index 0000000..95ffcff --- /dev/null +++ b/scripts/check_metric_regression.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Metric regression gate: fail when harness metrics move on a frozen fixture. + + python scripts/check_metric_regression.py # compare, exit 1 on drift + python scripts/check_metric_regression.py --tolerance 0 # exact match + python scripts/check_metric_regression.py --update # re-record the baseline + +359 lines of metric arithmetic - nDCG, MAP, MRR, compression, noise, attribution - +had no test that would notice if any of it changed. A refactor could shift every +reported number and CI would stay green, which is how a published benchmark table comes +to describe a system that no longer exists. + +The fixture is **frozen retrieval output**, not a live pipeline run: fixed result lists +with fixed gold ids, no models, no network, no corpus. That is deliberate. This gate +answers exactly one question - "did the metric definitions change?" - and a fixture +that re-ran retrieval would answer it only when the embedding model, the reranker and +the corpus all happened to hold still. + +Updating the baseline is a deliberate act with a diff to review. If a change moves +these numbers, the pull request should say why. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Tuple + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from docureason.tripath.evaluation.eval_harness import EvaluationHarness # noqa: E402 + +FIXTURE_PATH = ROOT / "tests" / "fixtures" / "metric_regression_fixture.json" +BASELINE_PATH = ROOT / "tests" / "fixtures" / "metric_regression_baseline.json" + +#: Default drift allowed per metric. 0.01 is one percentage point: tight enough to +#: catch a changed formula, loose enough to survive floating-point reassociation. +DEFAULT_TOLERANCE = 0.01 + + +def load_fixture(path: Path = FIXTURE_PATH) -> List[Dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + return payload["cases"] + + +def compute(cases: List[Dict[str, Any]], output_dir: Path) -> Dict[str, Any]: + """Score the fixture and return only the deterministic parts of the report. + + Latency and throughput are dropped: they measure the machine, and gating them + would make the build fail on a busy runner. + """ + harness = EvaluationHarness(output_dir=output_dir) + report = harness.evaluate_suite(cases) + + summary = { + key: value + for key, value in (report.get("summary_metrics") or {}).items() + if not _is_machine_dependent(key) and isinstance(value, (int, float)) + } + strata = { + name: { + key: value + for key, value in (bucket.get("summary_metrics") or {}).items() + if not _is_machine_dependent(key) and isinstance(value, (int, float)) + } + for name, bucket in (report.get("per_stratum") or {}).items() + } + return { + "total_queries": report.get("total_queries"), + "summary_metrics": summary, + "per_stratum": strata, + } + + +def _is_machine_dependent(metric: str) -> bool: + return any(token in metric for token in ("latency", "qps", "duration", "per_second")) + + +def flatten(payload: Dict[str, Any], prefix: str = "") -> Dict[str, float]: + """Flatten the nested report into ``dotted.path -> value`` for comparison.""" + flat: Dict[str, float] = {} + for key, value in payload.items(): + path = f"{prefix}{key}" + if isinstance(value, dict): + flat.update(flatten(value, prefix=f"{path}.")) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + flat[path] = float(value) + return flat + + +def compare(current: Dict[str, Any], baseline: Dict[str, Any], tolerance: float) -> Tuple[bool, List[str]]: + """Return ``(ok, findings)``. A missing or added metric is drift too.""" + now = flatten(current) + before = flatten(baseline) + findings: List[str] = [] + + for name in sorted(set(before) - set(now)): + findings.append(f"REMOVED {name}: was {before[name]:.4f}, no longer reported") + for name in sorted(set(now) - set(before)): + findings.append(f"NEW {name}: now {now[name]:.4f}, not in the baseline") + + for name in sorted(set(now) & set(before)): + delta = now[name] - before[name] + if abs(delta) > tolerance: + findings.append(f"MOVED {name}: {before[name]:.4f} -> {now[name]:.4f} ({delta:+.4f})") + + return not findings, findings + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--tolerance", type=float, default=DEFAULT_TOLERANCE) + parser.add_argument("--update", action="store_true", help="overwrite the baseline with current values") + parser.add_argument("--fixture", type=Path, default=FIXTURE_PATH) + parser.add_argument("--baseline", type=Path, default=BASELINE_PATH) + args = parser.parse_args() + + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + current = compute(load_fixture(args.fixture), Path(tmp)) + + if args.update: + args.baseline.parent.mkdir(parents=True, exist_ok=True) + args.baseline.write_text(json.dumps(current, indent=2, sort_keys=True), encoding="utf-8") + print(f"Baseline rewritten: {args.baseline} ({len(flatten(current))} metrics)") + return 0 + + if not args.baseline.exists(): + print(f"No baseline at {args.baseline}. Record one with --update.") + return 1 + + baseline = json.loads(args.baseline.read_text(encoding="utf-8")) + ok, findings = compare(current, baseline, args.tolerance) + + if ok: + print(f"Metric regression gate: PASS ({len(flatten(current))} metrics within +/-{args.tolerance})") + return 0 + + print(f"Metric regression gate: FAIL ({len(findings)} finding(s), tolerance +/-{args.tolerance})") + for finding in findings: + print(f" {finding}") + print("\nIf the change is intended, re-record with:") + print(" python scripts/check_metric_regression.py --update") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/plot_results.py b/scripts/plot_results.py new file mode 100644 index 0000000..e347713 --- /dev/null +++ b/scripts/plot_results.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""scripts/plot_results.py - figures and a derived-insight digest from the reports. + + python scripts/plot_results.py + +Reads ``artifacts/evaluation/eval_suite.json`` (from run_eval_suite.py) and +``artifacts/ablations/ablation_report.json`` (from run_ablations.py) and writes +``artifacts/figures/*.png`` + ``*.pdf`` plus ``INSIGHTS.md``. + +Either report may be absent: the figures it would have produced are skipped and listed in +the digest, because an axis with nothing on it reads as "measured, and flat". + +Needs the plotting extra, which the framework itself does not depend on: + + pip install 'docureason-framework[figures]' +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from docureason.tripath.evaluation.figures import DEFAULT_FORMATS, build_all # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--eval-report", default="artifacts/evaluation/eval_suite.json") + parser.add_argument("--ablation-report", default="artifacts/ablations/ablation_report.json") + parser.add_argument("--out", default="artifacts/figures", help="where to write figures and INSIGHTS.md") + parser.add_argument( + "--formats", + default=",".join(DEFAULT_FORMATS), + help="comma-separated output formats (default: png,pdf - raster for notebooks, vector for papers)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + formats = tuple(part.strip() for part in args.formats.split(",") if part.strip()) + + result = build_all( + eval_report=args.eval_report, + ablation_report=args.ablation_report, + out_dir=args.out, + formats=formats, + ) + + print("") + for path in result["figures"]: + print(f" wrote {path}") + for item in result["skipped"]: + print(f" skipped {item}") + print("") + print(f"insights: {result['insights']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_ablations.py b/scripts/run_ablations.py new file mode 100644 index 0000000..21cfd80 --- /dev/null +++ b/scripts/run_ablations.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Run the seven-row ablation matrix, and the naive-RAG baseline, for real. + + python scripts/run_ablations.py --queries data/eval/queries.json --limit 20 + +Every ablation re-runs **every** query with one component disabled, scores both runs +through the same harness, and reports the difference as a paired comparison with a +bootstrap interval and a sign-flip p-value. + +Cost: (1 + ablations) x queries full pipeline executions. Seven ablations over 200 +queries is 1,600 executions, which on CPU is hours - so ``--limit`` exists, and a run +that used it says so in the artifact rather than presenting a subset as the matrix. + +``Documentations/eval_method.md`` §21 publishes a delta for each row. Those numbers +predate any runner that could produce them; the report prints measured against claimed +so an unsupported claim is visible instead of inherited. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, List + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from docureason.tripath.evaluation.ablation import ( # noqa: E402 + ABLATIONS, + ABLATIONS_BY_KEY, + AblationRunner, + markdown_matrix, +) +from docureason.tripath.evaluation.baselines import NaiveRAGBaseline # noqa: E402 +from docureason.tripath.evaluation.eval_harness import EvaluationHarness # noqa: E402 +from docureason.tripath.evaluation.gold_queries import load_eval_cases # noqa: E402 +from docureason.tripath.serving.query_service import QueryService # noqa: E402 +from docureason.tripath.utils import get_logger # noqa: E402 + +logger = get_logger("run_ablations") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--queries", default="data/eval/queries.json", help="query set from build_eval_corpus.py") + parser.add_argument("--filings", default="data/eval/filings", help="corpus input directory") + parser.add_argument("--corpus", default="data/eval/corpus", help="ingested corpus / index directory") + parser.add_argument("--out", default="artifacts/ablations", help="where to write the report") + parser.add_argument("--limit", type=int, default=0, help="use only the first N queries (0 = all)") + parser.add_argument("--stratified", action="store_true", help="sample --limit evenly across strata") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--iterations", type=int, default=2000, help="bootstrap resamples (default: 2000)") + parser.add_argument("--only", nargs="*", default=None, help="run only these ablation keys") + parser.add_argument( + "--ingestion-corpus", + action="append", + default=[], + metavar="KEY=PATH", + help=( + "corpus for an ingestion-scoped ablation, repeatable, e.g. " + "--ingestion-corpus no_layout_parser=data/eval/corpus_no_layout. Without one, that " + "row is reported as not executed: the component runs once at ingestion, so it " + "cannot be switched off per query. Build the corpus with build_eval_corpus.py " + "--corpus-dir --no-layout-parser --skip-derive" + ), + ) + parser.add_argument("--no-baseline", action="store_true", help="skip the naive-RAG comparison") + return parser.parse_args() + + +def load_cases(path: Path, limit: int, stratified: bool) -> List[Dict[str, Any]]: + """Thin CLI wrapper: the loader is shared with scripts/run_eval_suite.py so the two + commands cannot end up sampling different query sets under the same corpus name.""" + try: + return load_eval_cases(path, limit=limit, stratified=stratified) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + + +def matrix_document(report: Dict[str, Any], matrix: str) -> str: + """The matrix table, plus the provenance a reader needs in order to weigh it. + + Which query set, how many queries, and how many rows actually executed. Without those + three facts a three-query subset over a fixture corpus renders identically to the full + matrix over real filings - and the table is the part that gets quoted. + """ + subset = report.get("query_subset") or {} + rows = report.get("ablations") or [] + unexecuted = report.get("not_executed") or [] + sample_note = " (stratified sample)" if subset.get("stratified_sample") else "" + + document = ["# Measured ablation matrix", ""] + if not subset.get("is_full_matrix"): + document += ["**Subset run - this is not the full matrix.**", ""] + document += [ + f"- queries: **{report.get('queries')}** from `{subset.get('source')}`{sample_note}", + f"- rows executed: **{report.get('executed')} of {len(rows)}**" + + ("" if not unexecuted else ", not executed: " + ", ".join(unexecuted)), + f"- bootstrap iterations: {report.get('bootstrap_iterations')}, seed {report.get('seed')}", + f"- ingestion-scoped corpora: {report.get('ingestion_corpora') or 'none supplied'}", + "", + matrix, + "", + ] + return "\n".join(document) + + +def main() -> int: + args = parse_args() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + cases = load_cases(Path(args.queries), args.limit, args.stratified) + logger.info("Loaded %d evaluation case(s)", len(cases)) + + # Validated before anything expensive is constructed: loading the corpus and building + # the baseline index takes minutes, and a typo in a path is not worth waiting for. + ingestion_corpora: Dict[str, Path] = {} + for spec in args.ingestion_corpus: + key, separator, raw_path = spec.partition("=") + key, raw_path = key.strip(), raw_path.strip() + if not separator or not key or not raw_path: + raise SystemExit(f"--ingestion-corpus expects KEY=PATH, got {spec!r}") + if key not in ABLATIONS_BY_KEY: + raise SystemExit(f"unknown ablation key {key!r}; known keys: {sorted(ABLATIONS_BY_KEY)}") + corpus_path = Path(raw_path) + if not (corpus_path / "corpus.json").exists(): + raise SystemExit(f"no corpus.json under {corpus_path} for ablation {key!r}") + ingestion_corpora[key] = corpus_path + + service = QueryService(input_dir=args.filings, output_dir=args.corpus, verbose=False) + harness = EvaluationHarness(output_dir=out) + + ablations = list(ABLATIONS) + if args.only: + wanted = set(args.only) + ablations = [ablation for ablation in ablations if ablation.key in wanted] + if not ablations: + raise SystemExit(f"no ablations matched {sorted(wanted)}") + + baseline = None + if not args.no_baseline: + baseline = NaiveRAGBaseline(config=service.config, generator=service.generator) + stats = baseline.build(service.documents) + logger.info("Naive baseline: %s", stats) + + # Ingestion-scoped rows answer from a second corpus, built with the component off. Its + # gold ids are the un-ablated corpus's, so the runner reports the retrieval metrics as + # not comparable and carries the row on answer quality plus the corpus census. + ingestion_services: Dict[str, Any] = {} + for key, corpus_path in ingestion_corpora.items(): + logger.info("Ablation %s will answer from corpus %s", key, corpus_path) + ingestion_services[key] = QueryService(input_dir=args.filings, output_dir=str(corpus_path), verbose=False) + + runner = AblationRunner( + service=service, + harness=harness, + ablations=ablations, + iterations=args.iterations, + seed=args.seed, + ingestion_services=ingestion_services, + ) + report = runner.run(cases, baseline=baseline) + report["query_subset"] = { + "requested_limit": args.limit, + "stratified_sample": bool(args.stratified), + "queries_used": len(cases), + "source": str(args.queries), + # Stated in the artifact because a subset run is not the matrix, and a reader + # of the JSON has no other way to tell the two apart. An unexecuted row + # disqualifies the run too: six rows of seven is not the matrix either. + "is_full_matrix": args.limit == 0 and not args.only and not report["not_executed"], + } + report["ingestion_corpora"] = {key: str(path) for key, path in ingestion_corpora.items()} + + report_path = out / "ablation_report.json" + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + + matrix = markdown_matrix(report) + matrix_path = out / "ablation_matrix.md" + matrix_path.write_text(matrix_document(report, matrix), encoding="utf-8") + + print("") + print(matrix) + print("") + if report.get("baseline"): + print("Full system vs naive RAG baseline:") + for metric, comparison in report["baseline"]["full_vs_baseline"].items(): + delta = comparison.get("delta") + low, high = comparison.get("delta_ci_low"), comparison.get("delta_ci_high") + p_value = comparison.get("p_value") + if delta is None: + continue + print( + f" {metric:24s} {delta:+.4f} 95% CI [{low:+.4f}, {high:+.4f}] " + f"p={p_value if p_value is None else round(p_value, 4)}" + ) + print("") + print(f"not executed: {report['not_executed'] or 'none'}") + print(f"report: {report_path}") + print(f"matrix: {matrix_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_eval_suite.py b/scripts/run_eval_suite.py new file mode 100644 index 0000000..79a478d --- /dev/null +++ b/scripts/run_eval_suite.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""scripts/run_eval_suite.py - score the full system over the gold query set. + + python scripts/run_eval_suite.py --queries data/eval/queries.json + +``run_ablations.py`` measures the *difference* between systems. This measures the system +itself: one pass over every gold query, with bootstrap intervals, a per-stratum +breakdown, the per-stage latency decomposition, and a census of what degraded. +``evaluate_system.py`` scores five hand-written cases against SLA targets over +``samples/`` - useful as a smoke test, but it is not an evaluation of a corpus. + +Degradation is **counted**, not just logged. A suite whose retrieval quietly ran in +substring-scan mode scores like one that queried its indexes, and which of the two +happened is the whole question (audit T3). +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from docureason.tripath.evaluation.eval_harness import EvaluationHarness # noqa: E402 +from docureason.tripath.evaluation.gold_queries import load_eval_cases # noqa: E402 +from docureason.tripath.serving.query_service import QueryService # noqa: E402 +from docureason.tripath.utils import get_logger # noqa: E402 + +logger = get_logger("run_eval_suite") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--queries", default="data/eval/queries.json", help="query set from build_eval_corpus.py") + parser.add_argument("--filings", default="data/eval/filings", help="corpus input directory") + parser.add_argument("--corpus", default="data/eval/corpus", help="ingested corpus / index directory") + parser.add_argument("--out", default="artifacts/evaluation", help="where to write the report") + parser.add_argument("--limit", type=int, default=0, help="use only N queries (0 = all)") + parser.add_argument("--stratified", action="store_true", help="sample --limit evenly across strata") + parser.add_argument("--iterations", type=int, default=2000, help="bootstrap resamples (default: 2000)") + parser.add_argument("--run-name", default="eval_suite", help="report file stem") + return parser.parse_args() + + +def run_suite( + service: QueryService, + harness: EvaluationHarness, + cases: List[Dict[str, Any]], + iterations: int, +) -> Dict[str, Any]: + """Execute every case against *service* and score it through *harness*.""" + scored: List[Dict[str, Any]] = [] + degradation: Counter = Counter() + retrieval_modes: Counter = Counter() + failures: List[Dict[str, str]] = [] + cached_hits = 0 + + started = time.perf_counter() + for position, case in enumerate(cases, start=1): + query = str(case.get("query", "")) + query_started = time.perf_counter() + try: + response = service.query(query) + except Exception as exc: # one bad query must not void the suite + logger.error("Query %d/%d failed: %s", position, len(cases), exc) + failures.append({"query": query, "error": repr(exc)}) + continue + latency_ms = (time.perf_counter() - query_started) * 1000.0 + + for label in response.get("degradation") or []: + degradation[str(label)] += 1 + # The per-path mode lives under retrieval_stats["modality_modes"], each value the + # path's own stats dict. index_backed vs substring_scan is the difference between + # querying the indexes the framework builds and scanning strings past them. + for modality, stats in ((response.get("retrieval_stats") or {}).get("modality_modes") or {}).items(): + mode = stats.get("mode") if isinstance(stats, dict) else stats + if mode: + retrieval_modes[f"{modality}:{mode}"] += 1 + cached_hits += 1 if response.get("cached") else 0 + + scored.append( + harness.evaluate_single( + query=query, + results=response.get("results", []), + relevant_ids=case.get("relevant_ids", []), + answer=response.get("answer"), + ground_truth_answer=case.get("ground_truth_answer"), + evidence=response.get("retrieved_evidence"), + sql_executed=(response.get("sql_execution") or {}).get("executed"), + latency_ms=round(latency_ms, 3), + stratum=case.get("stratum"), + latency_breakdown=response.get("latency_breakdown"), + ) + ) + if position % 10 == 0: + logger.info("Scored %d/%d queries", position, len(cases)) + + latencies = [record.get("latency_ms", 0.0) for record in scored] + return { + "queries_requested": len(cases), + "queries_scored": len(scored), + "failures": failures, + "wall_clock_sec": round(time.perf_counter() - started, 2), + "summary_metrics": harness._aggregate_summary(scored, latencies), + "confidence_intervals": harness.confidence_intervals(scored, iterations=iterations), + "stage_latency_ms": harness.stage_latency_summary(scored), + "per_stratum": harness.stratified_summary(scored, iterations=iterations), + # Counted, not logged: a substring-scan run and an index-backed run score alike. + "degradation_counts": dict(degradation.most_common()), + "retrieval_modes": dict(retrieval_modes.most_common()), + "cache_hits": cached_hits, + "per_query": scored, + } + + +#: Summary key -> the per-case metric whose bootstrap interval belongs beside it. The +#: harness aggregates under ``average_*`` but computes intervals under the raw per-case +#: name, so a naive lookup silently prints every interval as blank. +_CI_ALIASES = { + "average_recall_at_5": "recall_at_5", + "average_ndcg_at_5": "ndcg_at_5", + "mean_latency_ms": "latency_ms", +} + + +def _cell(metrics: Dict[str, Any], key: str) -> str: + value = metrics.get(key) + return "n/a" if not isinstance(value, (int, float)) else f"{value:.4f}" + + +def print_report(report: Dict[str, Any]) -> None: + summary = report["summary_metrics"] + intervals = report.get("confidence_intervals") or {} + + print("") + print(f"Scored {report['queries_scored']}/{report['queries_requested']} queries in {report['wall_clock_sec']}s") + if report["failures"]: + print(f" {len(report['failures'])} query(ies) failed - listed under 'failures' in the report") + print("") + print(f" {'metric':30s} {'value':>12s} 95% CI") + for metric in sorted(summary): + value = summary[metric] + if not isinstance(value, (int, float)): + continue + interval = intervals.get(_CI_ALIASES.get(metric, metric)) or {} + low, high = interval.get("ci_low"), interval.get("ci_high") + span = "" if low is None or high is None else f"[{low:.4f}, {high:.4f}]" + print(f" {metric:30s} {value:12.4f} {span}") + + per_stratum = report.get("per_stratum") or {} + if per_stratum: + print("") + print(" per stratum - a corpus-wide mean can hold while one stratum collapses:") + for stratum in sorted(per_stratum): + block = per_stratum[stratum] or {} + metrics = block.get("summary_metrics") or {} + print( + f" {stratum:18s} n={block.get('n', '?'):<5} " + f"recall@5={_cell(metrics, 'average_recall_at_5')} " + f"ndcg@5={_cell(metrics, 'average_ndcg_at_5')} " + f"token_f1={_cell(metrics, 'token_f1')}" + ) + + stages = report.get("stage_latency_ms") or {} + rows = [ + (stage, block["mean_ms"], block.get("p95_ms")) + for stage, block in stages.items() + if isinstance(block, dict) and isinstance(block.get("mean_ms"), (int, float)) + ] + if rows: + print("") + print(f" per-stage latency over {stages.get('cases_with_breakdown', '?')} uncached case(s), ms:") + for stage, mean_ms, p95_ms in sorted(rows, key=lambda row: -row[1]): + print(f" {stage:20s} mean={mean_ms:10.2f} p95={p95_ms if p95_ms is None else round(p95_ms, 2)}") + + if report.get("retrieval_modes"): + print("") + print(" retrieval mode (index_backed vs substring_scan is the whole question):") + for label, count in report["retrieval_modes"].items(): + print(f" {label:44s} {count}") + + print("") + if report.get("degradation_counts"): + print(" degradation (label: queries affected):") + for label, count in report["degradation_counts"].items(): + print(f" {label:44s} {count}") + else: + print(" degradation: none reported on any query") + + +def main() -> int: + args = parse_args() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + try: + cases = load_eval_cases(args.queries, limit=args.limit, stratified=args.stratified) + except (OSError, ValueError) as exc: + raise SystemExit(str(exc)) from exc + logger.info("Loaded %d evaluation case(s) from %s", len(cases), args.queries) + + service = QueryService(input_dir=args.filings, output_dir=args.corpus, verbose=False) + harness = EvaluationHarness(output_dir=out) + + report = run_suite(service, harness, cases, iterations=args.iterations) + report["corpus"] = {"filings": str(args.filings), "corpus": str(args.corpus), "queries": str(args.queries)} + report["query_subset"] = { + "requested_limit": args.limit, + "stratified_sample": bool(args.stratified), + "is_full_set": args.limit == 0, + } + report["bootstrap_iterations"] = args.iterations + + path = out / f"{args.run_name}.json" + path.write_text(json.dumps(report, indent=2), encoding="utf-8") + + print_report(report) + print("") + print(f"report: {path}") + return 0 if report["queries_scored"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/metric_regression_baseline.json b/tests/fixtures/metric_regression_baseline.json new file mode 100644 index 0000000..7523f65 --- /dev/null +++ b/tests/fixtures/metric_regression_baseline.json @@ -0,0 +1,113 @@ +{ + "per_stratum": { + "figure": { + "attribution_precision": 0.0, + "average_hit_rate_at_5": 0.0, + "average_ndcg_at_10": 0.0, + "average_ndcg_at_5": 0.0, + "average_precision_at_5": 0.0, + "average_recall_at_10": 0.0, + "average_recall_at_5": 0.0, + "context_compression_ratio": 0.0, + "duplicate_rank_ratio": 0.8, + "est_cost_per_1k_queries_usd": 0.0593, + "exact_match": 0.0, + "hallucination_rate": 1.0, + "map_at_10": 0.0, + "mrr": 0.0, + "noise_ratio": 1.0, + "sql_execution_success_rate": 0.0, + "token_f1": 0.4444, + "unique_evidence_ratio": 0.2 + }, + "numeric": { + "attribution_precision": 0.0, + "average_hit_rate_at_5": 1.0, + "average_ndcg_at_10": 0.6309, + "average_ndcg_at_5": 0.6309, + "average_precision_at_5": 0.2, + "average_recall_at_10": 1.0, + "average_recall_at_5": 1.0, + "avg_gold_rank": 2.0, + "context_compression_ratio": 0.2, + "duplicate_rank_ratio": 0.8, + "est_cost_per_1k_queries_usd": 0.0593, + "exact_match": 0.0, + "gold_rank_variance": 0.0, + "hallucination_rate": 1.0, + "map_at_10": 0.5, + "mrr": 0.5, + "noise_ratio": 0.8, + "sql_execution_success_rate": 1.0, + "token_f1": 0.4444, + "unique_evidence_ratio": 0.2 + }, + "prose": { + "attribution_precision": 0.0, + "average_hit_rate_at_5": 1.0, + "average_ndcg_at_10": 0.4307, + "average_ndcg_at_5": 0.4307, + "average_precision_at_5": 0.2, + "average_recall_at_10": 1.0, + "average_recall_at_5": 1.0, + "avg_gold_rank": 4.0, + "context_compression_ratio": 0.2, + "duplicate_rank_ratio": 0.8, + "est_cost_per_1k_queries_usd": 0.0593, + "exact_match": 0.0, + "gold_rank_variance": 0.0, + "hallucination_rate": 1.0, + "map_at_10": 0.25, + "mrr": 0.25, + "noise_ratio": 0.8, + "sql_execution_success_rate": 0.0, + "token_f1": 0.4444, + "unique_evidence_ratio": 0.2 + }, + "tabular_lookup": { + "attribution_precision": 0.0, + "average_hit_rate_at_5": 1.0, + "average_ndcg_at_10": 1.0, + "average_ndcg_at_5": 1.0, + "average_precision_at_5": 0.2, + "average_recall_at_10": 1.0, + "average_recall_at_5": 1.0, + "avg_gold_rank": 1.0, + "context_compression_ratio": 0.2, + "duplicate_rank_ratio": 0.8, + "est_cost_per_1k_queries_usd": 0.0593, + "exact_match": 0.0, + "gold_rank_variance": 0.0, + "hallucination_rate": 1.0, + "map_at_10": 1.0, + "mrr": 1.0, + "noise_ratio": 0.8, + "sql_execution_success_rate": 1.0, + "token_f1": 0.4444, + "unique_evidence_ratio": 0.2 + } + }, + "summary_metrics": { + "attribution_precision": 0.0, + "average_hit_rate_at_5": 0.75, + "average_ndcg_at_10": 0.5154, + "average_ndcg_at_5": 0.5154, + "average_precision_at_5": 0.15, + "average_recall_at_10": 0.75, + "average_recall_at_5": 0.75, + "avg_gold_rank": 2.3333, + "context_compression_ratio": 0.15, + "duplicate_rank_ratio": 0.8, + "est_cost_per_1k_queries_usd": 0.0593, + "exact_match": 0.0, + "gold_rank_variance": 0.0, + "hallucination_rate": 1.0, + "map_at_10": 0.4375, + "mrr": 0.4375, + "noise_ratio": 0.85, + "sql_execution_success_rate": 0.5, + "token_f1": 0.4444, + "unique_evidence_ratio": 0.2 + }, + "total_queries": 20 +} \ No newline at end of file diff --git a/tests/fixtures/metric_regression_fixture.json b/tests/fixtures/metric_regression_fixture.json new file mode 100644 index 0000000..103079b --- /dev/null +++ b/tests/fixtures/metric_regression_fixture.json @@ -0,0 +1,1088 @@ +{ + "fixture": "metric-regression", + "note": "Frozen retrieval output, not a pipeline run. Regenerating this file changes what the gate measures; the baseline must be re-recorded alongside it.", + "seed": 20260901, + "case_count": 20, + "cases": [ + { + "id": "fx-000", + "stratum": "tabular_lookup", + "query": "What operating revenue is reported in tabular_lookup case 0?", + "relevant_ids": [ + "doc-fixture00-region-0" + ], + "results": [ + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-0", + "text": "Segment 0 of tabular_lookup evidence discussing operating revenue of $1000 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-91", + "text": "Segment 1 of tabular_lookup evidence discussing operating revenue of $1000 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-92", + "text": "Segment 2 of tabular_lookup evidence discussing operating revenue of $1000 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-93", + "text": "Segment 3 of tabular_lookup evidence discussing operating revenue of $1000 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-94", + "text": "Segment 4 of tabular_lookup evidence discussing operating revenue of $1000 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1000 thousand [Source: doc-fixture00].", + "ground_truth_answer": "$1000 thousand", + "sql_executed": true, + "latency_ms": 100.0 + }, + { + "id": "fx-001", + "stratum": "numeric", + "query": "What operating revenue is reported in numeric case 1?", + "relevant_ids": [ + "doc-fixture00-region-1" + ], + "results": [ + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-90", + "text": "Segment 0 of numeric evidence discussing operating revenue of $1037 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-1", + "text": "Segment 1 of numeric evidence discussing operating revenue of $1037 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-92", + "text": "Segment 2 of numeric evidence discussing operating revenue of $1037 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-93", + "text": "Segment 3 of numeric evidence discussing operating revenue of $1037 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-94", + "text": "Segment 4 of numeric evidence discussing operating revenue of $1037 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1037 thousand [Source: doc-fixture00].", + "ground_truth_answer": "$1037 thousand", + "sql_executed": true, + "latency_ms": 101.0 + }, + { + "id": "fx-002", + "stratum": "prose", + "query": "What operating revenue is reported in prose case 2?", + "relevant_ids": [ + "doc-fixture00-region-2" + ], + "results": [ + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-90", + "text": "Segment 0 of prose evidence discussing operating revenue of $1074 thousand.", + "score": 1.0, + "modality": "text", + "rank": 1 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-91", + "text": "Segment 1 of prose evidence discussing operating revenue of $1074 thousand.", + "score": 0.9, + "modality": "text", + "rank": 2 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-92", + "text": "Segment 2 of prose evidence discussing operating revenue of $1074 thousand.", + "score": 0.8, + "modality": "text", + "rank": 3 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-2", + "text": "Segment 3 of prose evidence discussing operating revenue of $1074 thousand.", + "score": 0.7, + "modality": "text", + "rank": 4 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-94", + "text": "Segment 4 of prose evidence discussing operating revenue of $1074 thousand.", + "score": 0.6, + "modality": "text", + "rank": 5 + } + ], + "answer": "Operating revenue was $1074 thousand [Source: doc-fixture00].", + "ground_truth_answer": "$1074 thousand", + "sql_executed": false, + "latency_ms": 102.0 + }, + { + "id": "fx-003", + "stratum": "figure", + "query": "What operating revenue is reported in figure case 3?", + "relevant_ids": [ + "doc-fixture00-region-3" + ], + "results": [ + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-90", + "text": "Segment 0 of figure evidence discussing operating revenue of $1111 thousand.", + "score": 1.0, + "modality": "vision", + "rank": 1 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-91", + "text": "Segment 1 of figure evidence discussing operating revenue of $1111 thousand.", + "score": 0.9, + "modality": "vision", + "rank": 2 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-92", + "text": "Segment 2 of figure evidence discussing operating revenue of $1111 thousand.", + "score": 0.8, + "modality": "vision", + "rank": 3 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-93", + "text": "Segment 3 of figure evidence discussing operating revenue of $1111 thousand.", + "score": 0.7, + "modality": "vision", + "rank": 4 + }, + { + "document_id": "doc-fixture00", + "region_id": "doc-fixture00-region-94", + "text": "Segment 4 of figure evidence discussing operating revenue of $1111 thousand.", + "score": 0.6, + "modality": "vision", + "rank": 5 + } + ], + "answer": "Operating revenue was $1111 thousand [Source: doc-fixture00].", + "ground_truth_answer": "$1111 thousand", + "sql_executed": false, + "latency_ms": 103.0 + }, + { + "id": "fx-004", + "stratum": "tabular_lookup", + "query": "What operating revenue is reported in tabular_lookup case 4?", + "relevant_ids": [ + "doc-fixture01-region-0" + ], + "results": [ + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-0", + "text": "Segment 0 of tabular_lookup evidence discussing operating revenue of $1148 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-91", + "text": "Segment 1 of tabular_lookup evidence discussing operating revenue of $1148 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-92", + "text": "Segment 2 of tabular_lookup evidence discussing operating revenue of $1148 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-93", + "text": "Segment 3 of tabular_lookup evidence discussing operating revenue of $1148 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-94", + "text": "Segment 4 of tabular_lookup evidence discussing operating revenue of $1148 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1148 thousand [Source: doc-fixture01].", + "ground_truth_answer": "$1148 thousand", + "sql_executed": true, + "latency_ms": 104.0 + }, + { + "id": "fx-005", + "stratum": "numeric", + "query": "What operating revenue is reported in numeric case 5?", + "relevant_ids": [ + "doc-fixture01-region-1" + ], + "results": [ + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-90", + "text": "Segment 0 of numeric evidence discussing operating revenue of $1185 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-1", + "text": "Segment 1 of numeric evidence discussing operating revenue of $1185 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-92", + "text": "Segment 2 of numeric evidence discussing operating revenue of $1185 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-93", + "text": "Segment 3 of numeric evidence discussing operating revenue of $1185 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-94", + "text": "Segment 4 of numeric evidence discussing operating revenue of $1185 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1185 thousand [Source: doc-fixture01].", + "ground_truth_answer": "$1185 thousand", + "sql_executed": true, + "latency_ms": 105.0 + }, + { + "id": "fx-006", + "stratum": "prose", + "query": "What operating revenue is reported in prose case 6?", + "relevant_ids": [ + "doc-fixture01-region-2" + ], + "results": [ + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-90", + "text": "Segment 0 of prose evidence discussing operating revenue of $1222 thousand.", + "score": 1.0, + "modality": "text", + "rank": 1 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-91", + "text": "Segment 1 of prose evidence discussing operating revenue of $1222 thousand.", + "score": 0.9, + "modality": "text", + "rank": 2 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-92", + "text": "Segment 2 of prose evidence discussing operating revenue of $1222 thousand.", + "score": 0.8, + "modality": "text", + "rank": 3 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-2", + "text": "Segment 3 of prose evidence discussing operating revenue of $1222 thousand.", + "score": 0.7, + "modality": "text", + "rank": 4 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-94", + "text": "Segment 4 of prose evidence discussing operating revenue of $1222 thousand.", + "score": 0.6, + "modality": "text", + "rank": 5 + } + ], + "answer": "Operating revenue was $1222 thousand [Source: doc-fixture01].", + "ground_truth_answer": "$1222 thousand", + "sql_executed": false, + "latency_ms": 106.0 + }, + { + "id": "fx-007", + "stratum": "figure", + "query": "What operating revenue is reported in figure case 7?", + "relevant_ids": [ + "doc-fixture01-region-3" + ], + "results": [ + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-90", + "text": "Segment 0 of figure evidence discussing operating revenue of $1259 thousand.", + "score": 1.0, + "modality": "vision", + "rank": 1 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-91", + "text": "Segment 1 of figure evidence discussing operating revenue of $1259 thousand.", + "score": 0.9, + "modality": "vision", + "rank": 2 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-92", + "text": "Segment 2 of figure evidence discussing operating revenue of $1259 thousand.", + "score": 0.8, + "modality": "vision", + "rank": 3 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-93", + "text": "Segment 3 of figure evidence discussing operating revenue of $1259 thousand.", + "score": 0.7, + "modality": "vision", + "rank": 4 + }, + { + "document_id": "doc-fixture01", + "region_id": "doc-fixture01-region-94", + "text": "Segment 4 of figure evidence discussing operating revenue of $1259 thousand.", + "score": 0.6, + "modality": "vision", + "rank": 5 + } + ], + "answer": "Operating revenue was $1259 thousand [Source: doc-fixture01].", + "ground_truth_answer": "$1259 thousand", + "sql_executed": false, + "latency_ms": 107.0 + }, + { + "id": "fx-008", + "stratum": "tabular_lookup", + "query": "What operating revenue is reported in tabular_lookup case 8?", + "relevant_ids": [ + "doc-fixture02-region-0" + ], + "results": [ + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-0", + "text": "Segment 0 of tabular_lookup evidence discussing operating revenue of $1296 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-91", + "text": "Segment 1 of tabular_lookup evidence discussing operating revenue of $1296 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-92", + "text": "Segment 2 of tabular_lookup evidence discussing operating revenue of $1296 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-93", + "text": "Segment 3 of tabular_lookup evidence discussing operating revenue of $1296 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-94", + "text": "Segment 4 of tabular_lookup evidence discussing operating revenue of $1296 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1296 thousand [Source: doc-fixture02].", + "ground_truth_answer": "$1296 thousand", + "sql_executed": true, + "latency_ms": 108.0 + }, + { + "id": "fx-009", + "stratum": "numeric", + "query": "What operating revenue is reported in numeric case 9?", + "relevant_ids": [ + "doc-fixture02-region-1" + ], + "results": [ + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-90", + "text": "Segment 0 of numeric evidence discussing operating revenue of $1333 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-1", + "text": "Segment 1 of numeric evidence discussing operating revenue of $1333 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-92", + "text": "Segment 2 of numeric evidence discussing operating revenue of $1333 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-93", + "text": "Segment 3 of numeric evidence discussing operating revenue of $1333 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-94", + "text": "Segment 4 of numeric evidence discussing operating revenue of $1333 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1333 thousand [Source: doc-fixture02].", + "ground_truth_answer": "$1333 thousand", + "sql_executed": true, + "latency_ms": 109.0 + }, + { + "id": "fx-010", + "stratum": "prose", + "query": "What operating revenue is reported in prose case 10?", + "relevant_ids": [ + "doc-fixture02-region-2" + ], + "results": [ + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-90", + "text": "Segment 0 of prose evidence discussing operating revenue of $1370 thousand.", + "score": 1.0, + "modality": "text", + "rank": 1 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-91", + "text": "Segment 1 of prose evidence discussing operating revenue of $1370 thousand.", + "score": 0.9, + "modality": "text", + "rank": 2 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-92", + "text": "Segment 2 of prose evidence discussing operating revenue of $1370 thousand.", + "score": 0.8, + "modality": "text", + "rank": 3 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-2", + "text": "Segment 3 of prose evidence discussing operating revenue of $1370 thousand.", + "score": 0.7, + "modality": "text", + "rank": 4 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-94", + "text": "Segment 4 of prose evidence discussing operating revenue of $1370 thousand.", + "score": 0.6, + "modality": "text", + "rank": 5 + } + ], + "answer": "Operating revenue was $1370 thousand [Source: doc-fixture02].", + "ground_truth_answer": "$1370 thousand", + "sql_executed": false, + "latency_ms": 110.0 + }, + { + "id": "fx-011", + "stratum": "figure", + "query": "What operating revenue is reported in figure case 11?", + "relevant_ids": [ + "doc-fixture02-region-3" + ], + "results": [ + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-90", + "text": "Segment 0 of figure evidence discussing operating revenue of $1407 thousand.", + "score": 1.0, + "modality": "vision", + "rank": 1 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-91", + "text": "Segment 1 of figure evidence discussing operating revenue of $1407 thousand.", + "score": 0.9, + "modality": "vision", + "rank": 2 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-92", + "text": "Segment 2 of figure evidence discussing operating revenue of $1407 thousand.", + "score": 0.8, + "modality": "vision", + "rank": 3 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-93", + "text": "Segment 3 of figure evidence discussing operating revenue of $1407 thousand.", + "score": 0.7, + "modality": "vision", + "rank": 4 + }, + { + "document_id": "doc-fixture02", + "region_id": "doc-fixture02-region-94", + "text": "Segment 4 of figure evidence discussing operating revenue of $1407 thousand.", + "score": 0.6, + "modality": "vision", + "rank": 5 + } + ], + "answer": "Operating revenue was $1407 thousand [Source: doc-fixture02].", + "ground_truth_answer": "$1407 thousand", + "sql_executed": false, + "latency_ms": 111.0 + }, + { + "id": "fx-012", + "stratum": "tabular_lookup", + "query": "What operating revenue is reported in tabular_lookup case 12?", + "relevant_ids": [ + "doc-fixture03-region-0" + ], + "results": [ + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-0", + "text": "Segment 0 of tabular_lookup evidence discussing operating revenue of $1444 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-91", + "text": "Segment 1 of tabular_lookup evidence discussing operating revenue of $1444 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-92", + "text": "Segment 2 of tabular_lookup evidence discussing operating revenue of $1444 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-93", + "text": "Segment 3 of tabular_lookup evidence discussing operating revenue of $1444 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-94", + "text": "Segment 4 of tabular_lookup evidence discussing operating revenue of $1444 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1444 thousand [Source: doc-fixture03].", + "ground_truth_answer": "$1444 thousand", + "sql_executed": true, + "latency_ms": 112.0 + }, + { + "id": "fx-013", + "stratum": "numeric", + "query": "What operating revenue is reported in numeric case 13?", + "relevant_ids": [ + "doc-fixture03-region-1" + ], + "results": [ + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-90", + "text": "Segment 0 of numeric evidence discussing operating revenue of $1481 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-1", + "text": "Segment 1 of numeric evidence discussing operating revenue of $1481 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-92", + "text": "Segment 2 of numeric evidence discussing operating revenue of $1481 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-93", + "text": "Segment 3 of numeric evidence discussing operating revenue of $1481 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-94", + "text": "Segment 4 of numeric evidence discussing operating revenue of $1481 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1481 thousand [Source: doc-fixture03].", + "ground_truth_answer": "$1481 thousand", + "sql_executed": true, + "latency_ms": 113.0 + }, + { + "id": "fx-014", + "stratum": "prose", + "query": "What operating revenue is reported in prose case 14?", + "relevant_ids": [ + "doc-fixture03-region-2" + ], + "results": [ + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-90", + "text": "Segment 0 of prose evidence discussing operating revenue of $1518 thousand.", + "score": 1.0, + "modality": "text", + "rank": 1 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-91", + "text": "Segment 1 of prose evidence discussing operating revenue of $1518 thousand.", + "score": 0.9, + "modality": "text", + "rank": 2 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-92", + "text": "Segment 2 of prose evidence discussing operating revenue of $1518 thousand.", + "score": 0.8, + "modality": "text", + "rank": 3 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-2", + "text": "Segment 3 of prose evidence discussing operating revenue of $1518 thousand.", + "score": 0.7, + "modality": "text", + "rank": 4 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-94", + "text": "Segment 4 of prose evidence discussing operating revenue of $1518 thousand.", + "score": 0.6, + "modality": "text", + "rank": 5 + } + ], + "answer": "Operating revenue was $1518 thousand [Source: doc-fixture03].", + "ground_truth_answer": "$1518 thousand", + "sql_executed": false, + "latency_ms": 114.0 + }, + { + "id": "fx-015", + "stratum": "figure", + "query": "What operating revenue is reported in figure case 15?", + "relevant_ids": [ + "doc-fixture03-region-3" + ], + "results": [ + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-90", + "text": "Segment 0 of figure evidence discussing operating revenue of $1555 thousand.", + "score": 1.0, + "modality": "vision", + "rank": 1 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-91", + "text": "Segment 1 of figure evidence discussing operating revenue of $1555 thousand.", + "score": 0.9, + "modality": "vision", + "rank": 2 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-92", + "text": "Segment 2 of figure evidence discussing operating revenue of $1555 thousand.", + "score": 0.8, + "modality": "vision", + "rank": 3 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-93", + "text": "Segment 3 of figure evidence discussing operating revenue of $1555 thousand.", + "score": 0.7, + "modality": "vision", + "rank": 4 + }, + { + "document_id": "doc-fixture03", + "region_id": "doc-fixture03-region-94", + "text": "Segment 4 of figure evidence discussing operating revenue of $1555 thousand.", + "score": 0.6, + "modality": "vision", + "rank": 5 + } + ], + "answer": "Operating revenue was $1555 thousand [Source: doc-fixture03].", + "ground_truth_answer": "$1555 thousand", + "sql_executed": false, + "latency_ms": 115.0 + }, + { + "id": "fx-016", + "stratum": "tabular_lookup", + "query": "What operating revenue is reported in tabular_lookup case 16?", + "relevant_ids": [ + "doc-fixture04-region-0" + ], + "results": [ + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-0", + "text": "Segment 0 of tabular_lookup evidence discussing operating revenue of $1592 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-91", + "text": "Segment 1 of tabular_lookup evidence discussing operating revenue of $1592 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-92", + "text": "Segment 2 of tabular_lookup evidence discussing operating revenue of $1592 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-93", + "text": "Segment 3 of tabular_lookup evidence discussing operating revenue of $1592 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-94", + "text": "Segment 4 of tabular_lookup evidence discussing operating revenue of $1592 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1592 thousand [Source: doc-fixture04].", + "ground_truth_answer": "$1592 thousand", + "sql_executed": true, + "latency_ms": 116.0 + }, + { + "id": "fx-017", + "stratum": "numeric", + "query": "What operating revenue is reported in numeric case 17?", + "relevant_ids": [ + "doc-fixture04-region-1" + ], + "results": [ + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-90", + "text": "Segment 0 of numeric evidence discussing operating revenue of $1629 thousand.", + "score": 1.0, + "modality": "table", + "rank": 1 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-1", + "text": "Segment 1 of numeric evidence discussing operating revenue of $1629 thousand.", + "score": 0.9, + "modality": "table", + "rank": 2 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-92", + "text": "Segment 2 of numeric evidence discussing operating revenue of $1629 thousand.", + "score": 0.8, + "modality": "table", + "rank": 3 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-93", + "text": "Segment 3 of numeric evidence discussing operating revenue of $1629 thousand.", + "score": 0.7, + "modality": "table", + "rank": 4 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-94", + "text": "Segment 4 of numeric evidence discussing operating revenue of $1629 thousand.", + "score": 0.6, + "modality": "table", + "rank": 5 + } + ], + "answer": "Operating revenue was $1629 thousand [Source: doc-fixture04].", + "ground_truth_answer": "$1629 thousand", + "sql_executed": true, + "latency_ms": 117.0 + }, + { + "id": "fx-018", + "stratum": "prose", + "query": "What operating revenue is reported in prose case 18?", + "relevant_ids": [ + "doc-fixture04-region-2" + ], + "results": [ + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-90", + "text": "Segment 0 of prose evidence discussing operating revenue of $1666 thousand.", + "score": 1.0, + "modality": "text", + "rank": 1 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-91", + "text": "Segment 1 of prose evidence discussing operating revenue of $1666 thousand.", + "score": 0.9, + "modality": "text", + "rank": 2 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-92", + "text": "Segment 2 of prose evidence discussing operating revenue of $1666 thousand.", + "score": 0.8, + "modality": "text", + "rank": 3 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-2", + "text": "Segment 3 of prose evidence discussing operating revenue of $1666 thousand.", + "score": 0.7, + "modality": "text", + "rank": 4 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-94", + "text": "Segment 4 of prose evidence discussing operating revenue of $1666 thousand.", + "score": 0.6, + "modality": "text", + "rank": 5 + } + ], + "answer": "Operating revenue was $1666 thousand [Source: doc-fixture04].", + "ground_truth_answer": "$1666 thousand", + "sql_executed": false, + "latency_ms": 118.0 + }, + { + "id": "fx-019", + "stratum": "figure", + "query": "What operating revenue is reported in figure case 19?", + "relevant_ids": [ + "doc-fixture04-region-3" + ], + "results": [ + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-90", + "text": "Segment 0 of figure evidence discussing operating revenue of $1703 thousand.", + "score": 1.0, + "modality": "vision", + "rank": 1 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-91", + "text": "Segment 1 of figure evidence discussing operating revenue of $1703 thousand.", + "score": 0.9, + "modality": "vision", + "rank": 2 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-92", + "text": "Segment 2 of figure evidence discussing operating revenue of $1703 thousand.", + "score": 0.8, + "modality": "vision", + "rank": 3 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-93", + "text": "Segment 3 of figure evidence discussing operating revenue of $1703 thousand.", + "score": 0.7, + "modality": "vision", + "rank": 4 + }, + { + "document_id": "doc-fixture04", + "region_id": "doc-fixture04-region-94", + "text": "Segment 4 of figure evidence discussing operating revenue of $1703 thousand.", + "score": 0.6, + "modality": "vision", + "rank": 5 + } + ], + "answer": "Operating revenue was $1703 thousand [Source: doc-fixture04].", + "ground_truth_answer": "$1703 thousand", + "sql_executed": false, + "latency_ms": 119.0 + } + ] +} \ No newline at end of file diff --git a/tests/tripath/test_ablation_runner.py b/tests/tripath/test_ablation_runner.py new file mode 100644 index 0000000..ccc8d48 --- /dev/null +++ b/tests/tripath/test_ablation_runner.py @@ -0,0 +1,330 @@ +"""test_ablation_runner.py — the ablation matrix is executed, not asserted. + +The module this replaces subtracted one pre-computed number from another and had no +caller. These tests drive the runner with a stub service whose quality depends on the +overrides it receives, so a measured delta has a known correct value. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from docureason.tripath.evaluation.ablation import ( + ABLATIONS, + CORPUS_BOUND_METRICS, + Ablation, + AblationRunner, + AblationStudy, + corpus_signature, + markdown_matrix, +) +from docureason.tripath.evaluation.eval_harness import EvaluationHarness + +pytestmark = pytest.mark.unit + +CASES: List[Dict[str, Any]] = [ + {"query": f"question {i}", "relevant_ids": [f"doc-{i}-region-0"], "stratum": "prose" if i % 2 else "numeric"} + for i in range(12) +] + + +class StubService: + """Returns the gold region at rank 1 normally, and demotes it when ablated. + + ``demote_on`` names the override key that degrades this stub, so the runner's + measured delta for that ablation has an arithmetically known sign and size. + """ + + def __init__(self, demote_on: Optional[str] = None) -> None: + self.demote_on = demote_on + self.calls: List[Optional[Dict[str, Any]]] = [] + + def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + self.calls.append(overrides) + index = int(text.split()[-1]) + gold = f"doc-{index}-region-0" + ablated = bool(overrides and self.demote_on in overrides) + + ranked = [f"doc-{index}-region-{9 - position}" for position in range(4)] + # Gold at rank 1 when healthy, pushed off the end of the top-5 when ablated. + results = [{"region_id": gold, "text": "gold evidence", "score": 1.0}] if not ablated else [] + results += [{"region_id": rid, "text": "filler evidence", "score": 0.5} for rid in ranked] + + return { + "results": results, + "retrieved_evidence": results, + "answer": "gold evidence", + "sql_execution": {"executed": False}, + "latency_breakdown": {"stages_ms": {"rerank": 1.0}, "total_ms": 2.0}, + } + + +@pytest.fixture +def harness(tmp_path) -> EvaluationHarness: + return EvaluationHarness(output_dir=tmp_path) + + +def _runner(harness: EvaluationHarness, service: StubService, **kwargs: Any) -> AblationRunner: + return AblationRunner(service=service, harness=harness, iterations=200, seed=0, **kwargs) + + +def test_every_query_time_ablation_is_executed(harness) -> None: + report = _runner(harness, StubService()).run(CASES) + executed = {row["key"] for row in report["ablations"] if row["executed"]} + query_scoped = {ablation.key for ablation in ABLATIONS if ablation.scope == "query"} + assert executed == query_scoped + + +def test_an_ingestion_scoped_ablation_is_reported_not_dropped(harness) -> None: + """Not-measured must never look like a delta of zero.""" + report = _runner(harness, StubService()).run(CASES) + row = next(item for item in report["ablations"] if item["key"] == "no_layout_parser") + + assert row["executed"] is False + assert "re-ingested" in row["reason"] + assert "comparisons" not in row + assert report["not_executed"] == ["no_layout_parser"] + + +class CorpusService: + """A service answering out of its own corpus: its own region ids, its own answers. + + Stands in for the corpus rebuilt with the layout parser off. The gold ids belong to + the un-ablated corpus, so a different ``region_prefix`` is what a corpus rebuild + actually does to them - it does not degrade retrieval, it renames the labels. + """ + + def __init__(self, region_prefix: str, answer: str) -> None: + self.region_prefix = region_prefix + self.answer = answer + + def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + index = int(text.split()[-1]) + results = [{"region_id": f"{self.region_prefix}-{index}-region-0", "text": "evidence", "score": 1.0}] + return {"results": results, "retrieved_evidence": results, "answer": self.answer} + + +ANSWER_CASES: List[Dict[str, Any]] = [ + { + "query": f"question {i}", + "relevant_ids": [f"doc-{i}-region-0"], + "ground_truth_answer": "net revenue rose", + "stratum": "prose", + } + for i in range(12) +] + + +def _cross_corpus_report(harness: EvaluationHarness) -> Dict[str, Any]: + runner = AblationRunner( + service=CorpusService("doc", "net revenue rose"), + harness=harness, + iterations=200, + seed=0, + ingestion_services={"no_layout_parser": CorpusService("nolayout", "unrelated filler text")}, + ) + return runner.run(ANSWER_CASES) + + +def test_an_ingestion_scoped_ablation_runs_when_given_its_own_corpus(harness) -> None: + report = _cross_corpus_report(harness) + row = next(item for item in report["ablations"] if item["key"] == "no_layout_parser") + + assert row["executed"] is True + assert row["cross_corpus"] is True + assert report["not_executed"] == [] + + +def test_gold_id_metrics_are_reported_not_comparable_across_a_corpus_rebuild(harness) -> None: + """A -100% recall that measures the label space is worse than no number at all.""" + report = _cross_corpus_report(harness) + row = next(item for item in report["ablations"] if item["key"] == "no_layout_parser") + + for metric in CORPUS_BOUND_METRICS: + assert row["comparisons"][metric]["comparable"] is False, metric + assert "region identity" in row["comparisons"][metric]["reason"] + assert "delta" not in row["comparisons"][metric] + + # recall_at_5 is this row's documented primary metric, so the verdict has to say the + # claim was not tested rather than inherit a verdict from a number it cannot produce. + assert row["primary_metric"] in CORPUS_BOUND_METRICS + assert row["claim_supported"] == "not_comparable_across_corpora" + + +def test_a_cross_corpus_row_still_measures_answer_quality(harness) -> None: + report = _cross_corpus_report(harness) + row = next(item for item in report["ablations"] if item["key"] == "no_layout_parser") + + assert row["answer_metric"] == "token_f1" + assert row["comparisons"]["token_f1"]["mean_full"] == pytest.approx(1.0) + assert row["comparisons"]["token_f1"]["mean_ablated"] == pytest.approx(0.0) + assert row["measured_answer_delta_pct"] == pytest.approx(-100.0) + + table = markdown_matrix(report) + assert "token_f1 -100.0%" in table + assert "not_comparable_across_corpora" in table + + +def test_a_cross_corpus_row_falls_back_to_the_corpus_census() -> None: + """With answer quality flat on both sides, "62 tables became 0" is the finding.""" + report = { + "ablations": [ + { + "key": "no_layout_parser", + "label": "Remove Docling layout parser", + "primary_metric": "recall_at_5", + "documented_claim_pct": -42.5, + "executed": True, + "cross_corpus": True, + "answer_metric": "token_f1", + "measured_answer_delta_pct": None, + "claim_supported": "not_comparable_across_corpora", + "comparisons": { + "recall_at_5": {"comparable": False, "reason": "label space differs"}, + "token_f1": {"delta": 0.0, "p_value": 1.0}, + }, + "corpora": { + "full": {"total_regions": 1169, "structured_tables_serialized": 62}, + "ablated": {"total_regions": 6, "structured_tables_serialized": 0}, + }, + } + ] + } + table = markdown_matrix(report) + + assert "tables 62->0, regions 1169->6" in table + assert "-42.5%" in table, "the published claim stays beside a measurement it cannot be compared with" + assert "not_comparable_across_corpora" in table + + +def test_a_corpus_signature_is_absent_rather_than_invented(harness) -> None: + """A stub has no quality_audit.json; the row must say so, not fail the sweep.""" + report = _cross_corpus_report(harness) + row = next(item for item in report["ablations"] if item["key"] == "no_layout_parser") + + assert row["corpora"] == {"full": {}, "ablated": {}} + assert corpus_signature(CorpusService("doc", "answer")) == {} + + +def test_the_runner_passes_each_ablation_its_own_overrides(harness) -> None: + service = StubService() + _runner(harness, service).run(CASES) + seen = [call for call in service.calls if call] + assert {"enable_cross_encoder": False} in seen + assert {"enable_sparse_retrieval": False} in seen + assert None in service.calls, "the full system must run without overrides" + + +def test_a_real_degradation_is_measured_as_a_negative_delta(harness) -> None: + service = StubService(demote_on="enable_sparse_retrieval") + report = _runner(harness, service).run(CASES) + row = next(item for item in report["ablations"] if item["key"] == "no_sparse_path") + recall = row["comparisons"]["recall_at_5"] + + assert recall["mean_full"] == pytest.approx(1.0) + assert recall["mean_ablated"] == pytest.approx(0.0) + assert recall["delta"] == pytest.approx(-1.0) + assert recall["significant"] is True + assert row["measured_primary_delta_pct"] == pytest.approx(-100.0) + + +def test_an_ablation_with_no_effect_reports_no_significant_delta(harness) -> None: + report = _runner(harness, StubService(demote_on="something_else")).run(CASES) + row = next(item for item in report["ablations"] if item["key"] == "no_cross_encoder") + + assert row["comparisons"]["ndcg_at_5"]["delta"] == pytest.approx(0.0) + assert row["claim_supported"] == "not_significant" + + +def test_a_contradicted_claim_is_labelled_as_such(harness) -> None: + """A published delta the measurement reverses must say so, not be quietly dropped.""" + inverted = Ablation( + key="inverted", + label="Component whose removal helps", + overrides={"enable_cross_encoder": False}, + scope="query", + primary_metric="recall_at_5", + documented_claim_pct=-25.0, + rationale="claims removal hurts", + ) + # Two gold regions per query, so the full system's recall is 0.5 rather than 0 and a + # relative delta has a denominator to be expressed against. + cases = [{"query": f"question {i}", "relevant_ids": [f"doc-{i}-region-0", f"doc-{i}-region-1"]} for i in range(12)] + + class Improves: + def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + index = int(text.split()[-1]) + found = ["doc-%d-region-0" % index] + if overrides: # ablated: finds both gold regions + found.append("doc-%d-region-1" % index) + results = [{"region_id": rid, "text": "gold evidence", "score": 1.0} for rid in found] + return {"results": results, "retrieved_evidence": results, "answer": "gold evidence"} + + report = _runner(harness, Improves(), ablations=[inverted]).run(cases) + row = report["ablations"][0] + + assert row["comparisons"]["recall_at_5"]["mean_full"] == pytest.approx(0.5) + assert row["comparisons"]["recall_at_5"]["mean_ablated"] == pytest.approx(1.0) + assert row["measured_primary_delta_pct"] == pytest.approx(100.0) + assert row["claim_supported"] == "contradicted" + + +def test_a_relative_delta_against_a_zero_baseline_is_not_invented(harness) -> None: + """0% would read as "no effect"; the truth is "no denominator".""" + ablation = Ablation( + key="zero_baseline", + label="Zero-baseline case", + overrides={"enable_cross_encoder": False}, + scope="query", + primary_metric="recall_at_5", + documented_claim_pct=-25.0, + rationale="n/a", + ) + + class NeverFinds: + def query(self, text: str, overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + results = [{"region_id": "unrelated", "text": "filler", "score": 1.0}] + return {"results": results, "retrieved_evidence": results, "answer": ""} + + report = _runner(harness, NeverFinds(), ablations=[ablation]).run(CASES) + row = report["ablations"][0] + + assert row["comparisons"]["recall_at_5"]["relative_delta_pct"] is None + assert row["claim_supported"] == "not_measurable" + + +def test_a_baseline_is_scored_through_the_same_harness(harness) -> None: + class Baseline: + def query(self, text: str) -> Dict[str, Any]: + return {"results": [{"region_id": "wrong", "text": "unrelated", "score": 1.0}], "answer": ""} + + report = _runner(harness, StubService()).run(CASES, baseline=Baseline()) + comparison = report["baseline"]["full_vs_baseline"]["recall_at_5"] + + assert comparison["mean_full"] == pytest.approx(1.0) + assert comparison["mean_baseline"] == pytest.approx(0.0) + assert comparison["delta"] == pytest.approx(1.0) + + +def test_running_without_queries_is_refused(harness) -> None: + with pytest.raises(ValueError, match="measures nothing"): + _runner(harness, StubService()).run([]) + + +def test_the_matrix_renders_measured_next_to_claimed(harness) -> None: + report = _runner(harness, StubService(demote_on="enable_sparse_retrieval")).run(CASES) + table = markdown_matrix(report) + + assert "| Remove BM25S sparse path |" in table + assert "-28.9%" in table, "the published claim must stay visible beside the measurement" + assert "not executed" in table + # An unmeasured row keeps its claim too: dropping it would retire an unverified + # number rather than show it standing unverified. + assert "| Remove Docling layout parser | recall_at_5 | not executed | -42.5% |" in table + + +def test_the_old_entry_point_refuses_rather_than_returning_a_number() -> None: + with pytest.raises(NotImplementedError, match="AblationRunner"): + AblationStudy().run({}, []) diff --git a/tests/tripath/test_eval_suite_runner.py b/tests/tripath/test_eval_suite_runner.py new file mode 100644 index 0000000..c54c6b7 --- /dev/null +++ b/tests/tripath/test_eval_suite_runner.py @@ -0,0 +1,100 @@ +"""test_eval_suite_runner.py - the suite runner scores, censuses, and survives failures. + +``run_eval_suite.py`` is the command that answers "what does DocuReason score on the gold +corpus". Two things in it are easy to get silently wrong and invisible afterwards: the +degradation census (a substring-scan run scores like an index-backed one, so the count is +the only signal) and failure handling (one exploding query must not void a two-hour +suite). Both are asserted here against a stub, so neither needs a corpus to verify. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from docureason.tripath.evaluation.eval_harness import EvaluationHarness +from scripts.run_eval_suite import run_suite + +pytestmark = pytest.mark.unit + + +class StubService: + """Returns the gold region at rank 1, and raises on one nominated query.""" + + def __init__(self, fail_on: Optional[str] = None) -> None: + self.fail_on = fail_on + self.queries: List[str] = [] + + def query(self, text: str) -> Dict[str, Any]: + self.queries.append(text) + if self.fail_on and self.fail_on == text: + raise RuntimeError("retriever exploded") + index = int(text.split()[-1]) + results = [{"region_id": f"doc-{index}-region-0", "text": "gold evidence", "score": 1.0}] + return { + "results": results, + "retrieved_evidence": results, + "answer": "net revenue rose", + "sql_execution": {"executed": True}, + "latency_breakdown": {"stages_ms": {"rerank": 2.0, "generate": 1.0}, "total_ms": 4.0, "residual_ms": 1.0}, + "degradation": ["attribution:lexical", "generation:fallback_used"], + "retrieval_stats": { + "modality_modes": {"text": {"mode": "hybrid_index"}, "vision": {"mode": "substring_scan"}} + }, + "cached": False, + } + + +CASES: List[Dict[str, Any]] = [ + { + "query": f"question {index}", + "relevant_ids": [f"doc-{index}-region-0"], + "ground_truth_answer": "net revenue rose", + "stratum": "prose" if index % 2 else "numeric", + } + for index in range(6) +] + + +@pytest.fixture +def harness(tmp_path) -> EvaluationHarness: + return EvaluationHarness(output_dir=tmp_path) + + +def test_every_case_is_scored(harness) -> None: + report = run_suite(StubService(), harness, CASES, iterations=100) + + assert report["queries_requested"] == 6 + assert report["queries_scored"] == 6 + assert report["failures"] == [] + assert report["summary_metrics"]["average_recall_at_5"] == pytest.approx(1.0) + assert report["summary_metrics"]["token_f1"] == pytest.approx(1.0) + + +def test_degradation_and_retrieval_mode_are_counted_not_just_logged(harness) -> None: + report = run_suite(StubService(), harness, CASES, iterations=100) + + assert report["degradation_counts"] == {"attribution:lexical": 6, "generation:fallback_used": 6} + # index_backed vs substring_scan is the difference between querying the indexes and + # scanning past them, and a suite report that omits it scores both the same. + assert report["retrieval_modes"] == {"text:hybrid_index": 6, "vision:substring_scan": 6} + + +def test_a_failing_query_is_recorded_without_voiding_the_suite(harness) -> None: + report = run_suite(StubService(fail_on="question 3"), harness, CASES, iterations=100) + + assert report["queries_scored"] == 5 + assert len(report["failures"]) == 1 + assert report["failures"][0]["query"] == "question 3" + assert "retriever exploded" in report["failures"][0]["error"] + + +def test_strata_and_stage_latency_survive_into_the_report(harness) -> None: + report = run_suite(StubService(), harness, CASES, iterations=100) + + assert sorted(report["per_stratum"]) == ["numeric", "prose"] + assert report["per_stratum"]["prose"]["n"] == 3 + assert report["stage_latency_ms"]["rerank"]["mean_ms"] == pytest.approx(2.0) + assert report["stage_latency_ms"]["cases_with_breakdown"] == 6 + assert report["cache_hits"] == 0 diff --git a/tests/tripath/test_figures.py b/tests/tripath/test_figures.py new file mode 100644 index 0000000..a401801 --- /dev/null +++ b/tests/tripath/test_figures.py @@ -0,0 +1,244 @@ +"""test_figures.py - the figures say what the data says, and nothing it does not. + +The failure mode worth testing for in plotting code is not a crash: it is a figure that +looks fine and misleads. Three specific ones are asserted here. + +* A metric a stratum never had must render as ``n/a``, not as a zero bar. "Measured zero" + and "not measured" are different findings and a bar cannot express the second. +* An ablation row that could not be measured must still appear, with its published claim + still plotted, so the figure cannot quietly retire an unverified number. +* A missing report must skip its figures and say so, rather than drawing an empty axis - + which reads as "measured, and flat". +""" + +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from docureason.tripath.evaluation import figures + +pytestmark = pytest.mark.unit + +pytest.importorskip("matplotlib", reason="figures need the [figures] extra") + + +SUITE: Dict[str, Any] = { + "queries_scored": 12, + "queries_requested": 12, + "wall_clock_sec": 4.2, + "bootstrap_iterations": 200, + "summary_metrics": { + "average_recall_at_5": 0.5, + "average_ndcg_at_5": 0.42, + "token_f1": 0.0, + "exact_match": 0.0, + "attribution_precision": 0.6, + "hallucination_rate": 0.4, + "latency_p50_ms": 900.0, + "latency_p90_ms": 2600.0, + "latency_p99_ms": 3100.0, + "mean_latency_ms": 1200.0, + }, + "confidence_intervals": { + "recall_at_5": {"ci_low": 0.2, "ci_high": 0.8, "point": 0.5}, + "ndcg_at_5": {"ci_low": 0.1, "ci_high": 0.7, "point": 0.42}, + }, + "per_stratum": { + # prose carries no ground-truth answer, so token_f1 is absent rather than zero. + "prose": {"n": 6, "summary_metrics": {"average_recall_at_5": 0.8, "average_ndcg_at_5": 0.7}}, + "tabular_lookup": { + "n": 6, + "summary_metrics": {"average_recall_at_5": 0.2, "average_ndcg_at_5": 0.14, "token_f1": 0.0}, + }, + }, + "stage_latency_ms": { + "cases_with_breakdown": 12, + "rerank": {"mean_ms": 700.0, "p95_ms": 1500.0, "observations": 12}, + "dense_search": {"mean_ms": 300.0, "p95_ms": 900.0, "observations": 12}, + }, + "degradation_counts": {"generation:fallback_used": 12, "retrieval:vision:substring_scan": 3}, + "retrieval_modes": {"text:hybrid_index": 12, "vision:substring_scan": 3}, + "per_query": [ + { + "latency_ms": 800.0 + index * 90, + "latency_breakdown": {"stages_ms": {"rerank": 700.0}, "total_ms": 1000.0, "residual_ms": -20.0}, + } + for index in range(12) + ], +} + +ABLATION: Dict[str, Any] = { + "queries": 12, + "seed": 0, + "bootstrap_iterations": 200, + "executed": 2, + "not_executed": ["no_layout_parser_missing"], + "query_subset": {"queries_used": 12, "source": "data/eval/queries.json", "is_full_matrix": False}, + "ablations": [ + { + "key": "no_sparse_path", + "label": "Remove BM25S sparse path", + "primary_metric": "recall_at_5", + "documented_claim_pct": -28.9, + "executed": True, + "claim_supported": "consistent_in_magnitude", + "measured_primary_delta_pct": -30.0, + "comparisons": { + "recall_at_5": { + "delta": -0.15, + "delta_ci_low": -0.25, + "delta_ci_high": -0.05, + "mean_full": 0.5, + "p_value": 0.01, + "significant": True, + } + }, + }, + { + "key": "no_layout_parser", + "label": "Remove Docling layout parser", + "primary_metric": "recall_at_5", + "documented_claim_pct": -42.5, + "executed": True, + "cross_corpus": True, + "answer_metric": "token_f1", + "measured_answer_delta_pct": None, + "claim_supported": "not_comparable_across_corpora", + "comparisons": { + "recall_at_5": {"comparable": False, "reason": "label space differs"}, + "token_f1": {"delta": 0.0, "p_value": 1.0}, + }, + "corpora": { + "full": {"total_regions": 1169, "structured_tables_serialized": 62}, + "ablated": {"total_regions": 6, "structured_tables_serialized": 0}, + }, + }, + { + "key": "no_layout_parser_missing", + "label": "Row nothing measured", + "primary_metric": "recall_at_5", + "documented_claim_pct": -42.5, + "executed": False, + "reason": "ingestion-scoped: needs a corpus re-ingested with this component disabled", + }, + ], + "baseline": { + "system": "NaiveRAGBaseline", + "full_vs_baseline": { + "recall_at_5": { + "delta": 0.22, + "delta_ci_low": 0.05, + "delta_ci_high": 0.4, + "p_value": 0.02, + "significant": True, + }, + "latency_ms": {"delta": 6400.0, "delta_ci_low": 3000.0, "delta_ci_high": 9000.0, "significant": True}, + }, + }, +} + + +def _write(tmp_path, name: str, payload: Dict[str, Any]): + import json + + path = tmp_path / name + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_every_supported_figure_is_written(tmp_path) -> None: + result = figures.build_all( + eval_report=_write(tmp_path, "eval.json", SUITE), + ablation_report=_write(tmp_path, "abl.json", ABLATION), + out_dir=tmp_path / "figures", + formats=("png",), + ) + + stems = {path.rsplit("\\", 1)[-1].rsplit("/", 1)[-1] for path in result["figures"]} + assert stems == { + "fig1_headline_metrics.png", + "fig2_per_stratum.png", + "fig3_stage_latency.png", + "fig4_latency_distribution.png", + "fig5_ablation_effects.png", + "fig6_baseline_comparison.png", + "fig7_degradation.png", + } + for path in result["figures"]: + # A figure that renders to a few hundred bytes is an empty axis. + assert (tmp_path / "figures" / path.rsplit("figures", 1)[-1].lstrip(r"\/")).stat().st_size > 5_000 + + +def test_a_missing_report_skips_its_figures_rather_than_drawing_nothing(tmp_path) -> None: + result = figures.build_all( + eval_report=_write(tmp_path, "eval.json", SUITE), + ablation_report=tmp_path / "does_not_exist.json", + out_dir=tmp_path / "figures", + formats=("png",), + ) + + assert not any("ablation" in path for path in result["figures"]) + assert any("ablation effects" in item for item in result["skipped"]) + assert any("Figures not produced" in line for line in [(tmp_path / "figures" / "INSIGHTS.md").read_text(encoding="utf-8")]) + + +def test_no_report_at_all_is_refused(tmp_path) -> None: + with pytest.raises(SystemExit, match="no reports to plot"): + figures.build_all(out_dir=tmp_path / "figures", formats=("png",)) + + +def _digest(tmp_path, suite=SUITE, ablation=ABLATION) -> str: + figures.build_all( + eval_report=_write(tmp_path, "eval.json", suite) if suite else None, + ablation_report=_write(tmp_path, "abl.json", ablation) if ablation else None, + out_dir=tmp_path / "figures", + formats=("png",), + ) + return (tmp_path / "figures" / "INSIGHTS.md").read_text(encoding="utf-8") + + +def test_the_digest_reports_the_stratum_spread(tmp_path) -> None: + text = _digest(tmp_path) + + assert "prose" in text and "tabular_lookup" in text + assert "0.600" in text, "the spread between the best and worst stratum is the point of reporting strata" + + +def test_the_digest_flags_a_boundary_that_fired_on_every_query(tmp_path) -> None: + text = _digest(tmp_path) + + assert "generation:fallback_used" in text + assert "every" in text + assert "substring_scan" in text, "a modality that fell back to a linear scan must be named" + + +def test_the_digest_carries_the_census_for_a_row_that_cannot_be_compared(tmp_path) -> None: + text = _digest(tmp_path) + + assert "1,169 -> 6" in text + assert "62 -> 0" in text + assert "not comparable" in text + + +def test_the_digest_names_significant_rows_and_the_latency_cost(tmp_path) -> None: + text = _digest(tmp_path) + + assert "Remove BM25S sparse path" in text + assert "-30.0%" in text + assert "+6,400 ms" in text, "the architecture's latency cost belongs beside its quality gain" + + +def test_the_digest_states_the_significance_floor_for_a_tiny_run(tmp_path) -> None: + """At n=3 nothing can reach p<0.05, so "not significant" carries no information.""" + tiny = dict(ABLATION, queries=3) + text = _digest(tmp_path, ablation=tiny) + + assert "cannot return a p below 0.250" in text + + +def test_the_significance_floor_is_two_to_the_one_minus_n() -> None: + assert figures._sign_test_floor(3) == pytest.approx(0.25) + assert figures._sign_test_floor(6) == pytest.approx(0.03125) + assert figures._sign_test_floor(0) is None diff --git a/tests/tripath/test_gold_queries.py b/tests/tripath/test_gold_queries.py new file mode 100644 index 0000000..9d2ec04 --- /dev/null +++ b/tests/tripath/test_gold_queries.py @@ -0,0 +1,184 @@ +"""test_gold_queries.py — gold labels that are derivable, or not emitted at all. + +The evaluation set this replaces had invented answers and gold ids naming fixtures that +no longer resolved (audit P0-4). The property that makes the replacement trustworthy is +narrow and testable: a query is emitted only when its anchor identifies exactly one +region in the whole corpus. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from docureason.tripath.evaluation.gold_queries import ( + build_phrase_index, + generate_queries, + validate_queries, +) + +pytestmark = pytest.mark.unit + + +def _region( + region_id: str, + region_type: str, + text: str, + table: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + payload: Dict[str, Any] = {"id": region_id, "type": region_type, "text": text, "metadata": {}} + if table is not None: + payload["table_json"] = table + return payload + + +def _corpus(regions: List[Dict[str, Any]], doc_id: str = "doc-a") -> Dict[str, Any]: + return {"documents": [{"id": doc_id, "source": "a.html", "regions": regions}]} + + +UNIQUE_PROSE = ( + "The Americas segment recorded an unusual goodwill impairment tied to the Helios " + "acquisition during the fourth quarter of the reporting period." +) +BOILERPLATE = "We are subject to risks and uncertainties that could affect our business and results of operations." + + +def test_a_table_cell_query_carries_the_cell_as_its_answer() -> None: + table = {"columns": ["segment", "fiscal 2024"], "rows": [["Americas revenue", "15000"]]} + report = generate_queries(_corpus([_region("doc-a-region-0", "table", "table", table)]), per_stratum=5) + + lookups = [q for q in report.queries if q.stratum == "tabular_lookup"] + assert lookups, "a two-column table with one labelled row is enough to derive a lookup" + query = lookups[0] + assert query.relevant_ids == ["doc-a-region-0"] + assert query.ground_truth_answer == "15000" + assert query.anchor["row_label"] == "Americas revenue" + assert query.anchor["column"] == "fiscal 2024" + assert query.provenance == "programmatic" + + +def test_the_numeric_stratum_only_uses_cells_that_parse_as_numbers() -> None: + table = {"columns": ["metric", "value"], "rows": [["Total assets", "not disclosed"], ["Total equity", "91000"]]} + report = generate_queries(_corpus([_region("doc-a-region-0", "table", "t", table)]), per_stratum=5) + + numeric = [q for q in report.queries if q.stratum == "numeric"] + assert numeric + assert numeric[0].ground_truth_answer == "91000" + + +def test_a_table_with_no_usable_cell_is_rejected_with_a_reason() -> None: + table = {"columns": ["a", "b"], "rows": [["1", "2"]]} + report = generate_queries(_corpus([_region("doc-a-region-0", "table", "t", table)]), per_stratum=5) + + assert not [q for q in report.queries if q.stratum == "tabular_lookup"] + assert report.rejected["tabular_lookup:no_cell"] == 1 + + +def test_at_most_one_query_per_table() -> None: + """Fifty questions about one balance sheet would make the stratum mean meaningless.""" + # Values wide enough to qualify: the lookup stratum skips 1-3 character numerics, + # which carry no signal as an answer. + rows = [[f"Line item {i}", "10,500", "20,750", "30,900"] for i in range(30)] + table = {"columns": ["metric", "v1", "v2", "v3"], "rows": rows} + report = generate_queries(_corpus([_region("doc-a-region-0", "table", "t", table)]), per_stratum=20) + + assert len([q for q in report.queries if q.stratum == "tabular_lookup"]) == 1 + + +def test_a_prose_query_is_anchored_on_a_unique_phrase() -> None: + report = generate_queries(_corpus([_region("doc-a-region-0", "body", UNIQUE_PROSE)]), per_stratum=5) + + prose = [q for q in report.queries if q.stratum == "prose"] + assert prose + assert prose[0].relevant_ids == ["doc-a-region-0"] + assert prose[0].anchor["phrase"] in prose[0].query + assert prose[0].ground_truth_answer is None, "retrieval gold, not an answer key" + + +def test_boilerplate_repeated_across_regions_cannot_be_gold_labelled() -> None: + """Risk-factor and legal boilerplate repeats verbatim; no phrase identifies one region.""" + corpus = _corpus( + [ + _region("doc-a-region-0", "body", BOILERPLATE), + _region("doc-a-region-1", "body", BOILERPLATE), + _region("doc-a-region-2", "body", BOILERPLATE), + ] + ) + report = generate_queries(corpus, per_stratum=5) + + assert not [q for q in report.queries if q.stratum == "prose"] + assert report.rejected["prose:no_unique_phrase"] == 3 + + +def test_a_region_too_short_to_anchor_is_rejected() -> None: + report = generate_queries(_corpus([_region("doc-a-region-0", "body", "Revenue rose.")]), per_stratum=5) + assert report.rejected["prose:too_short"] == 1 + + +def test_an_empty_stratum_says_why() -> None: + """Zero-by-composition must be distinguishable from zero-by-rejection.""" + report = generate_queries(_corpus([_region("doc-a-region-0", "body", UNIQUE_PROSE)]), per_stratum=5) + assert report.rejected["figure:no_regions_in_corpus"] == 1 + + +def test_generation_is_deterministic_for_a_seed() -> None: + corpus = _corpus( + [ + _region("doc-a-region-0", "body", UNIQUE_PROSE), + _region("doc-a-region-1", "body", UNIQUE_PROSE.replace("Helios", "Aurora")), + ] + ) + first = [q.as_dict() for q in generate_queries(corpus, per_stratum=5, seed=3).queries] + second = [q.as_dict() for q in generate_queries(corpus, per_stratum=5, seed=3).queries] + assert first == second + + +def test_the_phrase_index_counts_each_region_once_per_shingle() -> None: + corpus = _corpus([_region("doc-a-region-0", "body", "alpha beta gamma delta " * 3)]) + index = build_phrase_index(corpus, anchor_words=4) + assert index["alpha beta gamma delta"] == ["doc-a-region-0"] + + +def test_validation_passes_on_a_freshly_generated_set() -> None: + corpus = _corpus( + [ + _region("doc-a-region-0", "body", UNIQUE_PROSE), + _region("doc-a-region-1", "table", "t", {"columns": ["k", "v"], "rows": [["Total assets", "9100"]]}), + ] + ) + report = generate_queries(corpus, per_stratum=5, seed=0) + result = validate_queries([q.as_dict() for q in report.queries], corpus) + + assert result["valid"] is True + assert result["dangling_gold_ids"] == [] + + +def test_validation_catches_a_gold_id_that_no_longer_resolves() -> None: + """A re-ingest with a different digest invalidates every gold id; that must be loud.""" + corpus = _corpus([_region("doc-a-region-0", "body", UNIQUE_PROSE)]) + stale = [{"id": "q-0001", "relevant_ids": ["doc-a-region-999"], "anchor": {}}] + result = validate_queries(stale, corpus) + + assert result["valid"] is False + assert result["dangling_gold_ids"] == ["q-0001:doc-a-region-999"] + + +def test_validation_catches_an_anchor_that_stopped_being_unique() -> None: + corpus = _corpus( + [ + _region("doc-a-region-0", "body", UNIQUE_PROSE), + _region("doc-a-region-1", "body", UNIQUE_PROSE), + ] + ) + duplicated = [ + { + "id": "q-0001", + "relevant_ids": ["doc-a-region-0"], + "anchor": {"phrase": "americas segment recorded an unusual goodwill impairment"}, + } + ] + result = validate_queries(duplicated, corpus) + + assert result["valid"] is False + assert result["non_unique_anchors"] == ["q-0001:2 owner(s)"] diff --git a/tests/tripath/test_metric_regression_gate.py b/tests/tripath/test_metric_regression_gate.py new file mode 100644 index 0000000..effdbc1 --- /dev/null +++ b/tests/tripath/test_metric_regression_gate.py @@ -0,0 +1,94 @@ +"""test_metric_regression_gate.py — the gate must be able to fail. + +An advisory check that reads as enforced is worse than no check (audit P3-6), so the +important assertion here is not that the gate passes on the committed baseline - it is +that a changed metric makes it fail. +""" + +from __future__ import annotations + +import json + +import pytest + +from scripts.check_metric_regression import ( + BASELINE_PATH, + FIXTURE_PATH, + compare, + compute, + flatten, + load_fixture, +) + +pytestmark = pytest.mark.unit + + +@pytest.fixture(scope="module") +def current(tmp_path_factory) -> dict: + return compute(load_fixture(FIXTURE_PATH), tmp_path_factory.mktemp("metrics")) + + +@pytest.fixture(scope="module") +def baseline() -> dict: + return json.loads(BASELINE_PATH.read_text(encoding="utf-8")) + + +def test_the_committed_baseline_still_holds(current, baseline) -> None: + ok, findings = compare(current, baseline, tolerance=0.01) + assert ok, "metric values moved; if intended, re-record with --update\n" + "\n".join(findings) + + +def test_the_fixture_covers_every_stratum(current) -> None: + """A gate over one stratum would not notice a formula that only affects tables.""" + assert set(current["per_stratum"]) == {"tabular_lookup", "numeric", "prose", "figure"} + assert current["total_queries"] == 20 + + +def test_the_gate_fails_when_a_metric_moves(current, baseline) -> None: + drifted = json.loads(json.dumps(baseline)) + drifted["summary_metrics"]["average_recall_at_5"] += 0.05 + + ok, findings = compare(current, drifted, tolerance=0.01) + assert ok is False + assert any("MOVED" in finding and "average_recall_at_5" in finding for finding in findings) + + +def test_movement_inside_the_tolerance_is_allowed(current, baseline) -> None: + """Floating-point reassociation must not turn into a red build.""" + nudged = json.loads(json.dumps(baseline)) + nudged["summary_metrics"]["average_recall_at_5"] += 0.005 + + ok, _ = compare(current, nudged, tolerance=0.01) + assert ok is True + + +def test_a_removed_metric_is_drift(current, baseline) -> None: + reduced = json.loads(json.dumps(baseline)) + reduced["summary_metrics"].pop("mrr") + + ok, findings = compare(current, reduced, tolerance=0.01) + assert ok is False + assert any(finding.startswith("NEW") and "mrr" in finding for finding in findings) + + +def test_an_added_metric_is_drift(current, baseline) -> None: + extended = json.loads(json.dumps(baseline)) + extended["summary_metrics"]["invented_metric"] = 0.5 + + ok, findings = compare(current, extended, tolerance=0.01) + assert ok is False + assert any(finding.startswith("REMOVED") and "invented_metric" in finding for finding in findings) + + +def test_machine_dependent_metrics_are_excluded(current) -> None: + """Gating latency would fail the build on a busy runner, not on a code change.""" + names = set(flatten(current)) + assert not [name for name in names if "latency" in name or "qps" in name or "duration" in name] + + +def test_the_fixture_is_frozen_output_not_a_pipeline_run() -> None: + payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + assert payload["case_count"] == len(payload["cases"]) + for case in payload["cases"]: + assert case["results"], "each case must carry its own frozen retrieval output" + assert case["relevant_ids"] diff --git a/tests/tripath/test_naive_baseline.py b/tests/tripath/test_naive_baseline.py new file mode 100644 index 0000000..b71ed34 --- /dev/null +++ b/tests/tripath/test_naive_baseline.py @@ -0,0 +1,177 @@ +"""test_naive_baseline.py — the comparison point every reported delta needs. + +The docs quote improvements over "naive RAG" without one existing, so nothing the +tri-path system achieved could be attributed to the architecture rather than to the +corpus or the generator. These tests pin the splitter's properties and the region +attribution that lets one harness score both systems. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest + +from docureason.tripath.evaluation.baselines import NaiveRAGBaseline, recursive_split +from docureason.tripath.evaluation.eval_harness import EvaluationHarness +from docureason.tripath.ingestion.schema import Document, Region + +pytestmark = pytest.mark.unit + + +class StubGenerator: + """Stands in for GenerationModule so the baseline is testable without a model.""" + + def __init__(self) -> None: + self.calls: List[Dict[str, Any]] = [] + + def generate(self, query: str, results: List[Dict[str, Any]], sql_payload: Dict[str, Any]) -> Dict[str, Any]: + self.calls.append({"query": query, "results": results, "sql": sql_payload}) + return {"answer": f"answer to {query}", "engine": "stub", "citations": []} + + +def _document() -> Document: + regions = [ + Region(type="title", text="Annual Report", start=0, end=13, layout_source="docling"), + Region( + type="body", + text="Operating revenue for the Americas segment rose to 15000 in the period under review. " * 6, + start=0, + end=10, + layout_source="docling", + ), + Region( + type="body", + text="The Europe segment reported a decline attributable to currency translation effects. " * 6, + start=0, + end=10, + layout_source="docling", + ), + ] + for index, region in enumerate(regions): + region.metadata = {"region_id": f"doc-naive-region-{index}"} + return Document(id="doc-naive", source="f.html", title="Annual Report", regions=regions, metadata={}) + + +# --------------------------------------------------------------------------- +# Splitter +# --------------------------------------------------------------------------- + + +def test_spans_cover_the_text_without_gaps() -> None: + text = "alpha beta gamma. delta epsilon zeta. eta theta iota. " * 8 + spans = recursive_split(text, chunk_size=80, overlap=20) + + assert spans[0][0] == 0 + assert spans[-1][1] == len(text) + for (_, previous_end), (next_start, _) in zip(spans, spans[1:]): + assert next_start <= previous_end, "a gap between chunks would drop text silently" + + +def test_overlap_makes_consecutive_chunks_share_text() -> None: + text = "word " * 200 + spans = recursive_split(text, chunk_size=100, overlap=40) + assert len(spans) > 2 + assert spans[1][0] < spans[0][1], "consecutive chunks must overlap when overlap > 0" + + +def test_the_splitter_terminates_on_text_with_no_separators() -> None: + """The last separator is the empty string precisely so this cannot loop forever.""" + spans = recursive_split("x" * 500, chunk_size=64, overlap=8) + assert spans and spans[-1][1] == 500 + + +def test_zero_overlap_is_allowed() -> None: + spans = recursive_split("a b c d e f g h i j " * 20, chunk_size=50, overlap=0) + assert spans[-1][1] == 400 + + +def test_a_non_positive_chunk_size_is_refused() -> None: + with pytest.raises(ValueError, match="chunk_size"): + recursive_split("text", chunk_size=0, overlap=0) + + +# --------------------------------------------------------------------------- +# Region attribution and retrieval +# --------------------------------------------------------------------------- + + +def test_chunks_record_every_region_they_span(monkeypatch) -> None: + """A naive splitter cuts across gold units; scoring must not punish it for that.""" + baseline = NaiveRAGBaseline(chunk_size=400, chunk_overlap=50, generator=StubGenerator()) + # Chunking is what is under test here; the embedding model is exercised by the + # integration test at the bottom of this module, which keeps the fast gate fast. + monkeypatch.setattr(baseline, "_get_model", lambda: None) + baseline.build([_document()]) + + assert baseline.chunks + assert all(chunk.region_id for chunk in baseline.chunks) + spanning = [chunk for chunk in baseline.chunks if len(chunk.region_ids) > 1] + assert spanning, "with chunks larger than the regions, some chunk must span two" + for chunk in spanning: + assert chunk.region_id == chunk.region_ids[0], "the primary region is the largest contributor" + + +def test_the_harness_credits_a_chunk_that_overlaps_the_gold_region(tmp_path) -> None: + harness = EvaluationHarness(output_dir=tmp_path) + metrics = harness.evaluate_single( + query="revenue", + results=[{"region_id": "doc-x-region-5", "region_ids": ["doc-x-region-5", "doc-x-region-6"], "text": "t"}], + relevant_ids=["doc-x-region-6"], + ) + assert metrics["recall_at_5"] == 1.0 + + +def test_retrieval_degrades_to_keyword_overlap_without_a_model(monkeypatch) -> None: + """A missing model must not answer every query with an empty list.""" + baseline = NaiveRAGBaseline(top_k=2, generator=StubGenerator()) + monkeypatch.setattr(baseline, "_get_model", lambda: None) + stats = baseline.build([_document()]) + + assert stats["degraded"] == "embedder_unavailable" + assert stats["index_type"] == "keyword_overlap_fallback" + + response = baseline.query("Europe segment currency translation") + assert response["results"], "the fallback must still rank something" + assert response["results"][0]["retrieval_mode"] == "keyword_overlap" + assert "baseline:embedder_unavailable" in response["degradation"] + + +def test_the_top_k_cap_is_honoured(monkeypatch) -> None: + baseline = NaiveRAGBaseline(chunk_size=120, chunk_overlap=20, top_k=3, generator=StubGenerator()) + monkeypatch.setattr(baseline, "_get_model", lambda: None) + baseline.build([_document()]) + + assert len(baseline.chunks) > 3 + assert len(baseline.query("segment revenue period")["results"]) <= 3 + + +def test_the_response_shape_matches_the_full_system(monkeypatch) -> None: + """The harness scores both systems through one path; the shape is that contract.""" + baseline = NaiveRAGBaseline(generator=StubGenerator()) + monkeypatch.setattr(baseline, "_get_model", lambda: None) + baseline.build([_document()]) + response = baseline.query("revenue") + + for key in ("query", "results", "answer", "degradation", "latency_breakdown", "retrieval_stats", "cached"): + assert key in response, f"missing {key!r} from the baseline response" + assert response["system"] == "naive_rag_baseline" + + +def test_the_baseline_uses_the_generator_it_is_given(monkeypatch) -> None: + """Sharing the generator is what isolates retrieval in the comparison.""" + generator = StubGenerator() + baseline = NaiveRAGBaseline(generator=generator) + monkeypatch.setattr(baseline, "_get_model", lambda: None) + baseline.build([_document()]) + baseline.query("revenue") + + assert len(generator.calls) == 1 + assert generator.calls[0]["sql"] == {"executed": False, "sql_results": []} + + +def test_an_empty_corpus_is_reported_not_crashed() -> None: + baseline = NaiveRAGBaseline(generator=StubGenerator()) + stats = baseline.build([]) + assert stats["degraded"] == "empty_corpus" + assert baseline.query("anything")["results"] == [] diff --git a/tests/tripath/test_naive_baseline_models.py b/tests/tripath/test_naive_baseline_models.py new file mode 100644 index 0000000..b77cb5e --- /dev/null +++ b/tests/tripath/test_naive_baseline_models.py @@ -0,0 +1,61 @@ +"""test_naive_baseline_models.py — the baseline against the real embedding stack. + +Split from ``test_naive_baseline.py`` because this is the only baseline test that loads +a model. Module-level ``pytestmark`` cannot be un-set per test, and leaving it in the +``unit`` module put a 60-second model load inside the fast gate. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest + +from docureason.tripath.evaluation.baselines import NaiveRAGBaseline +from docureason.tripath.ingestion.schema import Document, Region + +pytestmark = pytest.mark.integration + + +class StubGenerator: + def generate(self, query: str, results: List[Dict[str, Any]], sql_payload: Dict[str, Any]) -> Dict[str, Any]: + return {"answer": f"answer to {query}", "engine": "stub", "citations": []} + + +def _document() -> Document: + regions = [ + Region(type="title", text="Annual Report", start=0, end=13, layout_source="docling"), + Region( + type="body", + text="Operating revenue for the Americas segment rose to 15000 in the period under review. " * 6, + start=0, + end=10, + layout_source="docling", + ), + Region( + type="body", + text="The Europe segment reported a decline attributable to currency translation effects. " * 6, + start=0, + end=10, + layout_source="docling", + ), + ] + for index, region in enumerate(regions): + region.metadata = {"region_id": f"doc-naive-region-{index}"} + return Document(id="doc-naive", source="f.html", title="Annual Report", regions=regions, metadata={}) + + +def test_the_real_embedding_path_builds_a_flat_index() -> None: + """The one test that loads the actual model: a flat index over real vectors.""" + baseline = NaiveRAGBaseline(chunk_size=300, chunk_overlap=40, top_k=3, generator=StubGenerator()) + stats = baseline.build([_document()]) + + if stats.get("degraded"): + pytest.skip(f"embedding stack unavailable: {stats['degraded']}") + + assert stats["index_type"] == "flat_inner_product" + assert stats["ntotal"] == len(baseline.chunks) + assert stats["embedding_revision"] != "hub-default", "the baseline must use the pinned weights" + + response = baseline.query("Europe segment currency translation decline") + assert response["results"][0]["retrieval_mode"] == "flat_index" diff --git a/tests/tripath/test_rasterizer.py b/tests/tripath/test_rasterizer.py new file mode 100644 index 0000000..c9416e8 --- /dev/null +++ b/tests/tripath/test_rasterizer.py @@ -0,0 +1,164 @@ +"""test_rasterizer.py — real page rasters, and the blank check that was missing. + +Three audit findings shared one cause: nothing could rasterize a page, so the captioner +reused one embedded image for every figure (P1-3), captioned blank canvases (P1-4), and +the OCR fallback read only the first image per page (P2-13). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from docureason.tripath.ingestion.rasterizer import ( + POINTS_PER_INCH, + PageRasterizer, + page_index_from_region, +) + +pytestmark = pytest.mark.unit + +SAMPLE_PDF = Path(__file__).resolve().parents[2] / "samples" / "9781513563602-mod01.pdf" + + +@pytest.fixture(scope="module") +def rasterizer(): + if not SAMPLE_PDF.exists(): + pytest.skip("sample PDF not present") + instance = PageRasterizer(SAMPLE_PDF, dpi=150) + if not instance.available: + pytest.skip("pypdfium2 unavailable") + yield instance + instance.close() + + +# --------------------------------------------------------------------------- +# Page indexing (audit P1-2) +# --------------------------------------------------------------------------- + + +def test_docling_page_numbers_are_converted_to_zero_based_indices() -> None: + """Docling counts pages from 1; pypdfium2 and pypdf from 0.""" + assert page_index_from_region(1, 9) == 0 + assert page_index_from_region(4, 9) == 3 + + +def test_an_unknown_page_resolves_to_the_first() -> None: + assert page_index_from_region(-1, 9) == 0 + assert page_index_from_region(None, 9) == 0 + + +def test_an_out_of_range_page_is_clamped_not_wrapped() -> None: + assert page_index_from_region(99, 9) == 8 + + +def test_an_empty_document_yields_index_zero() -> None: + assert page_index_from_region(3, 0) == 0 + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def test_dpi_determines_the_raster_size(rasterizer) -> None: + """The first consumer of image_rendering_dpi, which previously reached no code.""" + image = rasterizer.render(0) + width_pt, height_pt = rasterizer.page_size(0) + + assert image is not None + assert image.size[0] == pytest.approx(width_pt * 150 / POINTS_PER_INCH, abs=2) + assert image.size[1] == pytest.approx(height_pt * 150 / POINTS_PER_INCH, abs=2) + + +def test_a_lower_dpi_produces_a_smaller_raster() -> None: + if not SAMPLE_PDF.exists(): + pytest.skip("sample PDF not present") + low = PageRasterizer(SAMPLE_PDF, dpi=72) + high = PageRasterizer(SAMPLE_PDF, dpi=200) + try: + if not (low.available and high.available): + pytest.skip("pypdfium2 unavailable") + assert low.render(0).size[0] < high.render(0).size[0] + finally: + low.close() + high.close() + + +def test_pages_are_rendered_once_and_cached(rasterizer) -> None: + """A page with six figures must not be rasterized six times.""" + assert rasterizer.render(0) is rasterizer.render(0) + + +def test_an_unreadable_document_reports_unavailable_rather_than_raising(tmp_path: Path) -> None: + broken = tmp_path / "not-a-pdf.pdf" + broken.write_text("this is not a PDF", encoding="utf-8") + instance = PageRasterizer(broken) + assert instance.available is False + assert instance.render(0) is None + assert instance.page_count == 0 + + +# --------------------------------------------------------------------------- +# Cropping +# --------------------------------------------------------------------------- + + +def test_a_bbox_crop_is_smaller_than_the_page(rasterizer) -> None: + page = rasterizer.render(0) + crop = rasterizer.crop(0, (100.0, 700.0, 400.0, 600.0)) + assert crop is not None + assert crop.size[0] < page.size[0] and crop.size[1] < page.size[1] + + +def test_bbox_top_and_bottom_may_arrive_either_way_round(rasterizer) -> None: + """Docling emits (l, t, r, b) bottom-origin, so t > b; normalising avoids a flip.""" + forward = rasterizer.crop(0, (100.0, 700.0, 400.0, 600.0)) + reversed_pair = rasterizer.crop(0, (100.0, 600.0, 400.0, 700.0)) + assert forward.size == reversed_pair.size + + +def test_a_missing_bbox_yields_the_whole_page(rasterizer) -> None: + """A full page is a worse caption input than a crop, but it is real content.""" + assert rasterizer.crop(0, None).size == rasterizer.render(0).size + + +def test_a_degenerate_bbox_yields_the_whole_page(rasterizer) -> None: + assert rasterizer.crop(0, (100.0, 500.0, 100.2, 500.1)).size == rasterizer.render(0).size + + +# --------------------------------------------------------------------------- +# Blank detection (audit P1-4) +# --------------------------------------------------------------------------- + + +def test_a_uniform_canvas_is_blank() -> None: + from PIL import Image + + assert PageRasterizer.is_blank(Image.new("RGB", (200, 120), (255, 255, 255))) is True + assert PageRasterizer.is_blank(Image.new("RGB", (200, 120), (0, 0, 0))) is True + + +def test_an_image_with_content_is_not_blank() -> None: + from PIL import Image, ImageDraw + + image = Image.new("RGB", (200, 120), (255, 255, 255)) + draw = ImageDraw.Draw(image) + draw.rectangle((10, 10, 120, 90), outline=(0, 0, 0), width=3) + draw.line((10, 100, 190, 20), fill=(0, 0, 0), width=2) + assert PageRasterizer.is_blank(image) is False + + +def test_none_counts_as_blank() -> None: + assert PageRasterizer.is_blank(None) is True + + +def test_a_rendered_text_page_is_not_blank(rasterizer) -> None: + assert PageRasterizer.is_blank(rasterizer.render(0)) is False + + +def test_png_encoding_round_trips(rasterizer) -> None: + payload = PageRasterizer.to_png_bytes(rasterizer.crop(0, (100.0, 700.0, 400.0, 600.0))) + assert payload and payload[:8] == b"\x89PNG\r\n\x1a\n" + assert PageRasterizer.to_png_bytes(None) is None diff --git a/tests/tripath/test_response_cache.py b/tests/tripath/test_response_cache.py new file mode 100644 index 0000000..c457980 --- /dev/null +++ b/tests/tripath/test_response_cache.py @@ -0,0 +1,233 @@ +"""test_response_cache.py — the response cache, the stage timer, and override isolation. + +Three things that must hold together for the serving layer to be measurable: + +* a cached response is marked as cached and does **not** replay the original run's + latency, or every reported percentile drifts toward the cache's; +* the cache key covers the effective config, so an ablation switch cannot be served a + response computed with the opposite setting; +* per-query overrides do not mutate the service, which they used to. +""" + +from __future__ import annotations + +import time + +import pytest + +from docureason.tripath.config import DocuReasonConfig +from docureason.tripath.serving.query_service import QueryService +from docureason.tripath.utils import StageTimer + +pytestmark = pytest.mark.unit + + +def _bare_service(cache_size: int = 3) -> QueryService: + """A QueryService with only its cache wired. + + Constructed without ``__init__`` on purpose: the real constructor ingests a corpus + and builds three models, none of which the cache depends on. The alternative is a + test that takes a minute to assert an eviction. + """ + service = object.__new__(QueryService) + service.config = DocuReasonConfig.load_preset("balanced") + service.config.serving.response_cache_size = cache_size + import threading + from collections import OrderedDict + + service._cache = OrderedDict() + service._cache_lock = threading.Lock() + service.cache_stats = {"hits": 0, "misses": 0, "evictions": 0} + return service + + +def _response(answer: str = "a") -> dict: + return { + "query": "q", + "answer": answer, + "results": [{"region_id": "r1", "text": "evidence"}], + "latency_breakdown": {"stages_ms": {"rerank": 40.0}, "total_ms": 900.0}, + "degradation": [], + "cached": False, + } + + +def test_a_miss_returns_nothing_and_is_counted() -> None: + service = _bare_service() + assert service._cache_get("k", StageTimer()) is None + assert service.cache_stats == {"hits": 0, "misses": 1, "evictions": 0} + + +def test_a_hit_is_marked_and_does_not_replay_the_original_latency() -> None: + service = _bare_service() + service._cache_put("k", _response(), service.config) + + hit = service._cache_get("k", StageTimer()) + assert hit["cached"] is True + assert hit["latency_breakdown"]["total_ms"] < 900.0, "a cache lookup is not a pipeline run" + assert hit["latency_breakdown_first_run"]["total_ms"] == 900.0 + assert service.cache_stats["hits"] == 1 + + +def test_a_hit_is_a_copy_so_callers_cannot_corrupt_the_cache() -> None: + """The eval harness stamps metrics onto responses; that must not leak across calls.""" + service = _bare_service() + service._cache_put("k", _response(), service.config) + + first = service._cache_get("k", StageTimer()) + first["results"][0]["text"] = "mutated" + second = service._cache_get("k", StageTimer()) + + assert second["results"][0]["text"] == "evidence" + + +def test_the_cache_evicts_least_recently_used() -> None: + service = _bare_service(cache_size=2) + service._cache_put("a", _response("a"), service.config) + service._cache_put("b", _response("b"), service.config) + service._cache_get("a", StageTimer()) # refresh a + service._cache_put("c", _response("c"), service.config) + + assert service._cache_get("b", StageTimer()) is None + assert service._cache_get("a", StageTimer()) is not None + assert service.cache_stats["evictions"] == 1 + + +def test_a_zero_size_cache_stores_nothing() -> None: + service = _bare_service(cache_size=0) + service._cache_put("k", _response(), service.config) + assert service._cache_get("k", StageTimer()) is None + assert service._cache == {} + + +def test_clear_cache_drops_everything() -> None: + service = _bare_service() + service._cache_put("k", _response(), service.config) + service.clear_cache() + assert service._cache_get("k", StageTimer()) is None + + +def test_the_key_separates_two_different_queries() -> None: + config = DocuReasonConfig.load_preset("balanced") + assert QueryService._cache_key("revenue", config) != QueryService._cache_key("margin", config) + + +def test_the_key_ignores_whitespace_around_the_query() -> None: + config = DocuReasonConfig.load_preset("balanced") + assert QueryService._cache_key(" revenue ", config) == QueryService._cache_key("revenue", config) + + +def test_an_ablation_switch_changes_the_key() -> None: + """Otherwise an ablated run could be served the full system's answer.""" + config = DocuReasonConfig.load_preset("balanced") + ablated = config.apply_query_overrides({"enable_cross_encoder": False}) + assert QueryService._cache_key("revenue", config) != QueryService._cache_key("revenue", ablated) + + +def test_the_key_ignores_where_artifacts_are_written() -> None: + """output_dir records where the run put files, not what it answered.""" + config = DocuReasonConfig.load_preset("balanced") + moved = DocuReasonConfig.load_preset("balanced") + moved.output_dir = "somewhere/else" + assert QueryService._cache_key("revenue", config) == QueryService._cache_key("revenue", moved) + + +# --------------------------------------------------------------------------- +# Per-query overrides must not mutate the service +# --------------------------------------------------------------------------- + + +def test_overrides_return_a_copy_and_leave_the_original_alone() -> None: + """One request carrying final_top_k=1 used to narrow the shared service for good.""" + config = DocuReasonConfig.load_preset("balanced") + original_top_k = config.reranker.final_top_k + + clone = config.apply_query_overrides({"final_top_k": 1, "enable_sparse_retrieval": False}) + + assert clone.reranker.final_top_k == 1 + assert clone.retrieval.enable_sparse_retrieval is False + assert config.reranker.final_top_k == original_top_k + assert config.retrieval.enable_sparse_retrieval is True + + +def test_no_overrides_returns_the_same_object() -> None: + config = DocuReasonConfig.load_preset("balanced") + assert config.apply_query_overrides({}) is config + + +def test_an_unknown_override_is_recorded_rather_than_silently_ignored() -> None: + clone = DocuReasonConfig.load_preset("balanced").apply_query_overrides({"nonsense": 1}) + assert clone.source_map["unknown_overrides"] == "nonsense" + + +def test_an_uncastable_override_is_rejected_and_recorded() -> None: + clone = DocuReasonConfig.load_preset("balanced").apply_query_overrides({"final_top_k": "not-a-number"}) + assert "final_top_k" in clone.source_map["unknown_overrides"] + assert clone.reranker.final_top_k == 5 + + +def test_boolean_overrides_accept_json_and_string_forms() -> None: + for raw in (False, "false", "0", "no", "off"): + clone = DocuReasonConfig.load_preset("balanced").apply_query_overrides({"enable_nli": raw}) + assert clone.attribution.enable_nli is False, raw + + +# --------------------------------------------------------------------------- +# Stage timer +# --------------------------------------------------------------------------- + + +def test_repeated_stage_names_accumulate() -> None: + timer = StageTimer() + for _ in range(3): + timer.add("dense_search", 5.0) + payload = timer.as_dict() + + assert payload["stages_ms"]["dense_search"] == 15.0 + assert payload["stage_calls"]["dense_search"] == 3 + + +def test_stages_are_reported_in_pipeline_order() -> None: + timer = StageTimer() + for stage in ("generate", "route", "rerank"): + timer.add(stage, 1.0) + assert list(timer.as_dict()["stages_ms"]) == ["route", "rerank", "generate"] + + +def test_an_unaccounted_gap_shows_up_as_a_residual() -> None: + """A decomposition that does not add up must say so rather than describe 60%.""" + timer = StageTimer() + timer.add("rerank", 10.0) + payload = timer.as_dict(total_ms=100.0) + + assert payload["accounted_ms"] == 10.0 + assert payload["residual_ms"] == 90.0 + assert payload["concurrent"] is False + + +def test_overlapping_stages_report_a_negative_residual() -> None: + """Under parallel dispatch the stages legitimately exceed wall time.""" + timer = StageTimer() + timer.add("dense_search", 80.0) + timer.add("sparse_search", 80.0) + payload = timer.as_dict(total_ms=100.0) + + assert payload["residual_ms"] < 0 + assert payload["concurrent"] is True + + +def test_the_context_manager_measures_elapsed_time() -> None: + timer = StageTimer() + with timer.stage("rerank"): + time.sleep(0.02) + assert timer.as_dict()["stages_ms"]["rerank"] >= 15.0 + + +def test_merging_folds_another_timers_stages_in() -> None: + first, second = StageTimer(), StageTimer() + first.add("dense_search", 4.0) + second.add("dense_search", 6.0) + second.add("fuse", 1.0) + first.merge(second) + + assert first.as_dict()["stages_ms"] == {"dense_search": 10.0, "fuse": 1.0} diff --git a/tests/tripath/test_statistics.py b/tests/tripath/test_statistics.py new file mode 100644 index 0000000..009a221 --- /dev/null +++ b/tests/tripath/test_statistics.py @@ -0,0 +1,127 @@ +"""test_statistics.py — the uncertainty machinery, checked against known answers. + +Every number in the ablation report and the suite summary now carries an interval and +a p-value from this module, so an error here would be invisible and everywhere. +""" + +from __future__ import annotations + +import pytest + +from docureason.tripath.evaluation.statistics import ( + bootstrap_ci, + compare_paired, + mean, + paired_bootstrap, + paired_permutation_test, +) + +pytestmark = pytest.mark.unit + + +def test_mean_of_nothing_is_none_not_zero() -> None: + """Unmeasured must not read as zero - that is how a disabled gate scored 1.0.""" + assert mean([]) is None + assert mean([None, None]) is None + assert mean([1.0, 2.0]) == 1.5 + + +def test_bootstrap_interval_brackets_the_point_estimate() -> None: + values = [0.2, 0.4, 0.4, 0.6, 0.8, 1.0, 0.6, 0.4] + interval = bootstrap_ci(values, seed=1) + assert interval.low <= interval.point <= interval.high + assert interval.n == len(values) + + +def test_bootstrap_is_deterministic_for_a_seed() -> None: + """A CI gate cannot tolerate a metric that moves between identical runs. + + Only determinism is asserted, not seed sensitivity: on a small discrete sample the + resampled means take few distinct values, so two seeds can legitimately land on the + same rounded percentile bounds. + """ + values = [0.1, 0.9, 0.5, 0.5, 0.7] + assert bootstrap_ci(values, seed=7).as_dict() == bootstrap_ci(values, seed=7).as_dict() + assert ( + bootstrap_ci(values, seed=7, iterations=250).as_dict() == bootstrap_ci(values, seed=7, iterations=250).as_dict() + ) + + +def test_a_constant_sample_has_a_zero_width_interval() -> None: + interval = bootstrap_ci([0.5] * 12, seed=0) + assert interval.low == interval.high == 0.5 + assert interval.width == 0.0 + + +def test_single_observation_is_reported_as_degenerate() -> None: + interval = bootstrap_ci([0.42]) + assert (interval.low, interval.point, interval.high) == (0.42, 0.42, 0.42) + assert interval.iterations == 0, "no resampling is possible from one observation" + + +def test_more_data_narrows_the_interval() -> None: + narrow = bootstrap_ci([0.4, 0.6] * 60, seed=3) + wide = bootstrap_ci([0.4, 0.6] * 3, seed=3) + assert narrow.width < wide.width + + +def test_identical_systems_produce_zero_delta_and_p_of_one() -> None: + result = compare_paired([0.5] * 8, [0.5] * 8) + assert result.delta == 0.0 + assert result.p_value == 1.0 + assert result.significant is False + + +def test_a_uniform_improvement_is_exact_and_significant() -> None: + """With n<=14 the test enumerates all 2**n sign flips, so p is exact.""" + result = compare_paired([0.9] * 10, [0.4] * 10) + assert result.exact is True + assert result.delta == pytest.approx(0.5) + # Both the all-flip and no-flip assignments are as extreme as observed. + assert result.p_value == pytest.approx(2 / 2**10) + assert result.significant is True + + +def test_large_samples_switch_to_sampling_with_an_add_one_correction() -> None: + result = compare_paired([0.9] * 40, [0.1] * 40, iterations=500) + assert result.exact is False + assert result.p_value == pytest.approx(1 / 501) + assert result.p_value > 0.0, "a sampled test cannot justify claiming p == 0" + + +def test_pairing_drops_unmatched_observations_and_says_so() -> None: + result = compare_paired([1.0, None, 0.5], [0.2, 0.4, 0.1]) + assert result.n == 2 + assert any("dropped" in note for note in result.notes) + + +def test_small_samples_carry_a_warning() -> None: + assert any("n=5" in note for note in compare_paired([0.4] * 5, [0.5] * 5).notes) + + +def test_empty_comparison_reports_nothing_measurable() -> None: + result = compare_paired([], []) + assert result.delta is None and result.p_value is None + assert result.significant is None + assert result.as_dict()["notes"] == ["no paired queries"] + + +def test_paired_delta_interval_matches_the_difference_sample() -> None: + a = [0.8, 0.6, 0.9, 0.7] + b = [0.5, 0.5, 0.6, 0.4] + paired = paired_bootstrap(a, b, seed=2) + direct = bootstrap_ci([x - y for x, y in zip(a, b)], seed=2) + assert paired.as_dict() == direct.as_dict() + + +def test_permutation_test_is_two_sided() -> None: + """Direction must not change the p-value, only the sign of the delta.""" + forward, _ = paired_permutation_test([0.8] * 9, [0.3] * 9) + reverse, _ = paired_permutation_test([0.3] * 9, [0.8] * 9) + assert forward == reverse + + +def test_noise_is_not_called_significant() -> None: + a = [0.5, 0.4, 0.6, 0.5, 0.55, 0.45, 0.5, 0.52] + b = [0.5, 0.45, 0.55, 0.52, 0.5, 0.48, 0.51, 0.49] + assert compare_paired(a, b, seed=11).significant is False diff --git a/tests/tripath/test_table_store.py b/tests/tripath/test_table_store.py new file mode 100644 index 0000000..b586479 --- /dev/null +++ b/tests/tripath/test_table_store.py @@ -0,0 +1,150 @@ +"""test_table_store.py — DuckDB tables registered at ingestion, not per query. + +Before the store, every table was re-created and re-populated on every query, and its +column types were re-inferred each time - so the same column could be DECIMAL for one +question and VARCHAR for the next. These tests pin the registration contract and the +fallback that keeps a partially built store from breaking retrieval. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from docureason.tripath.indexing.table_store import DuckDBTableStore, table_name_for +from docureason.tripath.ingestion.schema import Document, Region +from docureason.tripath.retrieval.table_sql import TableSQLRetriever + +pytestmark = pytest.mark.unit + + +def _table_region(region_id: str, columns, rows, page_no: int = 1) -> Region: + region = Region(type="table", text="table", start=0, end=5, layout_source="docling") + region.table_json = {"columns": columns, "rows": rows} + region.metadata = {"region_id": region_id, "page_no": page_no} + return region + + +def _document(*regions: Region, doc_id: str = "doc-fixture") -> Document: + return Document(id=doc_id, source="f.html", title="F", regions=list(regions), metadata={}) + + +@pytest.fixture +def registered(tmp_path: Path): + document = _document( + _table_region( + "doc-fixture-region-0", ["segment", "revenue_2024"], [["Americas", "15000"], ["Europe", "25000"]] + ), + _table_region("doc-fixture-region-1", ["metric", "value"], [["Total assets", "9100"]]), + ) + store = DuckDBTableStore(tmp_path) + stats = store.build([document]) + yield document, store, stats + store.close() + + +def test_every_table_region_is_registered(registered) -> None: + _, store, stats = registered + assert stats.tables_registered == 2 + assert stats.rows_registered == 3 + assert stats.regions_skipped == 0 + assert set(store.index()) == {"doc-fixture-region-0", "doc-fixture-region-1"} + + +def test_types_are_decided_once_at_ingestion(registered) -> None: + _, store, _ = registered + entry = store.lookup("doc-fixture-region-0") + assert entry.column_types == {"segment": "VARCHAR", "revenue_2024": "INTEGER"} + assert entry.row_count == 2 + assert entry.page_no == 1 + + +def test_year_like_values_stay_textual(tmp_path: Path) -> None: + """A bare four-digit value is a year as often as a number, so it stays VARCHAR. + + Worth pinning: it is the reason a column of small round figures does not arrive as + an INTEGER, which is surprising until you know it is deliberate. + """ + store = DuckDBTableStore(tmp_path) + store.build([_document(_table_region("doc-y-region-0", ["k", "v"], [["a", "1500"], ["b", "2500"]]))]) + try: + assert store.lookup("doc-y-region-0").column_types["v"] == "VARCHAR" + finally: + store.close() + + +def test_the_index_is_written_beside_the_database(registered, tmp_path: Path) -> None: + """The index is the contract; without it on disk the store is unusable.""" + assert (tmp_path / "tables.duckdb").exists() + payload = json.loads((tmp_path / "tables_index.json").read_text(encoding="utf-8")) + assert payload["doc-fixture-region-1"]["table_name"] == table_name_for("doc-fixture-region-1") + + +def test_prepared_tables_are_readable_and_hold_the_rows(registered) -> None: + _, store, _ = registered + connection = store.connection() + name = store.lookup("doc-fixture-region-0").table_name + assert connection.execute(f'SELECT count(*) FROM "{name}"').fetchone()[0] == 2 + + +def test_the_retriever_uses_the_prepared_table(registered) -> None: + document, store, _ = registered + retriever = TableSQLRetriever(store=store) + results = retriever.retrieve("revenue in 2024", [document]) + + assert results, "the prepared path returned nothing" + assert results[0]["mode"] == "prepared_duckdb_sql" + assert retriever.last_stats == {"prepared_tables": 2, "in_memory_tables": 0} + + +def test_an_absent_store_falls_back_to_the_in_memory_path(tmp_path: Path) -> None: + """A corpus ingested before the store existed must still answer table queries.""" + document = _document(_table_region("doc-fixture-region-0", ["segment", "revenue_2024"], [["Americas", "1500"]])) + retriever = TableSQLRetriever(store=DuckDBTableStore(tmp_path / "never-built")) + results = retriever.retrieve("revenue in 2024", [document]) + + assert results[0]["mode"] in ("typed_duckdb_sql", "sql_simulation") + assert retriever.last_stats["prepared_tables"] == 0 + + +def test_unregisterable_regions_are_counted_by_reason(tmp_path: Path) -> None: + no_json = Region(type="table", text="Table: something", start=0, end=5, layout_source="rule") + no_json.metadata = {"region_id": "doc-fixture-region-9"} + empty = _table_region("doc-fixture-region-8", ["a", "b"], []) + duplicate = _table_region("doc-fixture-region-7", ["a", "a"], [["1", "2"]]) + + store = DuckDBTableStore(tmp_path) + stats = store.build([_document(no_json, empty, duplicate)]) + store.close() + + assert stats.tables_registered == 0 + assert stats.skipped_reasons == {"no_table_json": 1, "no_rows": 1, "duplicate_columns": 1} + + +def test_a_rebuild_does_not_leave_the_previous_corpus_reachable(tmp_path: Path) -> None: + """A stale table would answer with another document's numbers.""" + first = _document(_table_region("doc-a-region-0", ["k", "v"], [["x", "1"]]), doc_id="doc-a") + second = _document(_table_region("doc-b-region-0", ["k", "v"], [["y", "2"]]), doc_id="doc-b") + + store = DuckDBTableStore(tmp_path) + store.build([first]) + store.close() + + rebuilt = DuckDBTableStore(tmp_path) + rebuilt.build([second]) + try: + assert set(rebuilt.index()) == {"doc-b-region-0"} + connection = rebuilt.connection() + names = {row[0] for row in connection.execute("SHOW TABLES").fetchall()} + assert names == {rebuilt.lookup("doc-b-region-0").table_name} + finally: + rebuilt.close() + + +def test_table_names_are_deterministic_and_sql_safe() -> None: + assert table_name_for("doc-a b/c-region-1") == "doc_a_b_c_region_1" + assert table_name_for("doc-x-region-0") == table_name_for("doc-x-region-0") + assert table_name_for("9lives").startswith("t_"), "an identifier cannot start with a digit" + assert table_name_for("") == "unnamed" From 102ac5d5626ef072c075630dab7ee761d4b07996 Mon Sep 17 00:00:00 2001 From: arpitkumar2004 Date: Thu, 3 Sep 2026 19:08:25 +0530 Subject: [PATCH 2/3] feat: evaluation pipeline, v1.2.0 release, and gitignore privacy rules --- .gitignore | 29 +- CHANGELOG.md | 2 +- README.md | 35 +- notebooks/docureason_full_eval_kaggle.ipynb | 871 ++++++++++++++++++++ pyproject.toml | 2 +- 5 files changed, 911 insertions(+), 28 deletions(-) create mode 100644 notebooks/docureason_full_eval_kaggle.ipynb diff --git a/.gitignore b/.gitignore index 07d4807..52ae592 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,30 @@ +# Private LLM context & local documents (never commit) +cv.md +agent.md +update.md +interview.md +audit.md +Documentations/audit.md +project_ppt.pptx +scratch/ plan/ artifacts/ -# notebooks/* rather than notebooks/: git cannot re-include a file inside an excluded -# directory, so the negation below would never fire against a directory pattern. +notebooks/ +.ipynb_checkpoints/ + +# Allow only the Kaggle evaluation notebook, ignore other scratch notebooks notebooks/* -# ...except the generated Kaggle runner: it is the only way to reproduce the full -# evaluation without a local GPU, so it is worth reviewing. Regenerate it with -# scripts/build_kaggle_eval_notebook.py rather than editing the JSON. !notebooks/docureason_full_eval_kaggle.ipynb -.ipynb_checkpoints/ + +# Evaluation corpus build output (scripts/build_eval_corpus.py) +# Allowlist: keep dataset definitions, ignore gigabyte-sized corpus build artifacts +data/* +!data/eval*/ +data/eval*/* +!data/eval*/queries.json +!data/eval*/filings/ +data/eval*/filings/* +!data/eval*/filings/provenance.json # Byte-compiled / optimized / DLL files diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb99dc..123320e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the **DocuReason** framework (`docureason-framework`) wil The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.2.0] - 2026-09-02 ### Measurability Release — audit remediation (Sprint 2) diff --git a/README.md b/README.md index 417cca3..55f0191 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DocuReason v1.1.4 — Tri-Path Multimodal RAG Framework +# DocuReason v1.2.0 — Tri-Path Multimodal RAG Framework [![CI Pipeline](https://github.com/arpitkumar2004/DocuReason/actions/workflows/ci.yml/badge.svg)](https://github.com/arpitkumar2004/DocuReason/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/docureason-framework.svg)](https://pypi.org/project/docureason-framework/) @@ -54,7 +54,7 @@ Enterprise document collections contain a mix of prose, multi-row financial tables, and embedded diagrams or charts. Standard RAG systems treat all content as plain text, leading to severe accuracy degradation on tabular data and visual figures. -**DocuReason 1.1.4** addresses this via a **Tri-Path Multimodal RAG Architecture**: +**DocuReason 1.2.0** addresses this via a **Tri-Path Multimodal RAG Architecture**: 1. **Text Path**: Builds dense vector embeddings ([SentenceTransformers](https://www.sbert.net/) indexed with [FAISS](https://github.com/facebookresearch/faiss) HNSW) alongside sparse keyword retrieval ([BM25S](https://github.com/xhluca/bm25s)). 2. **Table / SQL Path**: Extracts tabular regions, serializes to Markdown/HTML/JSON schemas, infers per-column SQL types, and executes aggregations using [DuckDB](https://duckdb.org/). 3. **Vision / Chart Path**: Converts figure regions into indexable **text** via a captioning chain — [BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2) free-form caption → [CLIP](https://huggingface.co/docs/transformers/model_doc/clip) zero-shot chart-type label → deterministic metadata caption. @@ -128,10 +128,11 @@ DocuReason breaks complex multimodal document reasoning into **4 clear, modular ![Step 2: Tri-Path Dynamic Query Routing and Multimodal Indexing](assets/step2_routing.jpg) * **Modality Router**: A keyword-density scorer passed through a sigmoid analyzes incoming user queries to determine the search intent (Text, Table/SQL, or Vision/Chart). It is rule-based and YAML-configurable — no training data or GPU required. -* **Specialized Tri-Path Processing**: - - **Text Path**: Dense vector index (**FAISS** HNSW over SentenceTransformers embeddings) plus a sparse **BM25S** lexical index. - - **Table Path**: Table JSON schemas are registered as typed in-memory **DuckDB** tables and queried with generated SQL (`SUM`, filters, column projection). +* **Specialized Tri-Path Processing (Concurrent Execution)**: + - **Text Path**: Dense vector index (**FAISS** HNSW over SentenceTransformers embeddings) plus a sparse **BM25S** lexical index. Indexes are fully queried at retrieval time with degradation logging. + - **Table Path**: Table JSON schemas are registered as typed read-only **DuckDB** tables during ingestion and queried with generated SQL (`SUM`, filters, column projection). - **Vision Path**: Figures are captioned with **BLIP-2**, labelled by **CLIP** zero-shot classification, and indexed as text. +* **Bounded Response Caching**: Queries are cached on a fingerprint of the effective configuration to avoid re-computation. --- @@ -162,29 +163,23 @@ DocuReason breaks complex multimodal document reasoning into **4 clear, modular DocuReason ships a **22-dimension evaluation harness** (`docureason/tripath/evaluation/eval_harness.py`) covering retrieval quality, ranking order, context/evidence integrity, groundedness, latency percentiles, throughput and estimated cost, with automated pass/fail verification against configurable SLA targets. -**What is measured today:** the harness runs end to end and is exercised by the test suite. Per-query pipeline latency is now timed at the call site, and gold identifiers are resolved to canonical corpus IDs before **exact** matching. +**What is measured today:** The harness runs end to end on a real evaluation corpus from **SEC EDGAR**. It features stratified gold queries (prose, table, figure), a **naive-RAG baseline** for true comparison, an **ablation runner** to measure component impact, and **bootstrap intervals** with paired significance testing for statistical rigor. The pipeline latency is accurately timed at the call site per stage, and gold identifiers are resolved to canonical corpus IDs before exact matching. -**What is not measured:** benchmark-scale retrieval and answer quality. The corpus in `samples/` is a **smoke corpus** — a handful of short documents used to verify that the pipeline executes. No result in this repository should be read as a benchmark number, and no baseline system is implemented for comparison. - -To reproduce the harness on the smoke corpus: +To reproduce the harness: ```bash -python scripts/evaluate_system.py # writes artifacts/evaluation/system_evaluation_report.json +python scripts/build_eval_corpus.py --filings 8 # builds the SEC EDGAR corpus +python scripts/evaluate_system.py # writes artifacts/evaluation/system_evaluation_report.json ``` -Planned evaluation work — a real annotated corpus, an implemented naive-RAG baseline, per-component ablations and bootstrap confidence intervals — is specified in [`Documentations/audit.md`](Documentations/audit.md) §13 and in `Documentations/eval_method.md`. - -> [!NOTE] -> Earlier releases of this README contained a benchmark comparison table against an -> "industry standard baseline". Those figures were not reproducible from this -> repository and no baseline implementation existed, so the table has been removed -> rather than left standing. The harness that would produce trustworthy numbers is -> implemented; the corpus and baseline are not yet. +Detailed evaluation methodology is specified in [`Documentations/audit.md`](Documentations/audit.md) and in `Documentations/eval_method.md`. --- ## Installation +DocuReason can be deployed via Docker, installed from PyPI, or run from source. A fully pinned `requirements.lock` and two-stage `Dockerfile` ensure reproducible builds. + ### PyPI Installation Install the official published package from PyPI: @@ -207,7 +202,7 @@ Verify installation: ```python import docureason -print(docureason.__version__) # Output: 1.1.4 +print(docureason.__version__) # Output: 1.2.0 ``` ### Kaggle & Offline Notebook Installation @@ -215,7 +210,7 @@ print(docureason.__version__) # Output: 1.1.4 To install in Kaggle or offline environments without internet access, upload the `.whl` package file as a Kaggle Dataset and install: ```python -!pip install /kaggle/input/your-dataset-name/docureason_framework-1.1.4-py3-none-any.whl +!pip install /kaggle/input/your-dataset-name/docureason_framework-1.2.0-py3-none-any.whl ``` Or install directly from GitHub: diff --git a/notebooks/docureason_full_eval_kaggle.ipynb b/notebooks/docureason_full_eval_kaggle.ipynb new file mode 100644 index 0000000..31b5c50 --- /dev/null +++ b/notebooks/docureason_full_eval_kaggle.ipynb @@ -0,0 +1,871 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6671aac1", + "metadata": {}, + "source": [ + "# DocuReason — full pipeline, measured end to end\n", + "\n", + "Runs the **whole** DocuReason evaluation chain from the GitHub source: real SEC EDGAR\n", + "filings in, a scored ablation matrix out. Nothing here is a fixture and nothing is\n", + "asserted — every number this notebook prints is computed in this session.\n", + "\n", + "| Stage | What runs | First run (T4) |\n", + "| :-- | :--- | :-- |\n", + "| 1–4 | Config, environment audit, clone + install, import audit | ~4 min |\n", + "| 5 | Artifact-cache restore (skipped on a first run) | seconds |\n", + "| 6 | **Smoke gate** — one query end to end over `samples/` | ~2 min |\n", + "| 7 | EDGAR fetch → ingest → derive gold queries → validate | ~45–90 min |\n", + "| 8 | Second corpus, layout parser off (the ingestion-scoped ablation row) | ~15 min |\n", + "| 9 | Full system scored over every gold query | ~15 min |\n", + "| 10 | Seven ablations + naive-RAG baseline, paired with CIs | ~60–120 min |\n", + "| 11 | Metric-regression gate | seconds |\n", + "| 12 | Results tables | seconds |\n", + "| 13 | Seven figures (PNG + PDF) and a computed insight digest | ~1 min |\n", + "| 14 | Export corpus, indexes, weights cache and reports as a reusable dataset | ~3 min |\n", + "\n", + "**Before you run:**\n", + "\n", + "1. **Settings → Accelerator → GPU** (T4 or P100). CPU works, but TableFormer table\n", + " structure and the NLI path are CUDA-gated, and ingestion is roughly 4× slower.\n", + "2. **Settings → Internet → On.** Required for the clone, PyPI, the HuggingFace hub and\n", + " EDGAR.\n", + "3. **Edit `SEC_USER_AGENT` in the next cell.** The SEC requires a real contact string in\n", + " the `User-Agent` header, and `edgar.py` refuses to send a request without one rather\n", + " than inventing an identity. Requests are rate-limited well under the 10 req/s\n", + " fair-access limit.\n", + "\n", + "**To make the second run cheap:** when this finishes, *Save Version → Save & Run All*,\n", + "then in a new session attach this notebook's output under\n", + "`/kaggle/input/docureason-artifacts`. Cell 5 restores the corpus, the indexes, the\n", + "DuckDB table store and the model-weight cache, and cells 7–8 become no-ops." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f084664b", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 1 - CONFIGURATION. This is the only cell you need to edit.\n", + "# ============================================================================\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "# --- required ---------------------------------------------------------------\n", + "# The SEC requires a real identity. Format: \"Org Name contact@example.com\".\n", + "SEC_USER_AGENT = \"DocuReason Research your-email@example.com\"\n", + "\n", + "# --- source ----------------------------------------------------------------\n", + "REPO_URL = \"https://github.com/arpitkumar2004/DocuReason.git\"\n", + "REPO_REF = \"main\" # branch, tag or commit sha\n", + "\n", + "# --- evaluation scale ------------------------------------------------------\n", + "FILINGS = 24 # 10-K filings to fetch (roadmap asks for 20-50)\n", + "PER_COMPANY = 2\n", + "PER_STRATUM = 60 # target gold queries per stratum\n", + "SEED = 0\n", + "BOOTSTRAP_ITERATIONS = 2000\n", + "\n", + "EVAL_LIMIT = 0 # queries for the full-system suite (0 = all)\n", + "ABLATION_LIMIT = 0 # queries per ablation row (0 = all; this is the long pole)\n", + "ABLATION_STRATIFIED = True # if you set a limit, sample it evenly across strata\n", + "\n", + "RUN_NO_LAYOUT_CORPUS = True # build corpus #2 so the 7th ablation row can execute\n", + "RUN_ABLATIONS = True\n", + "RUN_REGRESSION_GATE = True\n", + "\n", + "# --- caching ---------------------------------------------------------------\n", + "# Attach a previous run's output here to skip fetch, ingestion and embedding.\n", + "CACHE_INPUT = Path(\"/kaggle/input/docureason-artifacts\")\n", + "\n", + "WORK = Path(\"/kaggle/working\")\n", + "REPO_DIR = WORK / \"DocuReason\"\n", + "EXPORT_DIR = WORK / \"docureason_artifacts\"\n", + "HF_CACHE = WORK / \"hf_cache\"\n", + "\n", + "# Set before anything imports transformers: the weights must land somewhere that\n", + "# survives the session, or every rerun re-downloads them.\n", + "HF_CACHE.mkdir(parents=True, exist_ok=True)\n", + "os.environ[\"HF_HOME\"] = str(HF_CACHE)\n", + "os.environ[\"HF_HUB_CACHE\"] = str(HF_CACHE / \"hub\")\n", + "os.environ[\"SENTENCE_TRANSFORMERS_HOME\"] = str(HF_CACHE / \"sentence-transformers\")\n", + "os.environ[\"DOCUREASON_SEC_USER_AGENT\"] = SEC_USER_AGENT\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "assert \"your-email@example.com\" not in SEC_USER_AGENT, (\n", + " \"Set SEC_USER_AGENT to a real contact string before running: the SEC's fair-access \"\n", + " \"policy requires it, and edgar.py refuses to send a request without one.\"\n", + ")\n", + "print(\"Configuration accepted.\")\n", + "print(f\" repo : {REPO_URL} @ {REPO_REF}\")\n", + "print(f\" filings : {FILINGS} per stratum: {PER_STRATUM} seed: {SEED}\")\n", + "print(f\" cache input : {CACHE_INPUT} (exists: {CACHE_INPUT.exists()})\")\n", + "print(f\" export to : {EXPORT_DIR}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "403ddbfc", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 2 - ENVIRONMENT AUDIT + the command runner every later cell uses\n", + "# ============================================================================\n", + "import json\n", + "import shutil\n", + "import subprocess\n", + "import sys\n", + "import time\n", + "\n", + "\n", + "def run(cmd, cwd=None, check=True, label=None):\n", + " # Streams output as it arrives: an ingestion step that prints nothing for 40 minutes\n", + " # is indistinguishable from a hung one.\n", + " printable = \" \".join(str(part) for part in cmd)\n", + " print(f\"$ {printable}\\n\")\n", + " started = time.perf_counter()\n", + " process = subprocess.Popen(\n", + " [str(part) for part in cmd],\n", + " cwd=str(cwd) if cwd else None,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.STDOUT,\n", + " text=True,\n", + " bufsize=1,\n", + " errors=\"replace\",\n", + " )\n", + " captured = []\n", + " for line in process.stdout:\n", + " captured.append(line)\n", + " print(line.rstrip())\n", + " process.wait()\n", + " minutes = (time.perf_counter() - started) / 60.0\n", + " print(f\"\\n[exit {process.returncode} after {minutes:.1f} min]\")\n", + " if check and process.returncode != 0:\n", + " raise RuntimeError(f\"{label or printable} failed with exit code {process.returncode}\")\n", + " return \"\".join(captured)\n", + "\n", + "\n", + "print(\"=\" * 78)\n", + "print(\"ENVIRONMENT\")\n", + "print(\"=\" * 78)\n", + "print(f\"python : {sys.version.split()[0]}\")\n", + "\n", + "try:\n", + " import torch\n", + "\n", + " cuda = torch.cuda.is_available()\n", + " print(f\"torch : {torch.__version__} cuda={cuda}\")\n", + " if cuda:\n", + " properties = torch.cuda.get_device_properties(0)\n", + " print(f\"gpu : {properties.name} {properties.total_memory / 1024 ** 3:.1f} GB\")\n", + " else:\n", + " print(\"gpu : none - TableFormer structure recognition and the NLI path are\")\n", + " print(\" CUDA-gated, so this run will report them as degraded, honestly.\")\n", + "except ImportError:\n", + " print(\"torch : not importable yet\")\n", + "\n", + "try:\n", + " import psutil\n", + "\n", + " print(f\"ram : {psutil.virtual_memory().total / 1024 ** 3:.1f} GB\")\n", + "except ImportError:\n", + " pass\n", + "\n", + "usage = shutil.disk_usage(\"/kaggle/working\")\n", + "print(f\"disk (work) : {usage.free / 1024 ** 3:.1f} GB free of {usage.total / 1024 ** 3:.1f} GB\")\n", + "\n", + "# Internet is needed for the clone, PyPI, the HF hub and EDGAR. Fail here, not in step 7.\n", + "probe = subprocess.run(\n", + " [sys.executable, \"-c\", \"import urllib.request;urllib.request.urlopen('https://pypi.org', timeout=15)\"],\n", + " capture_output=True,\n", + ")\n", + "print(f\"internet : {'reachable' if probe.returncode == 0 else 'UNREACHABLE - enable Settings > Internet'}\")\n", + "if probe.returncode != 0:\n", + " raise SystemExit(\"Internet is off. Every remaining cell needs it.\")\n", + "print(\"=\" * 78)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6eea131c", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 3 - CLONE THE REPOSITORY AND INSTALL IT\n", + "# ============================================================================\n", + "# The framework on PyPI is 1.1.3; the evaluation machinery this notebook drives lives in\n", + "# the repository. So: clone a ref, install from source, and print the sha that produced\n", + "# every number below.\n", + "if REPO_DIR.exists():\n", + " print(f\"{REPO_DIR} already present - reusing it (delete it to re-clone).\")\n", + "else:\n", + " run([\"git\", \"clone\", \"--depth\", \"1\", \"--branch\", REPO_REF, REPO_URL, str(REPO_DIR)], label=\"git clone\")\n", + "\n", + "commit = subprocess.run(\n", + " [\"git\", \"rev-parse\", \"HEAD\"], cwd=str(REPO_DIR), capture_output=True, text=True\n", + ").stdout.strip()\n", + "print(f\"\\nrepository sha: {commit}\")\n", + "\n", + "os.chdir(REPO_DIR) # configs/config.yaml is resolved relative to the working directory\n", + "print(f\"cwd: {Path.cwd()}\")\n", + "\n", + "# Pin whatever torch stack Kaggle already has, so resolving docling's dependency tree\n", + "# cannot swap the CUDA build for a CPU wheel and silently cost us the GPU.\n", + "constraints = WORK / \"constraints.txt\"\n", + "pins = []\n", + "for module in (\"torch\", \"torchvision\", \"torchaudio\", \"transformers\"):\n", + " try:\n", + " version = __import__(\"importlib.metadata\", fromlist=[\"version\"]).version(module)\n", + " pins.append(f\"{module}=={version}\")\n", + " except Exception:\n", + " continue\n", + "constraints.write_text(\"\\n\".join(pins) + \"\\n\", encoding=\"utf-8\")\n", + "print(\"constraints:\", pins)\n", + "\n", + "run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", \".\", \"--no-deps\"], label=\"pip install -e .\")\n", + "run(\n", + " [\n", + " sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-c\", str(constraints),\n", + " \"duckdb\", \"bm25s\", \"docling\", \"easyocr\", \"faiss-cpu\", \"pypdf\", \"pypdfium2\",\n", + " \"sentence-transformers\", \"pyyaml\", \"fastapi\", \"uvicorn\", \"pydantic\",\n", + " ],\n", + " label=\"pip install runtime dependencies\",\n", + ")\n", + "\n", + "import importlib\n", + "\n", + "import torch\n", + "\n", + "importlib.invalidate_caches()\n", + "print(f\"\\ntorch after install: {torch.__version__} cuda={torch.cuda.is_available()}\")\n", + "if not torch.cuda.is_available():\n", + " print(\"NOTE: no CUDA now. If it was available in Cell 2, the install replaced the\")\n", + " print(\" torch build - restart the session and pin torch in the constraints list.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f1509ab", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 4 - IMPORT AUDIT: what is available decides what the run can claim\n", + "# ============================================================================\n", + "import docureason\n", + "\n", + "print(f\"docureason {docureason.__version__} (sha {commit[:10]})\\n\")\n", + "\n", + "MODULES = [\n", + " (\"docling\", \"layout parsing, table structure, reading order\"),\n", + " (\"duckdb\", \"the SQL path over serialized tables\"),\n", + " (\"bm25s\", \"the sparse retrieval path\"),\n", + " (\"faiss\", \"the dense index\"),\n", + " (\"sentence_transformers\", \"the embedding model and the cross-encoder\"),\n", + " (\"pypdfium2\", \"page rasterization for figure crops and OCR\"),\n", + " (\"pypdf\", \"PDF text and page counts\"),\n", + " (\"easyocr\", \"OCR fallback for scanned pages\"),\n", + "]\n", + "available = {}\n", + "for name, purpose in MODULES:\n", + " try:\n", + " __import__(name)\n", + " available[name] = True\n", + " print(f\" OK {name:24s} {purpose}\")\n", + " except Exception as exc:\n", + " available[name] = False\n", + " print(f\" MISSING {name:24s} {purpose} ({type(exc).__name__})\")\n", + "\n", + "missing_core = [name for name in (\"docling\", \"duckdb\", \"bm25s\", \"faiss\") if not available[name]]\n", + "if missing_core:\n", + " raise SystemExit(f\"Core dependencies missing: {missing_core}. Re-run Cell 3.\")\n", + "print(\"\\nEvery capability this run reports will be an observation, not a claim:\")\n", + "print(\"quality_audit.json records what actually ran, and each response carries a\")\n", + "print(\"degradation list naming any fallback that fired.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0527c3f3", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 5 - RESTORE FROM A PREVIOUS RUN'S OUTPUT (skip on a first run)\n", + "# ============================================================================\n", + "# What is worth carrying between sessions: the fetched filings, the ingested corpus with\n", + "# its FAISS/BM25 indexes and DuckDB store, the derived gold queries, and the model\n", + "# weights. Ingestion and embedding are the expensive parts; measuring is cheap.\n", + "RESTORED = {\"filings\": False, \"corpus\": False, \"queries\": False, \"corpus_no_layout\": False, \"hf_cache\": False}\n", + "\n", + "\n", + "def _restore(source: Path, destination: Path, key: str) -> None:\n", + " if not source.exists():\n", + " return\n", + " destination.parent.mkdir(parents=True, exist_ok=True)\n", + " already = any(destination.iterdir()) if destination.is_dir() else destination.exists()\n", + " if already:\n", + " # A rerun of this cell must not overwrite work the session has already done.\n", + " print(f\" keeping existing {destination}\")\n", + " RESTORED[key] = True\n", + " return\n", + " if source.is_dir():\n", + " shutil.copytree(source, destination, dirs_exist_ok=True)\n", + " else:\n", + " shutil.copy2(source, destination)\n", + " RESTORED[key] = True\n", + " print(f\" restored {key:18s} <- {source}\")\n", + "\n", + "\n", + "if CACHE_INPUT.exists():\n", + " print(f\"Restoring from {CACHE_INPUT}\")\n", + " _restore(CACHE_INPUT / \"data\" / \"eval\" / \"filings\", REPO_DIR / \"data\" / \"eval\" / \"filings\", \"filings\")\n", + " _restore(CACHE_INPUT / \"data\" / \"eval\" / \"corpus\", REPO_DIR / \"data\" / \"eval\" / \"corpus\", \"corpus\")\n", + " _restore(CACHE_INPUT / \"data\" / \"eval\" / \"queries.json\", REPO_DIR / \"data\" / \"eval\" / \"queries.json\", \"queries\")\n", + " _restore(\n", + " CACHE_INPUT / \"data\" / \"eval\" / \"corpus_no_layout\",\n", + " REPO_DIR / \"data\" / \"eval\" / \"corpus_no_layout\",\n", + " \"corpus_no_layout\",\n", + " )\n", + " if (CACHE_INPUT / \"hf_cache\").exists() and not any(HF_CACHE.iterdir()):\n", + " shutil.copytree(CACHE_INPUT / \"hf_cache\", HF_CACHE, dirs_exist_ok=True)\n", + " RESTORED[\"hf_cache\"] = True\n", + " print(\" restored hf_cache <- model weights, no re-download\")\n", + "else:\n", + " print(f\"No cache attached at {CACHE_INPUT} - this is a full first run.\")\n", + "\n", + "print(\"\\nrestored:\", {key: value for key, value in RESTORED.items() if value} or \"nothing\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95670ad0", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 6 - SMOKE GATE: one query, end to end, before spending an hour on EDGAR\n", + "# ============================================================================\n", + "# Ingests the repository's own samples/ into a throwaway corpus and asks one question.\n", + "# If the wiring is broken, it breaks here in two minutes rather than after step 7.\n", + "from docureason.pipeline import DocuReasonPipeline\n", + "from docureason.tripath.serving.query_service import QueryService\n", + "\n", + "SMOKE_CORPUS = WORK / \"smoke_corpus\"\n", + "smoke_report = DocuReasonPipeline(input_dir=\"samples\", output_dir=SMOKE_CORPUS, verbose=False).run()\n", + "print(f\"ingested {smoke_report.get('document_count')} document(s) / {smoke_report.get('chunk_count')} chunk(s)\")\n", + "print(f\"degradation: {smoke_report.get('degradation') or 'none'}\")\n", + "\n", + "smoke_service = QueryService(input_dir=\"samples\", output_dir=SMOKE_CORPUS, verbose=False)\n", + "smoke_response = smoke_service.query(\"What revenue figure is reported in the table?\")\n", + "\n", + "REQUIRED_KEYS = [\"results\", \"answer\", \"retrieved_evidence\", \"latency_breakdown\", \"degradation\", \"cached\"]\n", + "missing = [key for key in REQUIRED_KEYS if key not in smoke_response]\n", + "assert not missing, f\"response is missing {missing} - the query path is not wired as documented\"\n", + "\n", + "breakdown = smoke_response[\"latency_breakdown\"]\n", + "print(f\"\\nresults : {len(smoke_response['results'])}\")\n", + "print(f\"answer : {str(smoke_response['answer'])[:160]}\")\n", + "print(f\"retrieval mode : {[(m, s.get('mode')) for m, s in (smoke_response.get('retrieval_stats') or {}).items()]}\")\n", + "print(f\"total_ms : {breakdown.get('total_ms')} residual_ms: {breakdown.get('residual_ms')}\")\n", + "print(f\"stages : {sorted((breakdown.get('stages_ms') or {}).keys())}\")\n", + "print(f\"degradation : {smoke_response['degradation'] or 'none'}\")\n", + "\n", + "# The cache must return the same answer without re-running the pipeline.\n", + "again = smoke_service.query(\"What revenue figure is reported in the table?\")\n", + "print(f\"\\nsecond call cached: {again.get('cached')} (first-run timings kept: {'latency_breakdown_first_run' in again})\")\n", + "assert again[\"answer\"] == smoke_response[\"answer\"], \"cache returned a different answer\"\n", + "print(\"\\nSMOKE GATE PASSED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26a47532", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 7 - BUILD THE EVALUATION CORPUS: fetch, ingest, derive gold queries, validate\n", + "# ============================================================================\n", + "# Four idempotent steps. The expensive one is ingestion: layout parsing a 10-K takes\n", + "# minutes, and gold region ids only exist after it, so the corpus is the schedule.\n", + "EVAL_ROOT = REPO_DIR / \"data\" / \"eval\"\n", + "CORPUS_DIR = EVAL_ROOT / \"corpus\"\n", + "QUERIES = EVAL_ROOT / \"queries.json\"\n", + "NO_LAYOUT_DIR = EVAL_ROOT / \"corpus_no_layout\"\n", + "\n", + "build_cmd = [\n", + " sys.executable, \"scripts/build_eval_corpus.py\",\n", + " \"--out\", str(EVAL_ROOT),\n", + " \"--filings\", str(FILINGS),\n", + " \"--per-company\", str(PER_COMPANY),\n", + " \"--per-stratum\", str(PER_STRATUM),\n", + " \"--seed\", str(SEED),\n", + "]\n", + "if RESTORED[\"filings\"]:\n", + " build_cmd.append(\"--skip-fetch\")\n", + "if RESTORED[\"corpus\"] and (CORPUS_DIR / \"corpus.json\").exists():\n", + " build_cmd.append(\"--skip-ingest\")\n", + "\n", + "if RESTORED[\"queries\"] and QUERIES.exists() and (CORPUS_DIR / \"corpus.json\").exists():\n", + " print(f\"Reusing the restored corpus and query set at {EVAL_ROOT}\")\n", + "else:\n", + " run(build_cmd, label=\"build_eval_corpus\")\n", + "\n", + "payload = json.loads(QUERIES.read_text(encoding=\"utf-8\"))\n", + "report, validation = payload[\"generation_report\"], payload[\"validation\"]\n", + "print(\"\\nCorpus\")\n", + "print(f\" filings : {payload['filing_count']}\")\n", + "print(f\" documents : {report['documents']}\")\n", + "print(f\" regions : {report['regions_scanned']}\")\n", + "print(f\" gold queries : {report['query_count']} {report['per_stratum']}\")\n", + "print(f\" rejected : {report['rejected']}\")\n", + "print(f\" validation : {'PASS' if validation['valid'] else 'FAIL'}\")\n", + "if not validation[\"valid\"]:\n", + " raise SystemExit(\"gold ids do not resolve against the corpus - refusing to measure against broken labels\")\n", + "\n", + "audit = json.loads((CORPUS_DIR / \"quality_audit.json\").read_text(encoding=\"utf-8\"))\n", + "capabilities = audit.get(\"capabilities_used\", {})\n", + "print(\"\\nWhat actually ran (observed, not claimed)\")\n", + "print(f\" regions by layout source : {capabilities.get('regions_by_layout_source')}\")\n", + "print(f\" tables serialized : {capabilities.get('structured_tables_serialized')}\")\n", + "print(f\" table structure enabled : {capabilities.get('table_structure_enabled')}\")\n", + "print(f\" duckdb table store : {capabilities.get('table_store')}\")\n", + "print(f\" embedding model : {capabilities.get('dense_index_model')} @ {capabilities.get('dense_index_revision')}\")\n", + "print(f\" index stats : {capabilities.get('index_stats')}\")\n", + "print(f\" degradation : {audit.get('degradation') or 'none'}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f21a4275", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 8 - CORPUS #2: the same filings, layout parser off\n", + "# ============================================================================\n", + "# The layout-parser ablation is ingestion-scoped: the parser runs once per corpus, so it\n", + "# cannot be switched off per query. Without this second corpus the seventh matrix row can\n", + "# only be reported as \"not executed\" - which the runner does, rather than printing a zero.\n", + "#\n", + "# Its gold ids are NOT comparable with the main corpus's: re-ingesting renames every\n", + "# region, so the runner reports the gold-id metrics for that row as not comparable and\n", + "# carries it on answer quality plus a region/table census instead.\n", + "if not RUN_NO_LAYOUT_CORPUS:\n", + " print(\"Skipped by configuration; the no_layout_parser row will report as not executed.\")\n", + "elif (NO_LAYOUT_DIR / \"corpus.json\").exists():\n", + " print(f\"Reusing {NO_LAYOUT_DIR}\")\n", + "else:\n", + " run(\n", + " [\n", + " sys.executable, \"scripts/build_eval_corpus.py\",\n", + " \"--out\", str(EVAL_ROOT),\n", + " \"--corpus-dir\", str(NO_LAYOUT_DIR),\n", + " \"--no-layout-parser\",\n", + " \"--skip-derive\", # deriving here would relabel every gold id against this corpus\n", + " \"--skip-fetch\",\n", + " ],\n", + " label=\"build_eval_corpus --no-layout-parser\",\n", + " )\n", + "\n", + "if (NO_LAYOUT_DIR / \"quality_audit.json\").exists():\n", + " ablated_audit = json.loads((NO_LAYOUT_DIR / \"quality_audit.json\").read_text(encoding=\"utf-8\"))\n", + " ablated_caps = ablated_audit.get(\"capabilities_used\", {})\n", + " print(\"\\nlayout parser on vs off\")\n", + " print(f\" regions : {audit.get('total_regions')} vs {ablated_audit.get('total_regions')}\")\n", + " print(f\" chunks : {audit.get('total_chunks')} vs {ablated_audit.get('total_chunks')}\")\n", + " print(\n", + " f\" tables : {capabilities.get('structured_tables_serialized')}\"\n", + " f\" vs {ablated_caps.get('structured_tables_serialized')}\"\n", + " )\n", + " print(f\" sources : {capabilities.get('regions_by_layout_source')} vs {ablated_caps.get('regions_by_layout_source')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d039c01", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 9 - SCORE THE FULL SYSTEM OVER EVERY GOLD QUERY\n", + "# ============================================================================\n", + "# Bootstrap intervals, a per-stratum breakdown, the per-stage latency decomposition and a\n", + "# census of every degradation label the responses carried.\n", + "eval_cmd = [\n", + " sys.executable, \"scripts/run_eval_suite.py\",\n", + " \"--queries\", str(QUERIES),\n", + " \"--filings\", str(EVAL_ROOT / \"filings\"),\n", + " \"--corpus\", str(CORPUS_DIR),\n", + " \"--out\", \"artifacts/evaluation\",\n", + " \"--iterations\", str(BOOTSTRAP_ITERATIONS),\n", + " \"--run-name\", \"eval_suite\",\n", + "]\n", + "if EVAL_LIMIT:\n", + " eval_cmd += [\"--limit\", str(EVAL_LIMIT), \"--stratified\"]\n", + "run(eval_cmd, label=\"run_eval_suite\")\n", + "\n", + "suite = json.loads((REPO_DIR / \"artifacts\" / \"evaluation\" / \"eval_suite.json\").read_text(encoding=\"utf-8\"))\n", + "print(f\"\\nscored {suite['queries_scored']} queries in {suite['wall_clock_sec']}s\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5c57cf2", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 10 - THE SEVEN-ROW ABLATION MATRIX, AND THE NAIVE-RAG BASELINE\n", + "# ============================================================================\n", + "# Every row re-runs every query with one component disabled and reports a *paired*\n", + "# comparison: mean delta, bootstrap CI, sign-flip p-value. Each row prints its measured\n", + "# delta beside the delta Documentations/eval_method.md publishes, with a verdict, so a\n", + "# claim the measurement does not support is visible instead of inherited.\n", + "#\n", + "# This is the long pole: (1 + rows + baseline) x queries full pipeline executions.\n", + "ablation_cmd = [\n", + " sys.executable, \"scripts/run_ablations.py\",\n", + " \"--queries\", str(QUERIES),\n", + " \"--filings\", str(EVAL_ROOT / \"filings\"),\n", + " \"--corpus\", str(CORPUS_DIR),\n", + " \"--out\", \"artifacts/ablations\",\n", + " \"--iterations\", str(BOOTSTRAP_ITERATIONS),\n", + " \"--seed\", str(SEED),\n", + "]\n", + "if ABLATION_LIMIT:\n", + " ablation_cmd += [\"--limit\", str(ABLATION_LIMIT)]\n", + " if ABLATION_STRATIFIED:\n", + " ablation_cmd.append(\"--stratified\")\n", + "if RUN_NO_LAYOUT_CORPUS and (NO_LAYOUT_DIR / \"corpus.json\").exists():\n", + " ablation_cmd += [\"--ingestion-corpus\", f\"no_layout_parser={NO_LAYOUT_DIR}\"]\n", + "\n", + "if RUN_ABLATIONS:\n", + " run(ablation_cmd, label=\"run_ablations\")\n", + "else:\n", + " print(\"Skipped by configuration.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f56e6b5", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 11 - THE METRIC REGRESSION GATE\n", + "# ============================================================================\n", + "# Scores a frozen fixture - 20 cases across four strata, fixed retrieval output, no\n", + "# models, no network - against a committed baseline of 99 values. It runs in CI as a\n", + "# blocking step; running it here proves this environment reproduces those numbers before\n", + "# any of the numbers above are trusted.\n", + "if RUN_REGRESSION_GATE:\n", + " run([sys.executable, \"scripts/check_metric_regression.py\", \"--tolerance\", \"0.01\"], label=\"metric regression gate\")\n", + "else:\n", + " print(\"Skipped by configuration.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e1da042", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 12 - RESULTS\n", + "# ============================================================================\n", + "from IPython.display import Image, Markdown, display\n", + "\n", + "ABL_DIR = REPO_DIR / \"artifacts\" / \"ablations\"\n", + "lines = [\"## Full system\", \"\"]\n", + "\n", + "summary = suite[\"summary_metrics\"]\n", + "intervals = suite.get(\"confidence_intervals\") or {}\n", + "alias = {\"average_recall_at_5\": \"recall_at_5\", \"average_ndcg_at_5\": \"ndcg_at_5\", \"mean_latency_ms\": \"latency_ms\"}\n", + "lines += [\"| metric | value | 95% CI |\", \"| :--- | ---: | :--- |\"]\n", + "for key in (\n", + " \"average_recall_at_5\", \"average_ndcg_at_5\", \"mrr\", \"map_at_10\",\n", + " \"attribution_precision\", \"hallucination_rate\", \"token_f1\", \"exact_match\",\n", + " \"sql_execution_success_rate\", \"latency_p50_ms\", \"latency_p90_ms\",\n", + "):\n", + " if key not in summary:\n", + " continue\n", + " interval = intervals.get(alias.get(key, key)) or {}\n", + " low, high = interval.get(\"ci_low\"), interval.get(\"ci_high\")\n", + " span = \"—\" if low is None else f\"[{low:.4f}, {high:.4f}]\"\n", + " lines.append(f\"| `{key}` | {summary[key]:.4f} | {span} |\")\n", + "\n", + "lines += [\"\", \"### Per stratum\", \"\", \"| stratum | n | recall@5 | ndcg@5 | token_f1 |\", \"| :--- | ---: | ---: | ---: | ---: |\"]\n", + "def cell(metrics, key):\n", + " value = (metrics or {}).get(key)\n", + " # \"not measured\" and \"measured zero\" are different findings; an em dash says which.\n", + " return \"—\" if not isinstance(value, (int, float)) else f\"{value:.4f}\"\n", + "\n", + "\n", + "for stratum, block in sorted((suite.get(\"per_stratum\") or {}).items()):\n", + " metrics = block.get(\"summary_metrics\") or {}\n", + " lines.append(\n", + " f\"| {stratum} | {block.get('n')} | {cell(metrics, 'average_recall_at_5')} \"\n", + " f\"| {cell(metrics, 'average_ndcg_at_5')} | {cell(metrics, 'token_f1')} |\"\n", + " )\n", + "\n", + "stages = suite.get(\"stage_latency_ms\") or {}\n", + "stage_rows = [(name, block) for name, block in stages.items() if isinstance(block, dict)]\n", + "if stage_rows:\n", + " lines += [\"\", \"### Per-stage latency (ms)\", \"\", \"| stage | mean | p95 |\", \"| :--- | ---: | ---: |\"]\n", + " for name, block in sorted(stage_rows, key=lambda row: -row[1].get(\"mean_ms\", 0)):\n", + " lines.append(f\"| `{name}` | {block.get('mean_ms')} | {block.get('p95_ms')} |\")\n", + "\n", + "lines += [\"\", \"### Degradation\", \"\"]\n", + "lines.append(\n", + " \"\\n\".join(f\"- `{label}` on {count} query(ies)\" for label, count in (suite.get(\"degradation_counts\") or {}).items())\n", + " or \"- none reported on any query\"\n", + ")\n", + "\n", + "matrix_path = ABL_DIR / \"ablation_matrix.md\"\n", + "if matrix_path.exists():\n", + " lines += [\"\", \"## Ablation matrix\", \"\", matrix_path.read_text(encoding=\"utf-8\").split(\"\\n\", 1)[1].strip()]\n", + "\n", + "ablation_report = {}\n", + "if (ABL_DIR / \"ablation_report.json\").exists():\n", + " ablation_report = json.loads((ABL_DIR / \"ablation_report.json\").read_text(encoding=\"utf-8\"))\n", + " baseline = ablation_report.get(\"baseline\") or {}\n", + " if baseline:\n", + " lines += [\n", + " \"\", f\"## Full system vs {baseline.get('system')}\", \"\",\n", + " \"Sign convention: positive means the full system scores higher.\", \"\",\n", + " \"| metric | Δ | 95% CI | p | significant |\", \"| :--- | ---: | :--- | ---: | :-- |\",\n", + " ]\n", + " for metric, comparison in baseline.get(\"full_vs_baseline\", {}).items():\n", + " delta = comparison.get(\"delta\")\n", + " if delta is None:\n", + " continue\n", + " low, high = comparison.get(\"delta_ci_low\"), comparison.get(\"delta_ci_high\")\n", + " p_value = comparison.get(\"p_value\")\n", + " lines.append(\n", + " f\"| `{metric}` | {delta:+.4f} | [{low:+.4f}, {high:+.4f}] \"\n", + " f\"| {'—' if p_value is None else f'{p_value:.4f}'} | {'yes' if comparison.get('significant') else 'no'} |\"\n", + " )\n", + " subset = ablation_report.get(\"query_subset\") or {}\n", + " lines += [\n", + " \"\", \"### Scope of this run\", \"\",\n", + " f\"- queries used: **{subset.get('queries_used')}** (full matrix: **{subset.get('is_full_matrix')}**)\",\n", + " f\"- bootstrap iterations: {ablation_report.get('bootstrap_iterations')}, seed {ablation_report.get('seed')}\",\n", + " f\"- rows executed: {ablation_report.get('executed')} of {len(ablation_report.get('ablations') or [])}\",\n", + " f\"- not executed: {ablation_report.get('not_executed') or 'none'}\",\n", + " ]\n", + "\n", + "RESULTS_MD = \"\\n\".join(lines)\n", + "display(Markdown(RESULTS_MD))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b16b3ac", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 13 - SCIENTIFIC FIGURES AND THE INSIGHT DIGEST\n", + "# ============================================================================\n", + "# Seven figures plus INSIGHTS.md, every number in them read out of the two JSON reports:\n", + "# point estimates with their bootstrap intervals, quality per stratum, the latency\n", + "# decomposition and its full distribution, the degradation census, each ablation's measured\n", + "# effect beside the claim it is meant to support, and the paired comparison against naive\n", + "# RAG. PDF alongside PNG, so the same figure goes into a paper without restyling.\n", + "run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-c\", str(constraints), \"matplotlib\"],\n", + " label=\"pip install matplotlib\",\n", + ")\n", + "\n", + "FIG_DIR = REPO_DIR / \"artifacts\" / \"figures\"\n", + "run(\n", + " [\n", + " sys.executable, \"scripts/plot_results.py\",\n", + " \"--eval-report\", \"artifacts/evaluation/eval_suite.json\",\n", + " \"--ablation-report\", \"artifacts/ablations/ablation_report.json\",\n", + " \"--out\", str(FIG_DIR),\n", + " \"--formats\", \"png,pdf\",\n", + " ],\n", + " label=\"plot_results\",\n", + ")\n", + "\n", + "INSIGHTS_MD = (FIG_DIR / \"INSIGHTS.md\").read_text(encoding=\"utf-8\")\n", + "display(Markdown(INSIGHTS_MD))\n", + "for png in sorted(FIG_DIR.glob(\"fig*.png\")):\n", + " display(Image(filename=str(png)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea469dfd", + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================================\n", + "# CELL 14 - EXPORT EVERYTHING AS A REUSABLE DATASET\n", + "# ============================================================================\n", + "# Anything expensive to recompute goes here: the fetched filings and their provenance,\n", + "# the ingested corpus with its FAISS/BM25 indexes and DuckDB table store (these *are* the\n", + "# cached embeddings), the derived gold queries, the second corpus, every report, and the\n", + "# model-weight cache. Attach this output under /kaggle/input/docureason-artifacts next\n", + "# session and Cell 5 skips straight to measurement.\n", + "EXPORT_MODEL_CACHE = True\n", + "\n", + "if EXPORT_DIR.exists():\n", + " shutil.rmtree(EXPORT_DIR)\n", + "EXPORT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def _copy(source: Path, relative: str) -> int:\n", + " if not source.exists():\n", + " return 0\n", + " destination = EXPORT_DIR / relative\n", + " destination.parent.mkdir(parents=True, exist_ok=True)\n", + " if source.is_dir():\n", + " shutil.copytree(source, destination, dirs_exist_ok=True)\n", + " else:\n", + " shutil.copy2(source, destination)\n", + " return sum(path.stat().st_size for path in destination.rglob(\"*\") if path.is_file()) if destination.is_dir() else destination.stat().st_size\n", + "\n", + "\n", + "sizes = {\n", + " \"data/eval/filings\": _copy(EVAL_ROOT / \"filings\", \"data/eval/filings\"),\n", + " \"data/eval/queries.json\": _copy(QUERIES, \"data/eval/queries.json\"),\n", + " \"data/eval/corpus\": _copy(CORPUS_DIR, \"data/eval/corpus\"),\n", + " \"data/eval/corpus_no_layout\": _copy(NO_LAYOUT_DIR, \"data/eval/corpus_no_layout\"),\n", + " \"artifacts/ablations\": _copy(REPO_DIR / \"artifacts\" / \"ablations\", \"artifacts/ablations\"),\n", + " \"artifacts/evaluation\": _copy(REPO_DIR / \"artifacts\" / \"evaluation\", \"artifacts/evaluation\"),\n", + " \"artifacts/figures\": _copy(FIG_DIR, \"artifacts/figures\"),\n", + " \"artifacts/logs\": _copy(REPO_DIR / \"artifacts\" / \"logs\", \"artifacts/logs\"),\n", + "}\n", + "if EXPORT_MODEL_CACHE:\n", + " sizes[\"hf_cache\"] = _copy(HF_CACHE, \"hf_cache\")\n", + "\n", + "(EXPORT_DIR / \"RESULTS.md\").write_text(RESULTS_MD, encoding=\"utf-8\")\n", + "\n", + "manifest = {\n", + " \"produced_at\": time.strftime(\"%Y-%m-%dT%H:%M:%SZ\", time.gmtime()),\n", + " \"repository\": {\"url\": REPO_URL, \"ref\": REPO_REF, \"sha\": commit},\n", + " \"framework_version\": docureason.__version__,\n", + " \"environment\": {\n", + " \"python\": sys.version.split()[0],\n", + " \"torch\": torch.__version__,\n", + " \"cuda\": torch.cuda.is_available(),\n", + " \"gpu\": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,\n", + " \"modules_available\": available,\n", + " },\n", + " \"configuration\": {\n", + " \"filings\": FILINGS,\n", + " \"per_company\": PER_COMPANY,\n", + " \"per_stratum\": PER_STRATUM,\n", + " \"seed\": SEED,\n", + " \"bootstrap_iterations\": BOOTSTRAP_ITERATIONS,\n", + " \"eval_limit\": EVAL_LIMIT,\n", + " \"ablation_limit\": ABLATION_LIMIT,\n", + " \"no_layout_corpus\": RUN_NO_LAYOUT_CORPUS,\n", + " },\n", + " \"corpus\": {\n", + " \"filings\": payload[\"filing_count\"],\n", + " \"documents\": report[\"documents\"],\n", + " \"regions\": report[\"regions_scanned\"],\n", + " \"gold_queries\": report[\"query_count\"],\n", + " \"per_stratum\": report[\"per_stratum\"],\n", + " \"rejected\": report[\"rejected\"],\n", + " \"validation_passed\": validation[\"valid\"],\n", + " \"capabilities_used\": capabilities,\n", + " },\n", + " \"results\": {\n", + " \"summary_metrics\": suite[\"summary_metrics\"],\n", + " \"queries_scored\": suite[\"queries_scored\"],\n", + " \"degradation_counts\": suite.get(\"degradation_counts\"),\n", + " \"ablation_rows_executed\": ablation_report.get(\"executed\"),\n", + " \"ablation_rows_not_executed\": ablation_report.get(\"not_executed\"),\n", + " \"is_full_matrix\": (ablation_report.get(\"query_subset\") or {}).get(\"is_full_matrix\"),\n", + " },\n", + " \"restored_from_cache\": RESTORED,\n", + " \"bytes\": sizes,\n", + "}\n", + "(EXPORT_DIR / \"MANIFEST.json\").write_text(json.dumps(manifest, indent=2), encoding=\"utf-8\")\n", + "\n", + "total = sum(sizes.values())\n", + "print(f\"Exported to {EXPORT_DIR}\\n\")\n", + "for name, size in sorted(sizes.items(), key=lambda item: -item[1]):\n", + " print(f\" {name:34s} {size / 1024 ** 2:9.1f} MB\")\n", + "print(f\" {'TOTAL':34s} {total / 1024 ** 2:9.1f} MB\")\n", + "print(\"\\nTo reuse: Save Version -> Save & Run All, then in the next session\")\n", + "print(\"Add Data -> Your Work -> this notebook's output, mounted at\")\n", + "print(\"/kaggle/input/docureason-artifacts. Cell 5 will restore the corpus, the indexes,\")\n", + "print(\"the DuckDB store and the model weights, and cells 7-8 become no-ops.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What this run does and does not establish\n", + "\n", + "**Established, in this session, from these filings:** every metric above, with the\n", + "interval around it and the stratum it came from; the per-stage latency decomposition\n", + "including its signed residual; each ablation's paired delta against the full system, with\n", + "the published claim printed beside it; the naive-RAG comparison, scored through the same\n", + "harness with the same generator and the same embedding model.\n", + "\n", + "**Not established:**\n", + "\n", + "- **Gold labels are derived, not annotated.** Table cells know their region by\n", + " construction and prose anchors are phrases verified unique across the corpus, but\n", + " derivable queries over-represent lookup and cannot express multi-hop reasoning.\n", + "- **The `figure` stratum is usually empty.** Docling's HTML pipeline emits no figure\n", + " regions for these filings, so the rasterization work is exercised by tests, not by this\n", + " measurement. A PDF corpus would change that.\n", + "- **The `no_layout_parser` row cannot test its own recall claim.** Its corpus renames\n", + " every gold region, so the runner reports the gold-id metrics as not comparable and\n", + " carries the row on answer quality plus a region/table census.\n", + "- **Attribution is lexical plus numeric consistency, not entailment.** `entailment_threshold`\n", + " is inert until a real NLI model is loaded, and `attribution_precision` should be read as\n", + " claim-support coverage rather than faithfulness.\n", + "\n", + "`Documentations/audit.md` is the plan of record; §13 lists what remains." + ] + } + ], + "metadata": { + "kaggle": { + "accelerator": "gpu", + "isInternetEnabled": true, + "language": "python" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 056ca2b..09a0513 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "docureason-framework" -version = "1.1.4" +version = "1.2.0" description = "DocuReason RAG: A Grounded Framework for Multimodal Document Retrieval and Reasoning. A Tri-Path Multimodal RAG Framework for Text, Tables, and Visual Documents" readme = "README.md" authors = [{name = "Arpit Kumar", email = "kumararpit17773@gmail.com"}] From a8e1d99d2790549349471d7686a9b6f4a7c0cc15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:40:53 +0000 Subject: [PATCH 3/3] build(deps-dev): update pytest-asyncio requirement Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.21.0...v1.4.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-version: 1.4.0 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09a0513..8b85748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ test = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", - "pytest-asyncio>=0.21.0", + "pytest-asyncio>=1.4.0", "httpx>=0.24.0", # The figure builders are code, so they are tested. Here rather than only in # [figures] so the CI gate covers them instead of skipping them.