From 8d9ae62ae7e500531af48a777e018c6307cb8e7d Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Thu, 8 Oct 2026 17:56:33 -0400 Subject: [PATCH 1/4] Add isolated story model evaluation --- docs/story-evaluation.md | 117 ++++ evals/stories/cases.json | 477 ++++++++++++++++ evals/stories/models.json | 76 +++ roger/story_eval.py | 1088 +++++++++++++++++++++++++++++++++++++ tests/test_story_eval.py | 463 ++++++++++++++++ 5 files changed, 2221 insertions(+) create mode 100644 docs/story-evaluation.md create mode 100644 evals/stories/cases.json create mode 100644 evals/stories/models.json create mode 100644 roger/story_eval.py create mode 100644 tests/test_story_eval.py diff --git a/docs/story-evaluation.md b/docs/story-evaluation.md new file mode 100644 index 0000000..70d941c --- /dev/null +++ b/docs/story-evaluation.md @@ -0,0 +1,117 @@ +# Story model evaluation + +`python -m roger.story_eval` prints a network-free plan for the Curated story +comparison. It reads the synthetic cases and fixed model manifest, then lists +every request and its conservative reservation. It does not read credentials, +Scout, the production database, or Discord. + +The prepared screen is 48 writer calls (four models, six cases, two repeats) +and 48 reviewer calls (two models, twelve calibration fixtures, two repeats). +Each call reserves 32,768 input tokens and either 16,384 writer tokens or 8,192 +reviewer tokens. Reasoning tokens count against the output ceiling. The current +per-call ceilings total $27.60, so the proposed initial allowance is 96 calls +and $28. Up to two selected writer/reviewer pairings can later use 96 calls and +$64. A separately reviewed current-evidence acceptance packet can use four more +calls and $2.60. The exact reserved total is $92.60, with a proposed $100 outer +envelope for 196 calls. These are approval bounds, not expected spend. + +The initial screen uses separate immutable ledgers, so the operator must track +the aggregate approval across them: + +| Run | Calls | Per-run ceiling | +| --- | ---: | ---: | +| Astra writer | 12 | $13.80 | +| Sol writer | 12 | $2.76 | +| Sonnet writer | 12 | $2.76 | +| Gemini writer | 12 | $1.08 | +| Sol reviewer | 24 | $3.60 | +| Sonnet reviewer | 24 | $3.60 | + +The worst-case selected pairing is 48 calls and $31.20 when Astra writes and +Sol or Sonnet reviews. Two such pairing ledgers reserve 96 calls and $62.40. +No single ledger enforces the aggregate allowance across other artifact +directories. + +No paid comparison has been approved or run. Recheck the model catalog, +endpoint, provider and rates in `evals/stories/models.json` before proposing a +run. In particular, the Google endpoint's catalog status is recorded without +interpreting it. Do not substitute another provider or service tier under the +same approval. + +## Inspect the plan + +```bash +python -m roger.story_eval plan +``` + +The plan reports the exact dated model, accepted returned IDs, provider, +service tier, settings, input reservation, output ceiling and cost reservation +for each call. It does not construct an API client. The request omits +temperature, `top_p`, model chains and fallback. It requires strict JSON, +medium reasoning, one first-party provider and the standard price ceiling. + +## Run an approved slice + +Use a new artifact directory outside the repository. A directory is created +exclusively and is never resumed or reused. `OPENROUTER_API_KEY` is read only at +runtime and is not written to artifacts. + +```bash +OPENROUTER_API_KEY=... python -m roger.story_eval run \ + --stage writer --model sol --case straight_release --repeats 2 \ + --artifact-dir /tmp/roger-story-eval-sol \ + --max-calls 2 --max-cost-usd 0.46 --approve-paid-calls +``` + +Reviewer calibration uses `--stage reviewer --model sol`; each `--case` value +then names a reviewer fixture. Complete workflow testing uses `--stage +workflow --writer-model sol --reviewer-model sonnet`. The workflow calls the +production `stories.decide` path and remains bounded to four calls. + +A reviewed held-out current-evidence packet may replace the fixture file with +`--fixtures /path/to/reviewed.json`. Each case supplies its own aware `now`, raw +entries, confirmed history, author reference expectation and human rubric. The +harness still reads only that file. It never fetches live evidence. Use `plan +--fixtures ...` before asking for paid-run approval. + +Before every network request, the ledger durably reserves one call and its full +cost ceiling. Calls are sequential. There are no SDK retries, model fallbacks, +or automatic repeats. A request failure or uncertain result retains its +reservation. Missing or invalid usage totals, cost, route metadata, exact model +identity, or selected provider fails closed. A missing +optional reasoning-token breakdown stays `null`; total output still bounds and +bills it. A missing returned service tier is recorded as unknown and limits +latency comparisons, while an explicitly different tier fails closed. Provider +truncation, refusal, content filtering, pipeline interference, local parsing, +reviewer rejection and human editorial quality remain separate outcomes. +Reviewer calibration has six supported references and six single-fault drafts. +Any false accept, false reject, malformed review, incomplete run, or missing +fixture repeat fails its reference gate. Reference labels are still pending +human confirmation, so this gate alone cannot select a reviewer. + +The API cache is disabled so repeated requests remain real repeats. Artifacts +contain the complete visible response text, an allowlisted routing-metadata +view, token/cost accounting, sanitized error classes or status codes, and the +exact messages, schema and non-secret request settings needed to reproduce the +call. They do not contain the key, authorization headers, hidden reasoning or +raw exception text. + +## Assess results + +```bash +python -m roger.story_eval summarize /tmp/roger-story-eval-sol +``` + +`summary.json` leaves `human_rating` and `human_acceptance` pending. The fixture +labels and reference decisions are author expectations pending human review. +A parser pass or reviewer agreement is not human acceptance. Human assessment +scores useful detail, focus, what changed, grounded commentary, honest +uncertainty and readable public copy. A hold or skip on a reportable case is a +failure. + +Each writer and workflow result records the original and delivered excerpt +lengths and any omitted characters. The prepared synthetic cases fit inside the +current 1,600-character per-source cap. If held-out evidence is omitted, report +that limitation before interpreting quality. A fuller-evidence diagnostic would +need an explicit non-production profile using the same original source text; +it cannot count as production acceptance or silently relax the parser. diff --git a/evals/stories/cases.json b/evals/stories/cases.json new file mode 100644 index 0000000..b37eb6b --- /dev/null +++ b/evals/stories/cases.json @@ -0,0 +1,477 @@ +{ + "schema_version": 1, + "label_status": "author_expected_pending_human_review", + "fixed_now": "2026-10-08T12:00:00+00:00", + "matrix": { + "writer_profiles": ["astra", "sol", "sonnet", "gemini"], + "reviewer_profiles": ["sol", "sonnet"], + "repeats": 2 + }, + "human_rubric": { + "rating_scale": { + "0": "Unusable: absent, materially wrong, unsupported, or misleading.", + "1": "Major edit: the right subject is visible but a substantial rewrite or evidence repair is needed.", + "2": "Minor edit: accurate and useful, with a limited clarity, focus, or completeness edit needed.", + "3": "Ready: accurate, focused, complete for the evidence, and publishable as written." + }, + "dimensions": { + "useful_detail": "Names the concrete capability, limit, availability, or result a reader can act on.", + "story_focus": "Covers one development, or a cited relationship for combine, without a distractor.", + "what_changed": "States the cited new event, finding, correction, or supported hold reason.", + "grounded_commentary": "Separates source facts from interpretation and preserves every qualifier.", + "honest_uncertainty": "Names material unknowns and does not convert marketing or silence into fact.", + "readable_public_copy": "Uses concise, specific language whose importance is clear on one read." + }, + "hard_failures": [ + "Any unsupported public fact or material dropped qualifier.", + "Wrong action, story identity, delivery history, or correction target.", + "Combining an unrelated distractor or inventing a relationship.", + "Publishing duplicate or stale evidence as a material change.", + "Holding or skipping a case marked reportable." + ], + "acceptance": "No hard failure and every dimension scores at least 2; ready without edits requires every dimension to score 3.", + "rules": [ + "A parser or reviewer pass is not human acceptance.", + "A false hold on a reportable case fails the case.", + "Labels and reference decisions were authored for this fixture and still require human review." + ] + }, + "cases": [ + { + "id": "straight_release", + "now": "2026-10-08T12:00:00+00:00", + "description": "A straightforward generally available release.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:release.example.test", + "id": "atlas-2-release", + "published_at": "2026-10-08T09:00:00+00:00", + "observed_at": "2026-10-08T09:10:00+00:00", + "article": { + "status": "ok", + "url": "https://release.example.test/atlas-2", + "text": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command. The release notes describe the migration command as a local conversion step for existing project files. They list packages for x86-64 and arm64 Linux systems and include checksums beside every download. The maintainers say macOS and Windows packages are still being tested, without promising a date. The document does not compare performance with other tools and gives no adoption figures. Operators can keep Atlas 1.x installed while validating a converted copy." + } + } + ], + "history": {"stories": [], "delivered": []}, + "expected": { + "reportable": true, + "action": "publish", + "supported_conclusions": ["Atlas 2.0 is generally available with signed Linux packages and an offline migration command."], + "forbidden_inferences": ["Atlas 2.0 is faster than competitors.", "Windows packages are available."] + }, + "reference_decision": { + "action": "publish", + "reason": {"text": "Atlas 2.0 is now generally available for Linux.", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}]}, + "change": {"text": "The release adds signed Linux packages and an offline migration path.", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}]}, + "story_ids": ["url:a52cb8d5c3e329d711d5d974e9f29901396fe2844f83fef54d48ebf8087acd56"], + "claims": [ + {"field": "headline", "text": "Atlas 2.0 ships for Linux", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}], "mutable": true}, + {"field": "stage", "text": "rollout", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}], "mutable": true}, + {"field": "fact", "text": "Signed Linux packages cover x86-64 and arm64.", "citations": [{"source": 1, "quote": "They list packages for x86-64 and arm64 Linux systems and include checksums beside every download."}], "mutable": true}, + {"field": "why", "text": "Operators can keep Atlas 1.x installed while validating a converted copy.", "citations": [{"source": 1, "quote": "Operators can keep Atlas 1.x installed while validating a converted copy."}], "mutable": false} + ], + "unresolved_questions": ["When will macOS and Windows packages be ready?"], + "correction_of": null + } + }, + { + "id": "skeptical_marketing", + "now": "2026-10-08T12:00:00+00:00", + "description": "A useful preview wrapped in an unsupported comparison.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:marketing.example.test", + "id": "northstar-cache-preview", + "published_at": "2026-10-08T08:00:00+00:00", + "observed_at": "2026-10-08T08:15:00+00:00", + "article": { + "status": "ok", + "url": "https://marketing.example.test/northstar-cache-preview", + "text": "Northstar introduced a preview of its cache controls on October 8, 2026, and published configuration examples for opt-in testing. The controls let testers set a per-project retention window and clear individual cache namespaces. Northstar says its preview is twice as fast as every competing cache, but the post provides no benchmark data or comparison method. The preview is disabled by default and the company asks users to report eviction bugs. General availability, pricing, and compatibility outside the listed Linux test image remain unannounced. The configuration examples use only synthetic projects." + } + } + ], + "history": {"stories": [], "delivered": []}, + "expected": { + "reportable": true, + "action": "publish", + "supported_conclusions": ["Northstar released an opt-in preview with retention and namespace controls."], + "forbidden_inferences": ["Northstar is twice as fast as competitors.", "The preview is generally available."] + }, + "reference_decision": { + "action": "publish", + "reason": {"text": "Northstar has a concrete opt-in cache-control preview worth testing.", "citations": [{"source": 1, "quote": "Northstar introduced a preview of its cache controls on October 8, 2026, and published configuration examples for opt-in testing."}]}, + "change": {"text": "Testers can now configure retention and clear separate cache namespaces.", "citations": [{"source": 1, "quote": "The controls let testers set a per-project retention window and clear individual cache namespaces."}]}, + "story_ids": ["url:b5a853bae17844018fea3ec6e1ce329f1cf50f4ffb7ec31cff2f068d9c8dd5d5"], + "claims": [ + {"field": "headline", "text": "Northstar previews cache controls", "citations": [{"source": 1, "quote": "Northstar introduced a preview of its cache controls on October 8, 2026, and published configuration examples for opt-in testing."}], "mutable": true}, + {"field": "stage", "text": "preview", "citations": [{"source": 1, "quote": "The preview is disabled by default and the company asks users to report eviction bugs."}], "mutable": true}, + {"field": "fact", "text": "The preview adds per-project retention and namespace clearing.", "citations": [{"source": 1, "quote": "The controls let testers set a per-project retention window and clear individual cache namespaces."}], "mutable": true}, + {"field": "why", "text": "The broad speed claim has no benchmark data or comparison method.", "citations": [{"source": 1, "quote": "Northstar says its preview is twice as fast as every competing cache, but the post provides no benchmark data or comparison method."}], "mutable": false} + ], + "unresolved_questions": ["How does the preview perform under a disclosed benchmark?"], + "correction_of": null + } + }, + { + "id": "material_update", + "now": "2026-10-08T12:00:00+00:00", + "description": "A previously reported project adds a material platform rollout.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:updates.example.test", + "id": "river-windows-ga", + "published_at": "2026-10-08T07:00:00+00:00", + "observed_at": "2026-10-08T07:10:00+00:00", + "article": { + "status": "ok", + "url": "https://updates.example.test/river-windows", + "text": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1. The Windows build supports the same encrypted workspace format and can open workspaces created by the Linux release. The vendor documents Windows 11 as the supported operating system and provides signed installer checksums. Windows 10 is not listed. The announcement makes no claim about mobile support or market adoption. Existing Linux workspaces do not need conversion before they are opened on Windows. A seven-day preview preceded this release." + } + } + ], + "history": { + "stories": [ + { + "story_id": "url:d69b5b52a6a8018d8a2697d1d95cbca0e4bfe8d1b0e3baab028474d436381171", + "ts": 1790856000.0, + "data": { + "last_decision_id": 301, + "unresolved_questions": ["When will the Windows client leave preview?"], + "sources": [ + { + "story_id": "url:d69b5b52a6a8018d8a2697d1d95cbca0e4bfe8d1b0e3baab028474d436381171", + "source_id": "river-linux-prior", + "feed_url": "scout:updates.example.test", + "entry_id": "river-linux-ga", + "url": "https://updates.example.test/river-windows", + "published_at": "2026-10-01T07:00:00+00:00", + "observed_at": "2026-10-01T07:10:00+00:00", + "fresh": false, + "excerpt": "River's Linux client became generally available on October 1, 2026. The signed package supports encrypted workspaces on two Linux architectures. The announcement said a Windows build remained in preview and did not give a final release date. Existing preview users could keep their workspace data. The release notes documented installer checksums and an uninstall path. No macOS or mobile build was announced, and the vendor made no claim about market adoption or comparative performance." + } + ] + } + } + ], + "delivered": [ + { + "id": 301, + "action": "publish", + "reason": {"text": "River shipped its Linux client."}, + "change": {"text": "Linux became generally available."}, + "story_ids": ["url:d69b5b52a6a8018d8a2697d1d95cbca0e4bfe8d1b0e3baab028474d436381171"], + "claims": [{"field": "headline", "text": "River ships on Linux"}], + "correction_of": null, + "ts": 1790856000.0, + "message_id": "example-message-301", + "message_url": "https://discord.example.invalid/channels/example/301", + "sources": [ + { + "story_id": "url:d69b5b52a6a8018d8a2697d1d95cbca0e4bfe8d1b0e3baab028474d436381171", + "source_id": "river-linux-prior", + "feed_url": "scout:updates.example.test", + "entry_id": "river-linux-ga", + "url": "https://updates.example.test/river-windows", + "published_at": "2026-10-01T07:00:00+00:00", + "observed_at": "2026-10-01T07:10:00+00:00", + "fresh": false, + "excerpt": "River's Linux client became generally available on October 1, 2026. The signed package supports encrypted workspaces on two Linux architectures. The announcement said a Windows build remained in preview and did not give a final release date. Existing preview users could keep their workspace data. The release notes documented installer checksums and an uninstall path. No macOS or mobile build was announced, and the vendor made no claim about market adoption or comparative performance." + } + ] + } + ] + }, + "expected": { + "reportable": true, + "action": "update", + "supported_conclusions": ["River added a generally available Windows 11 client after its Linux release."], + "forbidden_inferences": ["Windows shipped before Linux.", "Windows 10 is supported."] + }, + "reference_decision": { + "action": "update", + "reason": {"text": "River added a generally available Windows client to its earlier Linux release.", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}]}, + "change": {"text": "Windows 11 is now supported and opens existing Linux workspaces without conversion.", "citations": [{"source": 1, "quote": "The vendor documents Windows 11 as the supported operating system and provides signed installer checksums."}, {"source": 1, "quote": "Existing Linux workspaces do not need conversion before they are opened on Windows."}]}, + "story_ids": ["url:d69b5b52a6a8018d8a2697d1d95cbca0e4bfe8d1b0e3baab028474d436381171"], + "claims": [ + {"field": "headline", "text": "River adds a Windows client", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, + {"field": "stage", "text": "rollout", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, + {"field": "fact", "text": "The signed Windows client supports Windows 11.", "citations": [{"source": 1, "quote": "The vendor documents Windows 11 as the supported operating system and provides signed installer checksums."}], "mutable": true}, + {"field": "why", "text": "Teams can move the same encrypted workspaces between Linux and Windows.", "citations": [{"source": 1, "quote": "The Windows build supports the same encrypted workspace format and can open workspaces created by the Linux release."}], "mutable": false} + ], + "unresolved_questions": ["Will River support Windows 10?"], + "correction_of": null + } + }, + { + "id": "correction", + "now": "2026-10-08T12:00:00+00:00", + "description": "Fresh evidence corrects a confirmed prior capacity claim.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:corrections.example.test", + "id": "pine-cap-correction", + "published_at": "2026-10-08T06:00:00+00:00", + "observed_at": "2026-10-08T06:05:00+00:00", + "article": { + "status": "ok", + "url": "https://corrections.example.test/pine-cap", + "text": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post. The company says the earlier number counted archived projects and has updated the pricing table. Active projects beyond 60 require the enterprise plan. The correction does not change the storage allowance, user limit, or launch date. Customers who created more than 60 active projects during the first day will receive direct guidance. No other plan limits were revised. The corrected pricing page was observed on October 8, 2026." + } + } + ], + "history": { + "stories": [ + { + "story_id": "url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb", + "ts": 1791374400.0, + "data": { + "last_decision_id": 401, + "unresolved_questions": [], + "sources": [ + { + "story_id": "url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb", + "source_id": "pine-launch-prior", + "feed_url": "scout:corrections.example.test", + "entry_id": "pine-launch", + "url": "https://corrections.example.test/pine-cap", + "published_at": "2026-10-07T06:00:00+00:00", + "observed_at": "2026-10-07T06:05:00+00:00", + "fresh": true, + "excerpt": "Pine launched its team plan on October 7, 2026. The original pricing table stated that the plan supported 100 active projects, ten users, and a fixed storage allowance. The launch page described archived projects separately but did not explain whether they counted toward the active-project limit. Enterprise pricing remained contact-only. The post included no performance comparison or adoption claim. A linked launch summary repeated the stated 100-project limit." + } + ] + } + } + ], + "delivered": [ + { + "id": 401, + "action": "publish", + "reason": {"text": "Pine launched a team plan."}, + "change": {"text": "The launch post listed 100 active projects."}, + "story_ids": ["url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb"], + "claims": [{"field": "fact", "text": "The plan supports 100 active projects."}], + "correction_of": null, + "ts": 1791374400.0, + "message_id": "example-message-401", + "message_url": "https://discord.example.invalid/channels/example/401", + "sources": [ + { + "story_id": "url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb", + "source_id": "pine-launch-prior", + "feed_url": "scout:corrections.example.test", + "entry_id": "pine-launch", + "url": "https://corrections.example.test/pine-cap", + "published_at": "2026-10-07T06:00:00+00:00", + "observed_at": "2026-10-07T06:05:00+00:00", + "fresh": true, + "excerpt": "Pine launched its team plan on October 7, 2026. The original pricing table stated that the plan supported 100 active projects, ten users, and a fixed storage allowance. The launch page described archived projects separately but did not explain whether they counted toward the active-project limit. Enterprise pricing remained contact-only. The post included no performance comparison or adoption claim. A linked launch summary repeated the stated 100-project limit." + } + ] + } + ] + }, + "expected": { + "reportable": true, + "action": "update", + "correction_of": 401, + "supported_conclusions": ["Pine corrected the active-project limit from 100 to 60."], + "forbidden_inferences": ["The storage allowance also changed.", "The corrected limit is 160."] + }, + "reference_decision": { + "action": "update", + "reason": {"text": "Pine corrected a material limit in its confirmed launch report.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}]}, + "change": {"text": "The active-project limit is 60 rather than the previously reported 100.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}]}, + "story_ids": ["url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb"], + "claims": [ + {"field": "headline", "text": "Pine corrects its team plan limit", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}], "mutable": true}, + {"field": "stage", "text": "demonstrated", "citations": [{"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, + {"field": "fact", "text": "The team plan permits 60 active projects.", "citations": [{"source": 1, "quote": "Active projects beyond 60 require the enterprise plan."}], "mutable": true}, + {"field": "why", "text": "The active-project capacity changed; storage, users, and launch date did not.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The correction does not change the storage allowance, user limit, or launch date."}], "mutable": false} + ], + "unresolved_questions": [], + "correction_of": 401 + } + }, + { + "id": "legitimate_combine", + "now": "2026-10-08T12:00:00+00:00", + "description": "Two independently sourced parts of one protocol rollout plus an unrelated distractor.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:combine.example.test", + "id": "lumen-protocol", + "published_at": "2026-10-08T05:00:00+00:00", + "observed_at": "2026-10-08T05:10:00+00:00", + "article": { + "status": "ok", + "url": "https://combine.example.test/lumen-protocol", + "text": "Lumen published version 1.0 of the Lantern synchronization protocol on October 8, 2026. The specification defines the wire format, conflict markers, and a compatibility test suite. Lumen's announcement names Orbit as the first independent client to pass that public suite. The protocol repository includes fixtures under a permissive license and records the exact suite revision used for the result. Passing the suite does not establish performance, security, or broad adoption. Lumen says two draft sections remain informational and are not required for compatibility." + } + }, + { + "feed_url": "scout:combine.example.test", + "id": "orbit-client", + "published_at": "2026-10-08T05:20:00+00:00", + "observed_at": "2026-10-08T05:25:00+00:00", + "article": { + "status": "ok", + "url": "https://combine.example.test/orbit-client", + "text": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite. The client can exchange the suite's fixture documents with Lumen's reference implementation. Orbit labels the build a developer preview and advises against production data. Its post does not claim that Lumen sponsored the client or that either project acquired the other. Performance and security review remain outside the compatibility suite. Source code and reproducible build instructions are available for the preview." + } + }, + { + "feed_url": "scout:distractor.example.test", + "id": "ceramic-keyboard", + "published_at": "2026-10-08T04:00:00+00:00", + "observed_at": "2026-10-08T04:10:00+00:00", + "article": { + "status": "ok", + "url": "https://distractor.example.test/ceramic-keyboard", + "text": "Ceramic Keys announced a compact mechanical keyboard with a glazed case and replaceable switches. Preorders open later in October and shipping is estimated for December. The vendor lists three switch options and two case colors. No review samples or independent measurements are available. The case uses a two-piece ceramic shell with a brass mounting plate, while the firmware supports remapping over a wired connection. Replacement switches use a standard hot-swap socket. Regional prices and final shipping quantities remain unannounced." + } + } + ], + "history": {"stories": [], "delivered": []}, + "expected": { + "reportable": true, + "action": "combine", + "supported_conclusions": ["Lumen published Lantern 1.0 and Orbit released an independent client that passed its public suite."], + "forbidden_inferences": ["Lumen acquired Orbit.", "The keyboard is part of Lantern."] + }, + "reference_decision": { + "action": "combine", + "reason": {"text": "The Lantern 1.0 specification and its first independently tested client are one development.", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}]}, + "change": {"text": "Lantern reached 1.0 with a public suite that Orbit's preview client passed.", "citations": [{"source": 1, "quote": "Lumen published version 1.0 of the Lantern synchronization protocol on October 8, 2026."}, {"source": 2, "quote": "The client can exchange the suite's fixture documents with Lumen's reference implementation."}]}, + "story_ids": ["url:8a37ed47d48d359e1aa575095bb9bc9afb8c96c28950b7d19a398cf5123ffebd", "url:6782db64ca26f1128b8676730fbe44979b5d747a4041d72c6b187ed6a547a72a"], + "claims": [ + {"field": "headline", "text": "Lantern 1.0 gets an independent client", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}], "mutable": true}, + {"field": "stage", "text": "demonstrated", "citations": [{"source": 2, "quote": "The client can exchange the suite's fixture documents with Lumen's reference implementation."}], "mutable": true}, + {"field": "fact", "text": "Orbit passed Lumen's public Lantern compatibility suite.", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}], "mutable": true}, + {"field": "why", "text": "The compatibility result does not establish performance or security.", "citations": [{"source": 2, "quote": "Performance and security review remain outside the compatibility suite."}], "mutable": false} + ], + "unresolved_questions": ["Will another independent client pass the suite?"], + "correction_of": null + } + }, + { + "id": "duplicate_stale_hold", + "now": "2026-10-08T12:00:00+00:00", + "description": "A rewrite of confirmed coverage plus stale, insufficient speculation.", + "max_age_hours": 48, + "entries": [ + { + "feed_url": "scout:duplicates.example.test", + "id": "atlas-guide-rewrite", + "published_at": "2026-10-08T03:00:00+00:00", + "observed_at": "2026-10-08T03:10:00+00:00", + "article": { + "status": "ok", + "url": "https://duplicates.example.test/atlas-guide", + "text": "A community guide walks through the Atlas offline conversion command and installation of the signed Linux packages. It adds screenshots of each terminal step and recommends validating a converted copy before removing Atlas 1.x. The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction. The guide uses the package checksums and commands from the October 7 maintainer documentation. It presents no measured result, new compatibility claim, or later product event." + } + }, + { + "feed_url": "scout:stale.example.test", + "id": "atlas-rumor", + "published_at": "2026-09-20T03:00:00+00:00", + "observed_at": "2026-09-20T03:10:00+00:00", + "article": { + "status": "ok", + "url": "https://stale.example.test/atlas-rumor", + "text": "An undated forum summary speculates that Atlas might add mobile packages someday, but it cites no maintainer statement, repository change, release artifact, or schedule. The post repeats a user's guess that mobile support would be convenient and discusses possible touch-screen layouts. No build, preview, test result, issue milestone, or availability change is linked. Replies ask whether the maintainers have commented, but none supplies a source. The page snapshot contains no date beyond the forum thread metadata recorded by the collector." + } + } + ], + "history": { + "stories": [ + { + "story_id": "url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863", + "ts": 1791374400.0, + "data": { + "last_decision_id": 601, + "unresolved_questions": ["Will Atlas support another platform?"], + "sources": [ + { + "story_id": "url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863", + "source_id": "atlas-guide-prior", + "feed_url": "scout:duplicates.example.test", + "entry_id": "atlas-guide-original", + "url": "https://duplicates.example.test/atlas-guide", + "published_at": "2026-10-07T03:00:00+00:00", + "observed_at": "2026-10-07T03:10:00+00:00", + "fresh": true, + "excerpt": "Atlas published an offline migration guide on October 7, 2026. The guide described the conversion command, signed Linux packages, and a validation step performed on a copied project. It advised keeping Atlas 1.x installed while checking the converted copy. The source made no mobile announcement and did not promise another platform. It included no benchmark or adoption number. Package checksums and commands were listed beside the Linux installation steps." + } + ] + } + } + ], + "delivered": [ + { + "id": 601, + "action": "publish", + "reason": {"text": "Atlas published its migration guide."}, + "change": {"text": "Offline migration instructions became available."}, + "story_ids": ["url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863"], + "claims": [{"field": "fact", "text": "Atlas documents offline conversion."}], + "correction_of": null, + "ts": 1791374400.0, + "message_id": "example-message-601", + "message_url": "https://discord.example.invalid/channels/example/601", + "sources": [ + { + "story_id": "url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863", + "source_id": "atlas-guide-prior", + "feed_url": "scout:duplicates.example.test", + "entry_id": "atlas-guide-original", + "url": "https://duplicates.example.test/atlas-guide", + "published_at": "2026-10-07T03:00:00+00:00", + "observed_at": "2026-10-07T03:10:00+00:00", + "fresh": true, + "excerpt": "Atlas published an offline migration guide on October 7, 2026. The guide described the conversion command, signed Linux packages, and a validation step performed on a copied project. It advised keeping Atlas 1.x installed while checking the converted copy. The source made no mobile announcement and did not promise another platform. It included no benchmark or adoption number. Package checksums and commands were listed beside the Linux installation steps." + } + ] + } + ] + }, + "expected": { + "reportable": false, + "action": "hold", + "supported_conclusions": ["The guide is duplicate coverage and the mobile rumor is stale and unsupported."], + "forbidden_inferences": ["Atlas announced mobile packages.", "A rewrite is a material update."] + }, + "reference_decision": { + "action": "hold", + "reason": {"text": "The fresh item repeats confirmed coverage, while the mobile speculation is stale and unsupported.", "citations": [{"source": 1, "quote": "The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction."}, {"source": 2, "quote": "No build, preview, test result, issue milestone, or availability change is linked."}]}, + "change": {"text": "", "citations": []}, + "story_ids": ["url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863"], + "claims": [], + "unresolved_questions": ["Will Atlas publish maintainer-backed evidence for another platform?"], + "correction_of": null + } + } + ], + "reviewer_fixtures": [ + {"id": "good_release", "case_id": "straight_release", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "good_skeptical_marketing", "case_id": "skeptical_marketing", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "good_update", "case_id": "material_update", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "good_correction", "case_id": "correction", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "good_combine", "case_id": "legitimate_combine", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "good_hold", "case_id": "duplicate_stale_hold", "seed": "none", "expected": {"review_accept": true, "failing_gates": []}}, + {"id": "unsupported_claim", "case_id": "straight_release", "seed": "unsupported_claim", "expected": {"review_accept": false, "failing_gates": ["supported"]}}, + {"id": "dropped_qualifier", "case_id": "skeptical_marketing", "seed": "dropped_qualifier", "expected": {"review_accept": false, "failing_gates": ["supported"]}}, + {"id": "invented_relationship", "case_id": "legitimate_combine", "seed": "invented_relationship", "expected": {"review_accept": false, "failing_gates": ["supported", "relationship_supported"]}}, + {"id": "chronology_error", "case_id": "material_update", "seed": "chronology_error", "expected": {"review_accept": false, "failing_gates": ["supported", "chronology_supported"]}}, + {"id": "stale_claim", "case_id": "duplicate_stale_hold", "seed": "stale_claim", "expected": {"review_accept": false, "failing_gates": ["freshness_supported", "material_change"]}}, + {"id": "incorrect_correction", "case_id": "correction", "seed": "incorrect_correction", "expected": {"review_accept": false, "failing_gates": ["supported", "correction_supported"]}} + ] +} diff --git a/evals/stories/models.json b/evals/stories/models.json new file mode 100644 index 0000000..2cc4cf0 --- /dev/null +++ b/evals/stories/models.json @@ -0,0 +1,76 @@ +{ + "checked_at": "2026-10-08", + "source": "OpenRouter public model and endpoint catalogs; no paid completion was sent", + "profiles": { + "astra": { + "model": "openai/gpt-6-astra-20260903", + "returned_models": [ + "openai/gpt-6-astra-20260903", + "openai/gpt-6-astra" + ], + "provider": "openai", + "provider_display_name": "OpenAI", + "service_tier": "default", + "pricing_usd_per_million": {"prompt": 10, "completion": 50}, + "reasoning_effort": "medium", + "writer_max_tokens": 16384, + "reviewer_max_tokens": 8192, + "available": true, + "note": "Strong capability baseline." + }, + "sol": { + "model": "openai/gpt-6.1-sol-20260929", + "returned_models": [ + "openai/gpt-6.1-sol-20260929", + "openai/gpt-6.1-sol" + ], + "provider": "openai", + "provider_display_name": "OpenAI", + "service_tier": "default", + "pricing_usd_per_million": {"prompt": 2, "completion": 10}, + "reasoning_effort": "medium", + "writer_max_tokens": 16384, + "reviewer_max_tokens": 8192, + "available": true, + "note": "Balanced OpenAI candidate." + }, + "sonnet": { + "model": "anthropic/claude-sonnet-5.5-20260928", + "returned_models": [ + "anthropic/claude-sonnet-5.5-20260928", + "anthropic/claude-sonnet-5.5" + ], + "provider": "anthropic", + "provider_display_name": "Anthropic", + "service_tier": "default", + "pricing_usd_per_million": {"prompt": 2, "completion": 10}, + "reasoning_effort": "medium", + "writer_max_tokens": 16384, + "reviewer_max_tokens": 8192, + "available": true, + "note": "Cross-provider writing and judgment candidate." + }, + "gemini": { + "model": "google/gemini-3.8-flash-20260902", + "returned_models": [ + "google/gemini-3.8-flash-20260902", + "google/gemini-3.8-flash" + ], + "provider": "google-ai-studio", + "provider_display_name": "Google AI Studio", + "service_tier": "default", + "pricing_usd_per_million": {"prompt": 0.75, "completion": 3.75}, + "reasoning_effort": "medium", + "writer_max_tokens": 16384, + "reviewer_max_tokens": 8192, + "available": true, + "note": "Catalog endpoint status -2 is recorded but uninterpreted; recheck before paid run." + } + }, + "notes": [ + "Every request pins one dated model and one first-party provider, disables fallback, and accepts only the dated ID or its verified canonical base ID.", + "Input is reserved at 32768 tokens per call. Output reservations include reasoning tokens and are rounded up to cents per call.", + "The Gemini provider choice must be rechecked before a paid run; google-vertex/global requires a separately recorded preflight choice.", + "The initial 96-call writer and reviewer matrix has a $28 proposed cap. Two selected pairings allow at most 96 more calls with a $64 proposed cap. Reserve four more calls for separately reviewed held-out current evidence; the exact reservation is 196 calls and $92.60 within a proposed $100 envelope." + ] +} diff --git a/roger/story_eval.py b/roger/story_eval.py new file mode 100644 index 0000000..05e8864 --- /dev/null +++ b/roger/story_eval.py @@ -0,0 +1,1088 @@ +"""Offline planning and explicitly budgeted model evaluation for Curated stories.""" + +from __future__ import annotations + +import argparse +import asyncio +import copy +import dataclasses +import datetime +import hashlib +import json +import os +import pathlib +import sys +import time +from decimal import Decimal, InvalidOperation +from typing import Any + +from openai import APIStatusError, AsyncOpenAI + +from roger.brains import stories +from roger.brains.curated import DraftError + +_ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_CASES = _ROOT / "evals/stories/cases.json" +DEFAULT_MODELS = _ROOT / "evals/stories/models.json" +_REQUEST_FRAMING_BYTES = 2_048 +_INPUT_TOKEN_RESERVE = 32_768 + + +class EvaluationError(RuntimeError): + """A local evaluation invariant failed.""" + + +@dataclasses.dataclass(frozen=True) +class ModelProfile: + name: str + model: str + returned_models: tuple[str, ...] + provider: str + service_tier: str + prompt_usd_per_million: Decimal + completion_usd_per_million: Decimal + reasoning_effort: str + writer_max_tokens: int + reviewer_max_tokens: int + available: bool + note: str + provider_display_name: str + + @classmethod + def from_record(cls, name: str, record: dict[str, Any]) -> ModelProfile: + pricing = record["pricing_usd_per_million"] + profile = cls( + name=name, + model=record["model"], + returned_models=tuple(record["returned_models"]), + provider=record["provider"], + service_tier=record["service_tier"], + prompt_usd_per_million=Decimal(str(pricing["prompt"])), + completion_usd_per_million=Decimal(str(pricing["completion"])), + reasoning_effort=record["reasoning_effort"], + writer_max_tokens=int(record["writer_max_tokens"]), + reviewer_max_tokens=int(record["reviewer_max_tokens"]), + available=bool(record["available"]), + note=str(record.get("note") or ""), + provider_display_name=record["provider_display_name"], + ) + if ( + not profile.model or not profile.returned_models or not profile.provider + or profile.service_tier != "default" + or profile.reasoning_effort != "medium" + or profile.writer_max_tokens <= 0 or profile.reviewer_max_tokens <= 0 + or any( + not price.is_finite() or price <= 0 + for price in ( + profile.prompt_usd_per_million, profile.completion_usd_per_million + ) + ) + ): + raise EvaluationError(f"invalid model profile: {name}") + return profile + + +@dataclasses.dataclass(frozen=True) +class CallSpec: + stage: str + case_id: str + repeat: int + profile: ModelProfile + messages: list[dict[str, str]] + response_format: dict[str, Any] + max_tokens: int + + @property + def request_id(self) -> str: + raw = f"{self.stage}:{self.profile.name}:{self.case_id}:{self.repeat}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + @property + def input_token_upper_bound(self) -> int: + content = json.dumps( + {"messages": self.messages, "response_format": self.response_format}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + # One token per UTF-8 byte plus fixed chat framing is deliberately conservative. + return len(content) + _REQUEST_FRAMING_BYTES + + @property + def reserved_cost_usd(self) -> Decimal: + if self.input_token_upper_bound > _INPUT_TOKEN_RESERVE: + raise EvaluationError( + f"request needs {self.input_token_upper_bound} input tokens, above the " + f"{_INPUT_TOKEN_RESERVE} token reservation" + ) + prompt = Decimal(_INPUT_TOKEN_RESERVE) * self.profile.prompt_usd_per_million + completion = Decimal(self.max_tokens) * self.profile.completion_usd_per_million + estimated = (prompt + completion) / Decimal(1_000_000) + return estimated.quantize(Decimal("0.01"), rounding="ROUND_CEILING") + + def inventory(self) -> dict[str, Any]: + return { + "request_id": self.request_id, + "stage": self.stage, + "case_id": self.case_id, + "repeat": self.repeat, + "profile": self.profile.name, + "model": self.profile.model, + "returned_models": list(self.profile.returned_models), + "provider": self.profile.provider, + "provider_display_name": self.profile.provider_display_name, + "service_tier": self.profile.service_tier, + "reasoning_effort": self.profile.reasoning_effort, + "max_tokens": self.max_tokens, + "timeout_seconds": 900 if "reviewer" in self.stage else 1200, + "input_token_upper_bound": self.input_token_upper_bound, + "input_token_reserve": _INPUT_TOKEN_RESERVE, + "reserved_cost_usd": str(self.reserved_cost_usd), + "available": self.profile.available, + } + + +class Ledger: + """A durable reservation ledger for one immutable run directory.""" + + def __init__(self, path: pathlib.Path, *, max_calls: int, max_cost_usd: Decimal) -> None: + if ( + type(max_calls) is not int or max_calls <= 0 + or not max_cost_usd.is_finite() or max_cost_usd <= 0 + ): + raise EvaluationError("run budgets must be positive and finite") + self.path = path + self.data: dict[str, Any] = { + "max_calls": max_calls, + "max_cost_usd": str(max_cost_usd), + "reserved_calls": 0, + "reserved_cost_usd": "0", + "used_calls": 0, + "used_cost_usd": "0", + "calls": [], + } + self._write() + + def _write(self) -> None: + temporary = self.path.with_suffix(".tmp") + with temporary.open("w", encoding="utf-8") as stream: + stream.write(json.dumps(self.data, indent=2) + "\n") + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(self.path) + directory_fd = os.open(self.path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + def reserve(self, spec: CallSpec) -> int: + calls = int(self.data["reserved_calls"]) + cost = Decimal(self.data["reserved_cost_usd"]) + if calls + 1 > int(self.data["max_calls"]): + raise EvaluationError("call budget would be exceeded") + if cost + spec.reserved_cost_usd > Decimal(self.data["max_cost_usd"]): + raise EvaluationError("cost budget would be exceeded") + row = { + **spec.inventory(), + "status": "reserved", + "actual_cost_usd": None, + "input_tokens": None, + "output_tokens": None, + "reasoning_tokens": None, + "latency_ms": None, + } + self.data["calls"].append(row) + self.data["reserved_calls"] = calls + 1 + self.data["reserved_cost_usd"] = str(cost + spec.reserved_cost_usd) + self._write() + return len(self.data["calls"]) - 1 + + def finish( + self, + index: int, + *, + status: str, + usage: dict[str, int | Decimal | None] | None = None, + latency_ms: int | None = None, + error: dict[str, Any] | None = None, + ) -> None: + row = self.data["calls"][index] + row["status"] = status + row["latency_ms"] = latency_ms + if error: + row["error"] = error + if usage is not None: + actual = Decimal(usage["cost_usd"]) + row.update( + actual_cost_usd=str(actual), + input_tokens=int(usage["input_tokens"]), + output_tokens=int(usage["output_tokens"]), + reasoning_tokens=usage["reasoning_tokens"], + ) + self.data["used_calls"] = int(self.data["used_calls"]) + 1 + self.data["used_cost_usd"] = str(Decimal(self.data["used_cost_usd"]) + actual) + if actual > Decimal(row["reserved_cost_usd"]): + row["status"] = "overspend" + self._write() + + +def _load_json(path: pathlib.Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise EvaluationError(f"cannot load {path}: {type(exc).__name__}") from exc + if not isinstance(value, dict): + raise EvaluationError(f"{path} must contain a JSON object") + return value + + +def load_profiles(path: pathlib.Path = DEFAULT_MODELS) -> dict[str, ModelProfile]: + raw = _load_json(path) + return { + name: ModelProfile.from_record(name, record) + for name, record in raw.get("profiles", {}).items() + } + + +def load_fixtures(path: pathlib.Path = DEFAULT_CASES) -> dict[str, Any]: + fixtures = _load_json(path) + if fixtures.get("label_status") != "author_expected_pending_human_review": + raise EvaluationError("fixture labels must be marked pending human review") + return fixtures + + +def _case_sources(case: dict[str, Any]) -> list[dict[str, Any]]: + return stories.evidence_bundle( + case["entries"], case["history"], + now=_case_now(case), max_age_hours=case["max_age_hours"] + ) + + +def _case_now(case: dict[str, Any]) -> datetime.datetime: + try: + now = datetime.datetime.fromisoformat(case["now"]) + except (KeyError, TypeError, ValueError) as exc: + raise EvaluationError(f"case {case.get('id')} has an invalid now") from exc + if now.tzinfo is None: + raise EvaluationError(f"case {case.get('id')} now must be timezone-aware") + return now.astimezone(datetime.UTC) + + +def _fixture_decision(fixture: dict[str, Any], case: dict[str, Any]) -> dict[str, Any]: + decision = copy.deepcopy(case["reference_decision"]) + seed = fixture["seed"] + if seed == "none": + return decision + if seed == "unsupported_claim": + decision["claims"][2]["text"] = "Atlas 2.0 is 40 percent faster than every competitor." + elif seed == "dropped_qualifier": + decision["claims"][2]["text"] = "Northstar is twice as fast as every competing cache." + decision["claims"][2]["citations"] = [{ + "source": 1, + "quote": ( + "Northstar says its preview is twice as fast as every competing cache, but the " + "post provides no benchmark data or comparison method." + ), + }] + elif seed == "invented_relationship": + decision["reason"]["text"] = "Lumen acquired Orbit to make Orbit its official client." + elif seed == "chronology_error": + decision["change"]["text"] = "The Windows client shipped before the Linux client." + decision["change"]["citations"] = [{ + "source": 1, + "quote": ( + "River's Windows client became generally available on October 8, 2026, after the " + "Linux client shipped on October 1." + ), + }] + elif seed == "stale_claim": + stale_quote = ( + "An undated forum summary speculates that Atlas might add mobile packages someday, " + "but it cites no maintainer statement, repository change, release artifact, or " + "schedule." + ) + decision = { + "action": "update", + "reason": {"text": "Atlas has made mobile packages available.", + "citations": [{"source": 2, "quote": stale_quote}]}, + "change": {"text": "Mobile support is a new material release.", + "citations": [{"source": 2, "quote": stale_quote}]}, + "story_ids": decision["story_ids"], + "claims": [ + {"field": "headline", "text": "Atlas adds mobile packages", + "citations": [{"source": 2, "quote": stale_quote}], "mutable": False}, + {"field": "stage", "text": "rollout", + "citations": [{"source": 2, "quote": stale_quote}], "mutable": False}, + {"field": "fact", "text": "Mobile packages are available now.", + "citations": [{"source": 2, "quote": stale_quote}], "mutable": False}, + {"field": "why", "text": "The package broadens Atlas platform support.", + "citations": [{"source": 2, "quote": stale_quote}], "mutable": False}, + ], + "unresolved_questions": [], + "correction_of": None, + } + elif seed == "incorrect_correction": + decision["change"]["text"] = "The active-project limit is 160 rather than 100." + else: + raise EvaluationError(f"unknown reviewer seed: {seed}") + return decision + + +def _writer_spec(case: dict[str, Any], profile: ModelProfile, repeat: int) -> CallSpec: + sources = _case_sources(case) + model_input = stories._model_input(sources, case["history"]) + return CallSpec( + stage="writer", + case_id=case["id"], + repeat=repeat, + profile=profile, + messages=stories._messages( + stories.SYSTEM, model_input, stories.DECISION_RESPONSE_FORMAT + ), + response_format=stories.DECISION_RESPONSE_FORMAT, + max_tokens=profile.writer_max_tokens, + ) + + +def _reviewer_spec( + fixture: dict[str, Any], cases: dict[str, dict[str, Any]], profile: ModelProfile, repeat: int +) -> CallSpec: + case = cases[fixture["case_id"]] + sources = _case_sources(case) + decision = stories.parse_decision( + json.dumps(_fixture_decision(fixture, case)), sources, case["history"] + ) + payload = { + "decision": stories._decision_projection(decision), + "sources": stories._source_metadata(sources), + "confirmed_sent": stories._delivered_view(case["history"]), + } + return CallSpec( + stage="reviewer", + case_id=fixture["id"], + repeat=repeat, + profile=profile, + messages=stories._messages( + stories.REVIEW, payload, stories.REVIEW_RESPONSE_FORMAT + ), + response_format=stories.REVIEW_RESPONSE_FORMAT, + max_tokens=profile.reviewer_max_tokens, + ) + + +def build_plan( + fixtures_path: pathlib.Path = DEFAULT_CASES, + models_path: pathlib.Path = DEFAULT_MODELS, +) -> dict[str, Any]: + fixtures = load_fixtures(fixtures_path) + profiles = load_profiles(models_path) + cases = {case["id"]: case for case in fixtures["cases"]} + writer_names = fixtures["matrix"]["writer_profiles"] + reviewer_names = fixtures["matrix"]["reviewer_profiles"] + repeats = int(fixtures["matrix"]["repeats"]) + writer = [ + _writer_spec(case, profiles[name], repeat).inventory() + for name in writer_names + for case in fixtures["cases"] + for repeat in range(1, repeats + 1) + ] + reviewer = [ + _reviewer_spec(fixture, cases, profiles[name], repeat).inventory() + for name in reviewer_names + for fixture in fixtures["reviewer_fixtures"] + for repeat in range(1, repeats + 1) + ] + + def total(rows: list[dict[str, Any]]) -> str: + return str(sum((Decimal(row["reserved_cost_usd"]) for row in rows), Decimal())) + + return { + "network_used": False, + "credentials_required": False, + "label_status": fixtures["label_status"], + "writer_screen": { + "calls": len(writer), + "cost_upper_bound_usd": total(writer), + "inventory": writer, + }, + "reviewer_calibration": { + "calls": len(reviewer), + "cost_upper_bound_usd": total(reviewer), + "inventory": reviewer, + "reference_gate": ( + "zero false accepts and zero false rejects; human confirmation still required" + ), + }, + "selected_pair_workflow": { + "cases": len(fixtures["cases"]), + "repeats": repeats, + "workflow_runs_per_pair": len(fixtures["cases"]) * repeats, + "maximum_calls_per_workflow": 4, + "maximum_calls_per_pair": len(fixtures["cases"]) * repeats * 4, + "maximum_calls_for_two_pairings": len(fixtures["cases"]) * repeats * 8, + "priced_after_writer_and_reviewer_selection": True, + }, + "notes": _load_json(models_path).get("notes", []), + } + + +def _request_kwargs(spec: CallSpec) -> dict[str, Any]: + return { + "model": spec.profile.model, + "messages": spec.messages, + "max_tokens": spec.max_tokens, + "response_format": spec.response_format, + "service_tier": spec.profile.service_tier, + "timeout": 900.0 if "reviewer" in spec.stage else 1200.0, + "extra_body": { + "provider": { + "only": [spec.profile.provider], + "allow_fallbacks": False, + "require_parameters": True, + "max_price": { + "prompt": float(spec.profile.prompt_usd_per_million), + "completion": float(spec.profile.completion_usd_per_million), + }, + }, + "reasoning": {"effort": spec.profile.reasoning_effort, "exclude": True}, + }, + } + + +def _request_record(spec: CallSpec) -> dict[str, Any]: + kwargs = _request_kwargs(spec) + return { + **spec.inventory(), + "messages": kwargs["messages"], + "response_format": kwargs["response_format"], + "parameters": { + key: value for key, value in kwargs.items() + if key not in {"messages", "response_format", "timeout"} + }, + } + + +def _route_metadata(response: Any) -> dict[str, Any] | None: + raw = getattr(response, "openrouter_metadata", None) + if raw is None: + extra = getattr(response, "model_extra", None) + raw = extra.get("openrouter_metadata") if isinstance(extra, dict) else None + if not isinstance(raw, dict): + return None + endpoints = raw.get("endpoints") + available = endpoints.get("available") if isinstance(endpoints, dict) else None + clean_endpoints = [] + if isinstance(available, list): + for endpoint in available: + if isinstance(endpoint, dict): + clean_endpoints.append({ + key: endpoint.get(key) for key in ("provider", "model", "selected") + }) + pipeline = raw.get("pipeline") + if isinstance(pipeline, list): + pipeline_stages = [str(stage) for stage in pipeline] + elif isinstance(pipeline, dict): + pipeline_stages = [str(key) for key, value in pipeline.items() if value] + else: + pipeline_stages = [] + attempts = raw.get("attempts") + return { + "requested": raw.get("requested"), + "strategy": raw.get("strategy"), + "attempt": raw.get("attempt"), + "attempt_count": len(attempts) if isinstance(attempts, list) else None, + "available": clean_endpoints, + "pipeline_stages": pipeline_stages, + } + + +def _visible_response(response: Any) -> dict[str, Any]: + choice = response.choices[0] + message = choice.message + return { + "model": getattr(response, "model", None), + "finish_reason": getattr(choice, "finish_reason", None), + "content": getattr(message, "content", None), + "refusal": getattr(message, "refusal", None), + "service_tier": getattr(response, "service_tier", None), + "route_metadata": _route_metadata(response), + } + + +def _usage(response: Any) -> dict[str, int | Decimal | None] | None: + usage = getattr(response, "usage", None) + if usage is None: + return None + cost = getattr(usage, "cost", None) + input_tokens = getattr(usage, "prompt_tokens", None) + output_tokens = getattr(usage, "completion_tokens", None) + if cost is None or input_tokens is None or output_tokens is None: + return None + details = getattr(usage, "completion_tokens_details", None) + reasoning_tokens = getattr(details, "reasoning_tokens", None) if details else None + try: + parsed_cost = Decimal(str(cost)) + except (InvalidOperation, TypeError, ValueError): + return None + counts = (input_tokens, output_tokens) + if ( + not parsed_cost.is_finite() or parsed_cost < 0 + or any(type(value) is not int or value < 0 for value in counts) + or (reasoning_tokens is not None + and (type(reasoning_tokens) is not int or reasoning_tokens < 0)) + ): + return None + return { + "cost_usd": parsed_cost, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "reasoning_tokens": reasoning_tokens, + } + + +def _response_status(visible: dict[str, Any], profile: ModelProfile) -> str: + if visible["model"] not in profile.returned_models: + return "model_mismatch" + if visible["service_tier"] is not None and visible["service_tier"] != profile.service_tier: + return "service_tier_mismatch" + if visible["finish_reason"] == "length": + return "finish_truncation" + if visible["finish_reason"] == "content_filter": + return "content_filter" + if visible["refusal"]: + return "refusal" + if not isinstance(visible["content"], str) or not visible["content"].strip(): + return "blank_response" + metadata = visible["route_metadata"] + if metadata is None: + return "missing_route_metadata" + if metadata["requested"] != profile.model or metadata["attempt"] != 1: + return "route_metadata_mismatch" + if metadata["pipeline_stages"]: + return "pipeline_interference" + if metadata["attempt_count"] not in (None, 1): + return "routing_multiple_attempts" + selected = [row for row in metadata["available"] if row["selected"] is True] + if len(selected) != 1: + return "missing_selected_endpoint" + expected_provider = "".join(profile.provider_display_name.lower().split()) + actual_provider = "".join(str(selected[0]["provider"]).lower().split()) + if actual_provider != expected_provider or selected[0]["model"] not in profile.returned_models: + return "route_mismatch" + return "completed" + + +def _safe_error(exc: Exception) -> dict[str, Any]: + result: dict[str, Any] = {"class": type(exc).__name__} + if isinstance(exc, APIStatusError): + result["status_code"] = exc.status_code + return result + + +class Runner: + def __init__(self, client: AsyncOpenAI, ledger: Ledger, responses: pathlib.Path) -> None: + self.client = client + self.ledger = ledger + self.responses = responses + self.call_count = 0 + + async def call(self, spec: CallSpec) -> tuple[Any, dict[str, Any]]: + if not spec.profile.available: + raise EvaluationError( + f"profile {spec.profile.name} is marked unavailable: {spec.profile.note}" + ) + index = self.ledger.reserve(spec) + started = time.monotonic() + try: + response = await self.client.chat.completions.create(**_request_kwargs(spec)) + except Exception as exc: + latency = round((time.monotonic() - started) * 1000) + self.ledger.finish( + index, status="request_error_uncertain", latency_ms=latency, + error=_safe_error(exc), + ) + raise EvaluationError(f"request failed: {type(exc).__name__}") from exc + latency = round((time.monotonic() - started) * 1000) + visible = _visible_response(response) + usage = _usage(response) + status = _response_status(visible, spec.profile) + if usage is None: + status = "missing_usage_or_cost" + elif ( + usage["input_tokens"] > _INPUT_TOKEN_RESERVE + or usage["output_tokens"] > spec.max_tokens + or ( + usage["reasoning_tokens"] is not None + and usage["reasoning_tokens"] > usage["output_tokens"] + ) + ): + status = "invalid_usage_accounting" + self.ledger.finish(index, status=status, usage=usage, latency_ms=latency) + artifact = { + "request": _request_record(spec), + "response": visible, + "accounting": usage and {key: str(value) for key, value in usage.items()}, + "status": self.ledger.data["calls"][index]["status"], + } + destination = self.responses / f"{index + 1:03d}-{spec.request_id}.json" + destination.write_text(json.dumps(artifact, indent=2) + "\n", encoding="utf-8") + self.call_count += 1 + if self.ledger.data["calls"][index]["status"] != "completed": + raise EvaluationError(f"call stopped: {self.ledger.data['calls'][index]['status']}") + return response, visible + + +class WorkflowLLM: + """The narrow adapter needed by the production ``stories.decide`` workflow.""" + + def __init__( + self, + runner: Runner, + writer: ModelProfile, + reviewer: ModelProfile, + case_id: str, + repeat: int, + ) -> None: + self.runner = runner + self.writer = writer + self.reviewer = reviewer + self.case_id = case_id + self.repeat = repeat + self.first_call = runner.call_count + + async def complete( + self, + brain: str, + messages: list[dict[str, str]], + tools: list[dict[str, Any]] | None = None, + *, + curated_review: bool = False, + curated_story: bool = False, + response_format: dict[str, Any] | None = None, + ) -> Any: + if brain != "curated" or tools or response_format is None: + raise EvaluationError("unexpected production workflow request") + if self.runner.call_count - self.first_call >= 4: + raise EvaluationError("production workflow exceeded four calls") + profile = self.reviewer if curated_review else self.writer + stage = "workflow-reviewer" if curated_review else "workflow-writer" + max_tokens = profile.reviewer_max_tokens if curated_review else profile.writer_max_tokens + spec = CallSpec( + stage=stage, + case_id=self.case_id, + repeat=self.repeat, + profile=profile, + messages=messages, + response_format=response_format, + max_tokens=max_tokens, + ) + response, _ = await self.runner.call(spec) + return response + + +def _create_run_dir(path: pathlib.Path) -> pathlib.Path: + try: + path.mkdir(parents=True, exist_ok=False) + except FileExistsError as exc: + raise EvaluationError("artifact directory already exists; runs cannot be resumed") from exc + responses = path / "responses" + responses.mkdir() + return responses + + +def _snapshot_inputs( + artifact_dir: pathlib.Path, + fixtures_path: pathlib.Path, + models_path: pathlib.Path, + fixtures: dict[str, Any], +) -> None: + fixtures_bytes = fixtures_path.read_bytes() + models_bytes = models_path.read_bytes() + (artifact_dir / "inputs.json").write_text(json.dumps({ + "fixtures_path": str(fixtures_path), + "fixtures_sha256": hashlib.sha256(fixtures_bytes).hexdigest(), + "fixtures": fixtures, + "models_path": str(models_path), + "models_sha256": hashlib.sha256(models_bytes).hexdigest(), + "models": _load_json(models_path), + }, indent=2) + "\n", encoding="utf-8") + + +def _validate_decision_output( + visible: dict[str, Any], case: dict[str, Any] +) -> tuple[str, dict[str, Any] | None]: + try: + decision = stories.parse_decision( + visible["content"], _case_sources(case), case["history"] + ) + except DraftError as exc: + return "local_parse_failure", {"class": type(exc).__name__, "code": str(exc)} + return "local_parse_pass", decision.record() + + +def _source_inventory(case: dict[str, Any]) -> list[dict[str, Any]]: + prepared = _case_sources(case) + by_url = {source["url"]: source for source in prepared} + inventory = [] + for entry in case["entries"]: + article = entry.get("article") or {} + text = str(article.get("text") or "") + source = by_url.get(article.get("url")) + delivered_chars = len(source["excerpt"]) if source else 0 + inventory.append({ + "url": article.get("url"), + "source_admitted": source is not None, + "original_chars": len(text), + "delivered_excerpt_chars": delivered_chars, + "omitted_chars": max(0, len(text) - delivered_chars), + }) + return inventory + + +def _review_verdict( + visible: dict[str, Any], fixture: dict[str, Any], case: dict[str, Any] +) -> dict[str, Any]: + expected_accept = fixture["expected"]["review_accept"] + + def invalid(error_class: str) -> dict[str, Any]: + return { + "status": "local_parse_failure", + "error_class": error_class, + "expected_label_pending_human_review": expected_accept, + "matches_expected": False, + } + + try: + verdict = json.loads(visible["content"]) + except (TypeError, json.JSONDecodeError) as exc: + return invalid(type(exc).__name__) + expected_keys = { + "supported", "material_change", "action_supported", "chronology_supported", + "relationship_supported", "freshness_supported", "correction_supported", + } + decision = stories.parse_decision( + json.dumps(_fixture_decision(fixture, case)), _case_sources(case), case["history"] + ) + supported = verdict.get("supported") if isinstance(verdict, dict) else None + gates = expected_keys - {"supported", "material_change"} + if ( + not isinstance(verdict, dict) + or set(verdict) != expected_keys + or not isinstance(supported, list) + or len(supported) != len(stories._decision_projection(decision)["items"]) + or any(type(value) is not bool for value in supported) + or any(type(verdict[key]) is not bool for key in gates | {"material_change"}) + ): + return invalid("InvalidReviewFields") + material_matches = verdict["material_change"] is decision.publishable + accepted = all(supported) and all(verdict[key] for key in gates) and material_matches + failed = ([] if all(supported) else ["supported"]) + failed.extend(key for key in sorted(gates) if not verdict[key]) + if not material_matches: + failed.append("material_change") + return { + "status": "reviewer_accept" if accepted else "reviewer_reject", + "expected_label_pending_human_review": expected_accept, + "matches_expected": accepted is expected_accept, + "failed_gates": failed, + "verdict": verdict, + } + + +async def run_matrix( + *, + stage: str, + artifact_dir: pathlib.Path, + max_calls: int, + max_cost_usd: Decimal, + profile_name: str | None, + writer_name: str | None, + reviewer_name: str | None, + case_ids: list[str], + repeats: int, + fixtures_path: pathlib.Path = DEFAULT_CASES, + models_path: pathlib.Path = DEFAULT_MODELS, + client: AsyncOpenAI | None = None, +) -> dict[str, Any]: + fixtures = load_fixtures(fixtures_path) + profiles = load_profiles(models_path) + cases = {case["id"]: case for case in fixtures["cases"]} + selected = case_ids or list(cases) + if stage != "reviewer" and (unknown := sorted(set(selected) - set(cases))): + raise EvaluationError(f"unknown cases: {', '.join(unknown)}") + if stage in {"writer", "reviewer"} and profile_name not in profiles: + raise EvaluationError(f"{stage} run requires a known --model") + if stage == "workflow" and (writer_name not in profiles or reviewer_name not in profiles): + raise EvaluationError("workflow run requires known --writer-model and --reviewer-model") + owns_client = client is None + if owns_client: + key = os.environ.get("OPENROUTER_API_KEY") + if not key: + raise EvaluationError("OPENROUTER_API_KEY is required only for run") + responses = _create_run_dir(artifact_dir) + _snapshot_inputs(artifact_dir, fixtures_path, models_path, fixtures) + ledger = Ledger( + artifact_dir / "ledger.json", max_calls=max_calls, max_cost_usd=max_cost_usd + ) + if owns_client: + client = AsyncOpenAI( + api_key=key, + base_url="https://openrouter.ai/api/v1", + max_retries=0, + timeout=1200.0, + default_headers={ + "X-OpenRouter-Metadata": "enabled", + "X-OpenRouter-Cache": "false", + }, + ) + runner = Runner(client, ledger, responses) + results: list[dict[str, Any]] = [] + + def persist(run_status: str, error: Exception | None = None) -> dict[str, Any]: + summary = { + "label_status": fixtures["label_status"], + "stage": stage, + "run_status": run_status, + "results": results, + "human_rubric": fixtures["human_rubric"], + "accounting": { + key: ledger.data[key] + for key in ( + "max_calls", "max_cost_usd", "reserved_calls", "reserved_cost_usd", + "used_calls", "used_cost_usd", + ) + }, + "human_acceptance": "pending", + } + if error is not None: + summary["terminal_error"] = _safe_error(error) + (artifact_dir / "summary.json").write_text( + json.dumps(summary, indent=2) + "\n", encoding="utf-8" + ) + return summary + + persist("running") + try: + if stage == "writer": + for case_id in selected: + for repeat in range(1, repeats + 1): + case = cases[case_id] + _, visible = await runner.call( + _writer_spec(case, profiles[profile_name], repeat) + ) + status, decision = _validate_decision_output(visible, case) + results.append( + {"case_id": case_id, "repeat": repeat, "status": status, + "decision": decision, + "reference_expected": case["expected"], + "false_hold_on_reportable": bool( + status == "local_parse_pass" and case["expected"]["reportable"] + and decision["action"] in {"hold", "skip"} + ), + "source_inventory": _source_inventory(case), + "human_rating": None} + ) + persist("running") + elif stage == "reviewer": + review_fixtures = { + fixture["id"]: fixture for fixture in fixtures["reviewer_fixtures"] + } + selected_fixtures = case_ids or list(review_fixtures) + if unknown := sorted(set(selected_fixtures) - set(review_fixtures)): + raise EvaluationError(f"unknown reviewer fixtures: {', '.join(unknown)}") + for fixture_id in selected_fixtures: + for repeat in range(1, repeats + 1): + fixture = review_fixtures[fixture_id] + _, visible = await runner.call( + _reviewer_spec(fixture, cases, profiles[profile_name], repeat) + ) + results.append({"fixture_id": fixture_id, "repeat": repeat, + **_review_verdict(visible, fixture, cases[fixture["case_id"]])}) + persist("running") + else: + for case_id in selected: + for repeat in range(1, repeats + 1): + before = runner.call_count + llm = WorkflowLLM( + runner, profiles[writer_name], profiles[reviewer_name], case_id, repeat + ) + try: + decision, _ = await stories.decide( + cases[case_id]["entries"], cases[case_id]["history"], llm, + now=_case_now(cases[case_id]), + max_age_hours=cases[case_id]["max_age_hours"], + ) + except DraftError as exc: + result = {"status": "workflow_rejected", "error_class": type(exc).__name__} + else: + result = {"status": "workflow_accepted", "decision": decision.record()} + results.append( + {"case_id": case_id, "repeat": repeat, + "calls": runner.call_count - before, + "reference_expected": cases[case_id]["expected"], + "source_inventory": _source_inventory(cases[case_id]), + "human_rating": None, **result} + ) + persist("running") + except Exception as exc: + persist("stopped", exc) + raise + finally: + if owns_client and client is not None and hasattr(client, "close"): + await client.close() + return persist("complete") + + +def summarize(path: pathlib.Path) -> dict[str, Any]: + ledger = _load_json(path / "ledger.json") + summary_path = path / "summary.json" + result = _load_json(summary_path) if summary_path.exists() else {} + statuses: dict[str, int] = {} + for row in ledger.get("calls", []): + statuses[row["status"]] = statuses.get(row["status"], 0) + 1 + reviewer_rows = [ + row for row in result.get("results", []) + if "expected_label_pending_human_review" in row + ] + expected_matches = [row["matches_expected"] for row in reviewer_rows] + confusion = { + "expected_accept": 0, + "expected_reject": 0, + "false_accept": 0, + "false_reject": 0, + } + for row in reviewer_rows: + expected = row["expected_label_pending_human_review"] + actual = row["status"] == "reviewer_accept" + confusion["expected_accept" if expected else "expected_reject"] += 1 + if actual and not expected: + confusion["false_accept"] += 1 + if expected and not actual: + confusion["false_reject"] += 1 + invalid_reviews = sum(row["status"] == "local_parse_failure" for row in reviewer_rows) + full_coverage = False + inputs_path = path / "inputs.json" + if result.get("stage") == "reviewer" and inputs_path.exists(): + inputs = _load_json(inputs_path) + fixture_data = inputs.get("fixtures") or {} + expected_repeats = (fixture_data.get("matrix") or {}).get("repeats") + expected_ids = [ + fixture.get("id") for fixture in fixture_data.get("reviewer_fixtures", []) + if isinstance(fixture, dict) and isinstance(fixture.get("id"), str) + ] + actual_counts = { + fixture_id: sum(row.get("fixture_id") == fixture_id for row in reviewer_rows) + for fixture_id in expected_ids + } + full_coverage = ( + type(expected_repeats) is int + and expected_repeats > 0 + and bool(expected_ids) + and len(reviewer_rows) == len(expected_ids) * expected_repeats + and all(count == expected_repeats for count in actual_counts.values()) + ) + completed_run = ( + result.get("run_status") == "complete" + and statuses.get("completed", 0) == len(reviewer_rows) + and ledger.get("reserved_calls") == len(reviewer_rows) + ) + reference_gate_pass = ( + completed_run + and full_coverage + and invalid_reviews == 0 + and confusion["false_accept"] == 0 + and confusion["false_reject"] == 0 + ) + return { + "calls_by_status": statuses, + "accounting": { + key: ledger.get(key) for key in ( + "max_calls", "max_cost_usd", "reserved_calls", "reserved_cost_usd", + "used_calls", "used_cost_usd", + ) + }, + "reviewer_expected_matches": { + "matched": sum(expected_matches), "total": len(expected_matches), + "labels": "author expected, pending human review", + **confusion, + "invalid_reviews": invalid_reviews, + "completed_run": completed_run, + "full_fixture_coverage": full_coverage, + "reference_gate_pass": reference_gate_pass, + "selection_authorized": False, + }, + "human_acceptance": "pending", + } + + +def _decimal(value: str) -> Decimal: + try: + parsed = Decimal(value) + except InvalidOperation as exc: + raise argparse.ArgumentTypeError("must be a decimal number") from exc + if not parsed.is_finite() or parsed <= 0: + raise argparse.ArgumentTypeError("must be positive and finite") + return parsed + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command") + plan = sub.add_parser("plan", help="print the network-free priced request inventory") + plan.add_argument("--fixtures", type=pathlib.Path, default=DEFAULT_CASES) + plan.add_argument("--models", type=pathlib.Path, default=DEFAULT_MODELS) + run = sub.add_parser("run", help="perform an explicitly approved paid evaluation run") + run.add_argument("--stage", choices=("writer", "reviewer", "workflow"), required=True) + run.add_argument("--artifact-dir", type=pathlib.Path, required=True) + run.add_argument("--max-calls", type=int, required=True) + run.add_argument("--max-cost-usd", type=_decimal, required=True) + run.add_argument("--approve-paid-calls", action="store_true", required=True) + run.add_argument("--model") + run.add_argument("--writer-model") + run.add_argument("--reviewer-model") + run.add_argument("--case", action="append", default=[]) + run.add_argument("--repeats", type=int, default=2) + run.add_argument("--fixtures", type=pathlib.Path, default=DEFAULT_CASES) + run.add_argument("--models", type=pathlib.Path, default=DEFAULT_MODELS) + report = sub.add_parser("summarize", help="summarize local artifacts without model calls") + report.add_argument("artifact_dir", type=pathlib.Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + command = args.command or "plan" + try: + if command == "plan": + output = build_plan( + getattr(args, "fixtures", DEFAULT_CASES), getattr(args, "models", DEFAULT_MODELS) + ) + elif command == "summarize": + output = summarize(args.artifact_dir) + else: + if args.repeats <= 0: + raise EvaluationError("repeats must be positive") + output = asyncio.run( + run_matrix( + stage=args.stage, + artifact_dir=args.artifact_dir, + max_calls=args.max_calls, + max_cost_usd=args.max_cost_usd, + profile_name=args.model, + writer_name=args.writer_model, + reviewer_name=args.reviewer_model, + case_ids=args.case, + repeats=args.repeats, + fixtures_path=args.fixtures, + models_path=args.models, + ) + ) + except EvaluationError as exc: + print(f"story evaluation stopped: {exc}", file=sys.stderr) + return 2 + print(json.dumps(output, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_story_eval.py b/tests/test_story_eval.py new file mode 100644 index 0000000..dcaba95 --- /dev/null +++ b/tests/test_story_eval.py @@ -0,0 +1,463 @@ +import json +from decimal import Decimal +from types import SimpleNamespace + +import httpx +import pytest +from openai import AsyncOpenAI + +from roger import story_eval +from roger.brains import stories + + +def _completion( + content, + *, + model="openai/gpt-6.1-sol", + cost=0.01, + reasoning_tokens=11, + finish_reason="stop", + provider="OpenAI", + include_usage=True, + pipeline=None, + service_tier="default", +): + payload = { + "id": "eval-response", + "object": "chat.completion", + "created": 1, + "model": model, + "service_tier": service_tier, + "choices": [{ + "index": 0, + "finish_reason": finish_reason, + "message": {"role": "assistant", "content": content}, + }], + "openrouter_metadata": { + "requested": "openai/gpt-6.1-sol-20260929", + "strategy": "only", + "attempt": 1, + "attempts": [{"provider": provider}], + "endpoints": {"available": [{ + "provider": provider, "model": model, "selected": True, + }]}, + "pipeline": pipeline or [], + }, + } + if include_usage: + payload["usage"] = { + "prompt_tokens": 100, + "completion_tokens": 25, + "total_tokens": 125, + "cost": float(cost), + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, + } + return payload + + +def _response(content): + return SimpleNamespace( + choices=[SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace(content=content, refusal=None), + )] + ) + + +def _review(accepted=True, item_count=6): + return json.dumps({ + "supported": [accepted, *([True] * (item_count - 1))], + "material_change": True, + "action_supported": True, + "chronology_supported": True, + "relationship_supported": True, + "freshness_supported": True, + "correction_supported": True, + }) + + +def _fixture_data(): + fixtures = story_eval.load_fixtures() + cases = {case["id"]: case for case in fixtures["cases"]} + reviews = {fixture["id"]: fixture for fixture in fixtures["reviewer_fixtures"]} + return fixtures, cases, reviews + + +def test_fixture_reference_decisions_and_seeded_reviews_use_delivered_evidence(): + fixtures, cases, _ = _fixture_data() + assert [case["id"] for case in fixtures["cases"]] == [ + "straight_release", + "skeptical_marketing", + "material_update", + "correction", + "legitimate_combine", + "duplicate_stale_hold", + ] + assert all(case["now"].endswith("+00:00") for case in fixtures["cases"]) + for case in fixtures["cases"]: + sources = story_eval._case_sources(case) + assert sources + assert all(source["url"].endswith(".test/" + source["url"].rsplit("/", 1)[-1]) + for source in sources) + assert all(len(entry["article"]["text"]) >= 350 for entry in case["entries"]) + current_ids = {entry["id"] for entry in case["entries"]} + for source in sources: + if source.get("entry_id") in current_ids: + assert source["excerpt"] in next( + entry["article"]["text"] for entry in case["entries"] + if entry["article"]["url"] == source["url"] + ) + decision = stories.parse_decision( + json.dumps(case["reference_decision"]), sources, case["history"] + ) + assert decision.action == case["expected"]["action"] + for value in (decision.reason, decision.change, *decision.claims): + for citation in value["citations"]: + assert citation["quote"] in sources[citation["source"] - 1]["excerpt"] + + for fixture in fixtures["reviewer_fixtures"]: + case = cases[fixture["case_id"]] + stories.parse_decision( + json.dumps(story_eval._fixture_decision(fixture, case)), + story_eval._case_sources(case), + case["history"], + ) + + +def test_fixture_history_targets_are_confirmed_and_first_four(): + _, cases, _ = _fixture_data() + assert cases["material_update"]["history"]["delivered"][0]["id"] == 301 + correction = cases["correction"] + assert correction["reference_decision"]["correction_of"] == 401 + assert correction["history"]["delivered"][0]["id"] == 401 + for case in cases.values(): + for delivered in case["history"]["delivered"]: + assert delivered["message_id"].startswith("example-message-") + assert delivered["message_url"].startswith("https://discord.example.invalid/") + + +def test_offline_plan_does_not_construct_client(monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("plan constructed a client") + + monkeypatch.setattr(story_eval, "AsyncOpenAI", forbidden) + plan = story_eval.build_plan() + assert plan["network_used"] is False + assert plan["credentials_required"] is False + assert plan["writer_screen"]["calls"] == 48 + assert plan["reviewer_calibration"]["calls"] == 48 + assert plan["writer_screen"]["cost_upper_bound_usd"] == "20.40" + assert plan["reviewer_calibration"]["cost_upper_bound_usd"] == "7.20" + assert plan["selected_pair_workflow"]["maximum_calls_for_two_pairings"] == 96 + + +def test_input_reservation_counts_utf8_bytes_and_full_schema(): + _, cases, _ = _fixture_data() + profile = story_eval.load_profiles()["sol"] + spec = story_eval._writer_spec(cases["straight_release"], profile, 1) + replacement = story_eval.CallSpec( + **{**spec.__dict__, "messages": [*spec.messages, {"role": "user", "content": "é"}]} + ) + ascii_replacement = story_eval.CallSpec( + **{**spec.__dict__, "messages": [*spec.messages, {"role": "user", "content": "e"}]} + ) + assert replacement.input_token_upper_bound == ascii_replacement.input_token_upper_bound + 1 + assert spec.inventory()["input_token_reserve"] == 32768 + + +@pytest.mark.asyncio +async def test_sdk_request_is_exactly_pinned_and_has_no_retry_or_sampling(tmp_path): + _, cases, _ = _fixture_data() + profile = story_eval.load_profiles()["sol"] + spec = story_eval._writer_spec(cases["straight_release"], profile, 1) + requests = [] + + async def handle(request): + requests.append(request) + content = json.dumps(cases["straight_release"]["reference_decision"]) + return httpx.Response(200, json=_completion(content)) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + client = AsyncOpenAI( + api_key="test", base_url="https://openrouter.ai/api/v1", + max_retries=0, http_client=http_client, + default_headers={"X-OpenRouter-Metadata": "enabled", "X-OpenRouter-Cache": "false"}, + ) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1")) + responses = tmp_path / "responses" + responses.mkdir() + runner = story_eval.Runner(client, ledger, responses) + await runner.call(spec) + await client.close() + + assert len(requests) == 1 + request = json.loads(requests[0].content) + assert request["model"] == "openai/gpt-6.1-sol-20260929" + assert request["service_tier"] == "default" + assert request["max_tokens"] == 16384 + assert request["provider"] == { + "only": ["openai"], + "allow_fallbacks": False, + "require_parameters": True, + "max_price": {"prompt": 2.0, "completion": 10.0}, + } + assert request["reasoning"] == {"effort": "medium", "exclude": True} + assert request["response_format"]["json_schema"]["strict"] is True + assert not {"temperature", "top_p", "models"}.intersection(request) + assert requests[0].headers["x-openrouter-metadata"] == "enabled" + assert requests[0].headers["x-openrouter-cache"] == "false" + + +def test_ledger_reserves_before_calls_and_enforces_both_limits(tmp_path): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("0.23")) + ledger.reserve(spec) + durable = json.loads((tmp_path / "ledger.json").read_text()) + assert durable["reserved_calls"] == 1 + assert durable["reserved_cost_usd"] == "0.23" + with pytest.raises(story_eval.EvaluationError, match="call budget"): + ledger.reserve(spec) + + costly = story_eval.Ledger(tmp_path / "cost.json", max_calls=2, max_cost_usd=Decimal("0.22")) + with pytest.raises(story_eval.EvaluationError, match="cost budget"): + costly.reserve(spec) + + +def test_run_directory_is_exclusive(tmp_path): + path = tmp_path / "one-run" + story_eval._create_run_dir(path) + with pytest.raises(story_eval.EvaluationError, match="cannot be resumed"): + story_eval._create_run_dir(path) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("response", "expected"), + [ + (_completion("{}", include_usage=False), "missing_usage_or_cost"), + (_completion("{}", cost=Decimal("2")), "overspend"), + (_completion("{}", pipeline=["response_healing"]), "pipeline_interference"), + ], +) +async def test_failed_accounting_and_pipeline_keep_reservation(tmp_path, response, expected): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) + + async def handle(request): + return httpx.Response(200, json=response) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + client = AsyncOpenAI( + api_key="test", base_url="https://openrouter.ai/api/v1", + max_retries=0, http_client=http_client, + ) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("3")) + responses = tmp_path / "responses" + responses.mkdir() + with pytest.raises(story_eval.EvaluationError, match="call stopped"): + await story_eval.Runner(client, ledger, responses).call(spec) + await client.close() + assert ledger.data["reserved_calls"] == 1 + assert ledger.data["calls"][0]["status"] == expected + + +@pytest.mark.asyncio +async def test_missing_optional_reasoning_usage_stays_unknown(tmp_path): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec( + cases["straight_release"], story_eval.load_profiles()["sol"], 1 + ) + + async def handle(request): + return httpx.Response( + 200, json=_completion("{}", reasoning_tokens=None, service_tier=None) + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + client = AsyncOpenAI( + api_key="test", base_url="https://openrouter.ai/api/v1", + max_retries=0, http_client=http_client, + ) + ledger = story_eval.Ledger( + tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1") + ) + responses = tmp_path / "responses" + responses.mkdir() + await story_eval.Runner(client, ledger, responses).call(spec) + await client.close() + assert ledger.data["calls"][0]["status"] == "completed" + assert ledger.data["calls"][0]["reasoning_tokens"] is None + + +@pytest.mark.asyncio +async def test_request_error_has_one_attempt_and_sanitized_durable_error(tmp_path): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) + attempts = 0 + + async def handle(request): + nonlocal attempts + attempts += 1 + return httpx.Response(503, json={"error": {"message": "sensitive upstream text"}}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + client = AsyncOpenAI( + api_key="test", base_url="https://openrouter.ai/api/v1", + max_retries=0, http_client=http_client, + ) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1")) + responses = tmp_path / "responses" + responses.mkdir() + with pytest.raises(story_eval.EvaluationError, match="InternalServerError"): + await story_eval.Runner(client, ledger, responses).call(spec) + await client.close() + assert attempts == 1 + row = json.loads((tmp_path / "ledger.json").read_text())["calls"][0] + assert row["status"] == "request_error_uncertain" + assert row["error"] == {"class": "InternalServerError", "status_code": 503} + assert "sensitive" not in json.dumps(row) + + +def test_reviewer_report_matches_production_validation_rules(): + _, cases, reviews = _fixture_data() + fixture = reviews["good_release"] + accepted = story_eval._review_verdict( + {"content": _review()}, fixture, cases["straight_release"] + ) + assert accepted["status"] == "reviewer_accept" + assert accepted["matches_expected"] is True + + invalid = story_eval._review_verdict( + {"content": _review(item_count=0)}, fixture, cases["straight_release"] + ) + assert invalid["status"] == "local_parse_failure" + assert invalid["expected_label_pending_human_review"] is True + assert invalid["matches_expected"] is False + + seeded = reviews["unsupported_claim"] + rejected = story_eval._review_verdict( + {"content": _review(accepted=False)}, seeded, cases["straight_release"] + ) + assert rejected["status"] == "reviewer_reject" + assert rejected["matches_expected"] is True + + +class _QueueRunner: + def __init__(self, contents): + self.contents = iter(contents) + self.call_count = 0 + + async def call(self, spec): + self.call_count += 1 + response = _response(next(self.contents)) + return response, {"content": response.choices[0].message.content} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("contents", "expected_calls"), + [ + ("accepted", 2), + ("local_revision", 3), + ("review_revision", 4), + ], +) +async def test_actual_production_workflow_uses_two_three_or_four_calls(contents, expected_calls): + _, cases, _ = _fixture_data() + case = cases["straight_release"] + good = json.dumps(case["reference_decision"]) + if contents == "accepted": + queue = [good, _review()] + elif contents == "local_revision": + queue = ["{}", good, _review()] + else: + queue = [good, _review(accepted=False), good, _review()] + runner = _QueueRunner(queue) + profiles = story_eval.load_profiles() + llm = story_eval.WorkflowLLM(runner, profiles["sol"], profiles["sonnet"], case["id"], 1) + decision, _ = await stories.decide( + case["entries"], case["history"], llm, + now=story_eval._case_now(case), max_age_hours=case["max_age_hours"], + ) + assert decision.action == "publish" + assert runner.call_count == expected_calls + + +@pytest.mark.asyncio +async def test_run_saves_visible_parse_failure_and_leaves_human_assessment_pending(tmp_path): + async def handle(request): + return httpx.Response(200, json=_completion("{}")) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + client = AsyncOpenAI( + api_key="test", base_url="https://openrouter.ai/api/v1", + max_retries=0, http_client=http_client, + ) + artifact_dir = tmp_path / "artifacts" + summary = await story_eval.run_matrix( + stage="writer", artifact_dir=artifact_dir, + max_calls=1, max_cost_usd=Decimal("1"), profile_name="sol", + writer_name=None, reviewer_name=None, + case_ids=["straight_release"], repeats=1, client=client, + ) + await client.close() + assert summary["results"][0]["status"] == "local_parse_failure" + assert summary["human_acceptance"] == "pending" + response = json.loads(next((artifact_dir / "responses").iterdir()).read_text()) + assert response["response"]["content"] == "{}" + assert "reasoning" not in response["response"] + + +def test_summary_reports_reviewer_false_accepts_without_claiming_human_acceptance(tmp_path): + (tmp_path / "ledger.json").write_text(json.dumps({ + "max_calls": 1, "max_cost_usd": "1", "reserved_calls": 1, + "reserved_cost_usd": "0.15", "used_calls": 1, "used_cost_usd": "0.01", + "calls": [{"status": "completed"}], + })) + (tmp_path / "summary.json").write_text(json.dumps({"results": [{ + "status": "reviewer_accept", + "expected_label_pending_human_review": False, + "matches_expected": False, + }]})) + report = story_eval.summarize(tmp_path) + assert report["reviewer_expected_matches"]["false_accept"] == 1 + assert report["reviewer_expected_matches"]["reference_gate_pass"] is False + assert report["human_acceptance"] == "pending" + + +def test_summary_does_not_drop_invalid_review_from_mixed_calibration(tmp_path): + (tmp_path / "ledger.json").write_text(json.dumps({ + "max_calls": 2, "max_cost_usd": "1", "reserved_calls": 2, + "reserved_cost_usd": "0.30", "used_calls": 2, "used_cost_usd": "0.02", + "calls": [{"status": "completed"}, {"status": "completed"}], + })) + (tmp_path / "inputs.json").write_text(json.dumps({ + "fixtures": { + "matrix": {"repeats": 1}, + "reviewer_fixtures": [{"id": "good"}, {"id": "malformed"}], + } + })) + (tmp_path / "summary.json").write_text(json.dumps({ + "stage": "reviewer", + "run_status": "complete", + "results": [ + { + "fixture_id": "good", + "status": "reviewer_accept", + "expected_label_pending_human_review": True, + "matches_expected": True, + }, + { + "fixture_id": "malformed", + "status": "local_parse_failure", + "expected_label_pending_human_review": False, + "matches_expected": False, + }, + ], + })) + report = story_eval.summarize(tmp_path)["reviewer_expected_matches"] + assert report["invalid_reviews"] == 1 + assert report["completed_run"] is True + assert report["full_fixture_coverage"] is True + assert report["reference_gate_pass"] is False From 7989d38af0fc92b63b73e0594d2f0fca86acfa24 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 13 Oct 2026 09:07:36 -0400 Subject: [PATCH 2/4] Tighten evaluation reference claims --- evals/stories/cases.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/stories/cases.json b/evals/stories/cases.json index b37eb6b..9b076a0 100644 --- a/evals/stories/cases.json +++ b/evals/stories/cases.json @@ -202,7 +202,7 @@ {"field": "headline", "text": "River adds a Windows client", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, {"field": "stage", "text": "rollout", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, {"field": "fact", "text": "The signed Windows client supports Windows 11.", "citations": [{"source": 1, "quote": "The vendor documents Windows 11 as the supported operating system and provides signed installer checksums."}], "mutable": true}, - {"field": "why", "text": "Teams can move the same encrypted workspaces between Linux and Windows.", "citations": [{"source": 1, "quote": "The Windows build supports the same encrypted workspace format and can open workspaces created by the Linux release."}], "mutable": false} + {"field": "why", "text": "Windows users can open encrypted workspaces created on Linux.", "citations": [{"source": 1, "quote": "The Windows build supports the same encrypted workspace format and can open workspaces created by the Linux release."}], "mutable": false} ], "unresolved_questions": ["Will River support Windows 10?"], "correction_of": null @@ -292,9 +292,9 @@ "story_ids": ["url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb"], "claims": [ {"field": "headline", "text": "Pine corrects its team plan limit", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}], "mutable": true}, - {"field": "stage", "text": "demonstrated", "citations": [{"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, + {"field": "stage", "text": "announcement", "citations": [{"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, {"field": "fact", "text": "The team plan permits 60 active projects.", "citations": [{"source": 1, "quote": "Active projects beyond 60 require the enterprise plan."}], "mutable": true}, - {"field": "why", "text": "The active-project capacity changed; storage, users, and launch date did not.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The correction does not change the storage allowance, user limit, or launch date."}], "mutable": false} + {"field": "why", "text": "The reported active-project limit was corrected; storage, users, and launch date did not change.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The correction does not change the storage allowance, user limit, or launch date."}], "mutable": false} ], "unresolved_questions": [], "correction_of": 401 @@ -349,7 +349,7 @@ }, "reference_decision": { "action": "combine", - "reason": {"text": "The Lantern 1.0 specification and its first independently tested client are one development.", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}]}, + "reason": {"text": "The Lantern 1.0 specification and the first independent client to pass its public suite are one development.", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}]}, "change": {"text": "Lantern reached 1.0 with a public suite that Orbit's preview client passed.", "citations": [{"source": 1, "quote": "Lumen published version 1.0 of the Lantern synchronization protocol on October 8, 2026."}, {"source": 2, "quote": "The client can exchange the suite's fixture documents with Lumen's reference implementation."}]}, "story_ids": ["url:8a37ed47d48d359e1aa575095bb9bc9afb8c96c28950b7d19a398cf5123ffebd", "url:6782db64ca26f1128b8676730fbe44979b5d747a4041d72c6b187ed6a547a72a"], "claims": [ From 6c5c5a74cf7c5d1007ff47a02c0c3a94f36297ee Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 13 Oct 2026 09:27:28 -0400 Subject: [PATCH 3/4] Preserve evaluation failures and tighten calibration --- docs/story-evaluation.md | 16 ++-- evals/stories/cases.json | 10 +-- roger/story_eval.py | 94 +++++++++++++++++++--- tests/test_story_eval.py | 163 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 261 insertions(+), 22 deletions(-) diff --git a/docs/story-evaluation.md b/docs/story-evaluation.md index 70d941c..045632c 100644 --- a/docs/story-evaluation.md +++ b/docs/story-evaluation.md @@ -32,11 +32,12 @@ Sol or Sonnet reviews. Two such pairing ledgers reserve 96 calls and $62.40. No single ledger enforces the aggregate allowance across other artifact directories. -No paid comparison has been approved or run. Recheck the model catalog, -endpoint, provider and rates in `evals/stories/models.json` before proposing a -run. In particular, the Google endpoint's catalog status is recorded without -interpreting it. Do not substitute another provider or service tier under the -same approval. +Every paid comparison requires explicit owner approval for its finite call and +cost allowance, plus a new immutable ledger in its own artifact directory. +Recheck the model catalog, endpoint, provider and rates in +`evals/stories/models.json` before proposing a run. In particular, the Google +endpoint's catalog status is recorded without interpreting it. Do not +substitute another provider or service tier under the same approval. ## Inspect the plan @@ -84,6 +85,11 @@ bills it. A missing returned service tier is recorded as unknown and limits latency comparisons, while an explicitly different tier fails closed. Provider truncation, refusal, content filtering, pipeline interference, local parsing, reviewer rejection and human editorial quality remain separate outcomes. +The exact pass-only OpenAI moderation annotation with `flagged: false` is +recorded as structured routing metadata and may proceed. Any extra action, +redaction or unknown field, a flagged result, an unknown stage, or a modifying +stage such as response healing, context compression, a plugin or server tools +stops the run. Reviewer calibration has six supported references and six single-fault drafts. Any false accept, false reject, malformed review, incomplete run, or missing fixture repeat fails its reference gate. Reference labels are still pending diff --git a/evals/stories/cases.json b/evals/stories/cases.json index 9b076a0..ab36f2c 100644 --- a/evals/stories/cases.json +++ b/evals/stories/cases.json @@ -70,7 +70,7 @@ "claims": [ {"field": "headline", "text": "Atlas 2.0 ships for Linux", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}], "mutable": true}, {"field": "stage", "text": "rollout", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}], "mutable": true}, - {"field": "fact", "text": "Signed Linux packages cover x86-64 and arm64.", "citations": [{"source": 1, "quote": "They list packages for x86-64 and arm64 Linux systems and include checksums beside every download."}], "mutable": true}, + {"field": "fact", "text": "Signed Linux packages cover x86-64 and arm64.", "citations": [{"source": 1, "quote": "Atlas 2.0 became generally available on October 8, 2026, with signed Linux packages and an offline migration command."}, {"source": 1, "quote": "They list packages for x86-64 and arm64 Linux systems and include checksums beside every download."}], "mutable": true}, {"field": "why", "text": "Operators can keep Atlas 1.x installed while validating a converted copy.", "citations": [{"source": 1, "quote": "Operators can keep Atlas 1.x installed while validating a converted copy."}], "mutable": false} ], "unresolved_questions": ["When will macOS and Windows packages be ready?"], @@ -201,7 +201,7 @@ "claims": [ {"field": "headline", "text": "River adds a Windows client", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, {"field": "stage", "text": "rollout", "citations": [{"source": 1, "quote": "River's Windows client became generally available on October 8, 2026, after the Linux client shipped on October 1."}], "mutable": true}, - {"field": "fact", "text": "The signed Windows client supports Windows 11.", "citations": [{"source": 1, "quote": "The vendor documents Windows 11 as the supported operating system and provides signed installer checksums."}], "mutable": true}, + {"field": "fact", "text": "River documents Windows 11 support and provides signed installer checksums.", "citations": [{"source": 1, "quote": "The vendor documents Windows 11 as the supported operating system and provides signed installer checksums."}], "mutable": true}, {"field": "why", "text": "Windows users can open encrypted workspaces created on Linux.", "citations": [{"source": 1, "quote": "The Windows build supports the same encrypted workspace format and can open workspaces created by the Linux release."}], "mutable": false} ], "unresolved_questions": ["Will River support Windows 10?"], @@ -292,7 +292,7 @@ "story_ids": ["url:f26629c29a623279c70fba3ad8616b003b027b03f960cf386707a78c62a419fb"], "claims": [ {"field": "headline", "text": "Pine corrects its team plan limit", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}], "mutable": true}, - {"field": "stage", "text": "announcement", "citations": [{"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, + {"field": "stage", "text": "announcement", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, {"field": "fact", "text": "The team plan permits 60 active projects.", "citations": [{"source": 1, "quote": "Active projects beyond 60 require the enterprise plan."}], "mutable": true}, {"field": "why", "text": "The reported active-project limit was corrected; storage, users, and launch date did not change.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The correction does not change the storage allowance, user limit, or launch date."}], "mutable": false} ], @@ -350,7 +350,7 @@ "reference_decision": { "action": "combine", "reason": {"text": "The Lantern 1.0 specification and the first independent client to pass its public suite are one development.", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}]}, - "change": {"text": "Lantern reached 1.0 with a public suite that Orbit's preview client passed.", "citations": [{"source": 1, "quote": "Lumen published version 1.0 of the Lantern synchronization protocol on October 8, 2026."}, {"source": 2, "quote": "The client can exchange the suite's fixture documents with Lumen's reference implementation."}]}, + "change": {"text": "Lantern reached 1.0 with a public suite that Orbit's preview client passed.", "citations": [{"source": 1, "quote": "Lumen published version 1.0 of the Lantern synchronization protocol on October 8, 2026."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}, {"source": 2, "quote": "Orbit labels the build a developer preview and advises against production data."}]}, "story_ids": ["url:8a37ed47d48d359e1aa575095bb9bc9afb8c96c28950b7d19a398cf5123ffebd", "url:6782db64ca26f1128b8676730fbe44979b5d747a4041d72c6b187ed6a547a72a"], "claims": [ {"field": "headline", "text": "Lantern 1.0 gets an independent client", "citations": [{"source": 1, "quote": "Lumen's announcement names Orbit as the first independent client to pass that public suite."}, {"source": 2, "quote": "Orbit released an independent Lantern 1.0 client and links to its passing result for Lumen's public compatibility suite."}], "mutable": true}, @@ -451,7 +451,7 @@ }, "reference_decision": { "action": "hold", - "reason": {"text": "The fresh item repeats confirmed coverage, while the mobile speculation is stale and unsupported.", "citations": [{"source": 1, "quote": "The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction."}, {"source": 2, "quote": "No build, preview, test result, issue milestone, or availability change is linked."}]}, + "reason": {"text": "The fresh item repeats confirmed coverage, while the mobile speculation is stale and unsupported.", "citations": [{"source": 1, "quote": "The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction."}, {"source": 2, "quote": "An undated forum summary speculates that Atlas might add mobile packages someday, but it cites no maintainer statement, repository change, release artifact, or schedule."}]}, "change": {"text": "", "citations": []}, "story_ids": ["url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863"], "claims": [], diff --git a/roger/story_eval.py b/roger/story_eval.py index 05e8864..1d47d85 100644 --- a/roger/story_eval.py +++ b/roger/story_eval.py @@ -462,6 +462,48 @@ def _request_record(spec: CallSpec) -> dict[str, Any]: } +def _is_passed_moderation(stage: Any) -> bool: + if not isinstance(stage, dict) or set(stage) != {"type", "name", "summary", "data"}: + return False + data = stage["data"] + return ( + stage["type"] == "guardrail" + and stage["name"] == "moderation" + and stage["summary"] == "Content passed OpenAI moderation" + and isinstance(data, dict) + and set(data) == {"engine", "flagged", "moderator_ids"} + and data["engine"] == "openai-moderations" + and data["flagged"] is False + and data["moderator_ids"] == ["OpenAIOmniModerator"] + ) + + +def _pipeline_annotation(stage: Any) -> dict[str, Any]: + if not isinstance(stage, dict): + return {"type": str(stage), "recognized_pass_only": False} + annotation = { + key: stage[key] + for key in ("type", "name", "summary") + if isinstance(stage.get(key), str) + } + data = stage.get("data") + if isinstance(data, dict): + clean_data = {} + if isinstance(data.get("engine"), str): + clean_data["engine"] = data["engine"] + if isinstance(data.get("flagged"), bool): + clean_data["flagged"] = data["flagged"] + moderator_ids = data.get("moderator_ids") + if isinstance(moderator_ids, list) and all( + isinstance(value, str) for value in moderator_ids + ): + clean_data["moderator_ids"] = moderator_ids + if clean_data: + annotation["data"] = clean_data + annotation["recognized_pass_only"] = _is_passed_moderation(stage) + return annotation + + def _route_metadata(response: Any) -> dict[str, Any] | None: raw = getattr(response, "openrouter_metadata", None) if raw is None: @@ -480,11 +522,14 @@ def _route_metadata(response: Any) -> dict[str, Any] | None: }) pipeline = raw.get("pipeline") if isinstance(pipeline, list): - pipeline_stages = [str(stage) for stage in pipeline] - elif isinstance(pipeline, dict): - pipeline_stages = [str(key) for key, value in pipeline.items() if value] - else: + pipeline_stages = [_pipeline_annotation(stage) for stage in pipeline] + pipeline_pass_only = all(_is_passed_moderation(stage) for stage in pipeline) + elif pipeline is None: pipeline_stages = [] + pipeline_pass_only = True + else: + pipeline_stages = [{"type": "invalid", "recognized_pass_only": False}] + pipeline_pass_only = False attempts = raw.get("attempts") return { "requested": raw.get("requested"), @@ -493,13 +538,29 @@ def _route_metadata(response: Any) -> dict[str, Any] | None: "attempt_count": len(attempts) if isinstance(attempts, list) else None, "available": clean_endpoints, "pipeline_stages": pipeline_stages, + "pipeline_pass_only": pipeline_pass_only, } def _visible_response(response: Any) -> dict[str, Any]: - choice = response.choices[0] - message = choice.message - return { + choices = getattr(response, "choices", None) + shape_error = None + if choices is None: + choice = None + shape_error = "null_choices" + elif not isinstance(choices, list) or not choices: + choice = None + shape_error = "empty_choices" if choices == [] else "invalid_choices" + else: + choice = choices[0] + if choice is None: + shape_error = "null_choice" + elif len(choices) != 1: + shape_error = "multiple_choices" + message = getattr(choice, "message", None) if choice is not None else None + if choice is not None and message is None: + shape_error = "missing_message" + visible = { "model": getattr(response, "model", None), "finish_reason": getattr(choice, "finish_reason", None), "content": getattr(message, "content", None), @@ -507,6 +568,9 @@ def _visible_response(response: Any) -> dict[str, Any]: "service_tier": getattr(response, "service_tier", None), "route_metadata": _route_metadata(response), } + if shape_error is not None: + visible["shape_error"] = shape_error + return visible def _usage(response: Any) -> dict[str, int | Decimal | None] | None: @@ -541,6 +605,8 @@ def _usage(response: Any) -> dict[str, int | Decimal | None] | None: def _response_status(visible: dict[str, Any], profile: ModelProfile) -> str: + if visible.get("shape_error") is not None: + return "malformed_response" if visible["model"] not in profile.returned_models: return "model_mismatch" if visible["service_tier"] is not None and visible["service_tier"] != profile.service_tier: @@ -558,7 +624,7 @@ def _response_status(visible: dict[str, Any], profile: ModelProfile) -> str: return "missing_route_metadata" if metadata["requested"] != profile.model or metadata["attempt"] != 1: return "route_metadata_mismatch" - if metadata["pipeline_stages"]: + if not metadata["pipeline_pass_only"]: return "pipeline_interference" if metadata["attempt_count"] not in (None, 1): return "routing_multiple_attempts" @@ -606,9 +672,9 @@ async def call(self, spec: CallSpec) -> tuple[Any, dict[str, Any]]: visible = _visible_response(response) usage = _usage(response) status = _response_status(visible, spec.profile) - if usage is None: + if usage is None and status != "malformed_response": status = "missing_usage_or_cost" - elif ( + elif usage is not None and ( usage["input_tokens"] > _INPUT_TOKEN_RESERVE or usage["output_tokens"] > spec.max_tokens or ( @@ -722,12 +788,16 @@ def _validate_decision_output( def _source_inventory(case: dict[str, Any]) -> list[dict[str, Any]]: prepared = _case_sources(case) - by_url = {source["url"]: source for source in prepared} + by_identity = {} + for source in prepared: + identity = (source.get("feed_url"), source.get("entry_id"), source.get("url")) + by_identity.setdefault(identity, source) inventory = [] for entry in case["entries"]: article = entry.get("article") or {} text = str(article.get("text") or "") - source = by_url.get(article.get("url")) + identity = (entry.get("feed_url"), entry.get("id"), article.get("url")) + source = by_identity.get(identity) delivered_chars = len(source["excerpt"]) if source else 0 inventory.append({ "url": article.get("url"), diff --git a/tests/test_story_eval.py b/tests/test_story_eval.py index dcaba95..7770a3d 100644 --- a/tests/test_story_eval.py +++ b/tests/test_story_eval.py @@ -5,6 +5,7 @@ import httpx import pytest from openai import AsyncOpenAI +from openai.types.chat import ChatCompletion from roger import story_eval from roger.brains import stories @@ -64,6 +65,19 @@ def _response(content): ) +def _passed_moderation(): + return { + "type": "guardrail", + "name": "moderation", + "summary": "Content passed OpenAI moderation", + "data": { + "engine": "openai-moderations", + "flagged": False, + "moderator_ids": ["OpenAIOmniModerator"], + }, + } + + def _review(accepted=True, item_count=6): return json.dumps({ "supported": [accepted, *([True] * (item_count - 1))], @@ -136,6 +150,24 @@ def test_fixture_history_targets_are_confirmed_and_first_four(): assert delivered["message_url"].startswith("https://discord.example.invalid/") +def test_source_inventory_matches_current_versions_and_reports_real_truncation(): + _, cases, _ = _fixture_data() + for case_id in ("material_update", "correction", "duplicate_stale_hold"): + inventory = story_eval._source_inventory(cases[case_id]) + assert all(row["omitted_chars"] == 0 for row in inventory) + assert all( + row["delivered_excerpt_chars"] == row["original_chars"] + for row in inventory + ) + + long_case = json.loads(json.dumps(cases["straight_release"])) + long_case["entries"][0]["article"]["text"] = "x" * 1700 + inventory = story_eval._source_inventory(long_case) + assert inventory[0]["original_chars"] == 1700 + assert inventory[0]["delivered_excerpt_chars"] == 1600 + assert inventory[0]["omitted_chars"] == 100 + + def test_offline_plan_does_not_construct_client(monkeypatch): def forbidden(*args, **kwargs): raise AssertionError("plan constructed a client") @@ -236,6 +268,7 @@ def test_run_directory_is_exclusive(tmp_path): ("response", "expected"), [ (_completion("{}", include_usage=False), "missing_usage_or_cost"), + (_completion("{}", cost=Decimal("-0.01")), "missing_usage_or_cost"), (_completion("{}", cost=Decimal("2")), "overspend"), (_completion("{}", pipeline=["response_healing"]), "pipeline_interference"), ], @@ -262,6 +295,64 @@ async def handle(request): assert ledger.data["calls"][0]["status"] == expected +@pytest.mark.asyncio +async def test_passed_moderation_pipeline_is_recorded_and_allowed(tmp_path): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec( + cases["straight_release"], story_eval.load_profiles()["sol"], 1 + ) + response = ChatCompletion.model_validate( + _completion("{}", pipeline=[_passed_moderation()]) + ) + + async def create(**kwargs): + return response + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + ledger = story_eval.Ledger( + tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1") + ) + responses = tmp_path / "responses" + responses.mkdir() + + await story_eval.Runner(client, ledger, responses).call(spec) + + artifact = json.loads(next(responses.iterdir()).read_text()) + metadata = artifact["response"]["route_metadata"] + assert artifact["status"] == "completed" + assert metadata["pipeline_pass_only"] is True + assert metadata["pipeline_stages"] == [ + {**_passed_moderation(), "recognized_pass_only": True} + ] + + +@pytest.mark.parametrize( + "pipeline", + [ + [{**_passed_moderation(), "unknown": "field"}], + [{**_passed_moderation(), "data": {**_passed_moderation()["data"], "flagged": True}}], + [{**_passed_moderation(), "data": {**_passed_moderation()["data"], "flagged": 0}}], + [{**_passed_moderation(), "data": {**_passed_moderation()["data"], "action": "allow"}}], + [{**_passed_moderation(), "data": {**_passed_moderation()["data"], "redact": False}}], + [{"type": "response_healing", "name": "response-healing", "data": {}}], + [{"type": "context_compression", "name": "context-compression", "data": {}}], + [{"type": "plugin", "name": "web-search", "data": {}}], + [{"type": "server_tools", "name": "server-tools", "data": {}}], + [{"type": "future_stage", "name": "unknown", "data": {}}], + ], +) +def test_pipeline_rejects_modified_or_unknown_stages(pipeline): + response = ChatCompletion.model_validate(_completion("{}", pipeline=pipeline)) + visible = story_eval._visible_response(response) + + assert visible["route_metadata"]["pipeline_pass_only"] is False + assert story_eval._response_status( + visible, story_eval.load_profiles()["sol"] + ) == "pipeline_interference" + + @pytest.mark.asyncio async def test_missing_optional_reasoning_usage_stays_unknown(tmp_path): _, cases, _ = _fixture_data() @@ -290,6 +381,78 @@ async def handle(request): assert ledger.data["calls"][0]["reasoning_tokens"] is None +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("choices", "shape_error", "include_usage"), + [ + (None, "null_choices", True), + ([], "empty_choices", False), + ([SimpleNamespace(finish_reason="stop")], "missing_message", True), + ], +) +async def test_malformed_sdk_response_persists_safe_artifacts_and_accounting( + tmp_path, choices, shape_error, include_usage +): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec( + cases["straight_release"], story_eval.load_profiles()["sol"], 1 + ) + usage = None + if include_usage: + usage = SimpleNamespace( + prompt_tokens=100, + completion_tokens=25, + cost=0.01, + completion_tokens_details=SimpleNamespace(reasoning_tokens=11), + ) + response = ChatCompletion.model_construct( + id="eval-response", + object="chat.completion", + created=1, + model="openai/gpt-6.1-sol", + service_tier="default", + choices=choices, + usage=usage, + openrouter_metadata=_completion("{}")["openrouter_metadata"], + ) + calls = 0 + + async def create(**kwargs): + nonlocal calls + calls += 1 + return response + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + ledger = story_eval.Ledger( + tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1") + ) + responses = tmp_path / "responses" + responses.mkdir() + + with pytest.raises(story_eval.EvaluationError, match="call stopped: malformed_response"): + await story_eval.Runner(client, ledger, responses).call(spec) + + assert calls == 1 + durable = json.loads((tmp_path / "ledger.json").read_text()) + row = durable["calls"][0] + assert row["status"] == "malformed_response" + assert durable["reserved_calls"] == 1 + assert durable["used_calls"] == (1 if include_usage else 0) + assert durable["reserved_cost_usd"] == "0.23" + assert durable["used_cost_usd"] == ("0.01" if include_usage else "0") + + artifact = json.loads(next(responses.iterdir()).read_text()) + assert artifact["status"] == "malformed_response" + assert artifact["response"]["shape_error"] == shape_error + assert artifact["response"]["model"] == "openai/gpt-6.1-sol" + assert artifact["response"]["route_metadata"]["attempt"] == 1 + assert (artifact["accounting"] is not None) is include_usage + assert "choices" not in artifact["response"] + assert "reasoning" not in artifact["response"] + + @pytest.mark.asyncio async def test_request_error_has_one_attempt_and_sanitized_durable_error(tmp_path): _, cases, _ = _fixture_data() From aaa9212e33adc2a4eefbda0b9a607e0414409deb Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 13 Oct 2026 09:38:58 -0400 Subject: [PATCH 4/4] Make reference citations establish exact claims --- evals/stories/cases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/stories/cases.json b/evals/stories/cases.json index ab36f2c..d301717 100644 --- a/evals/stories/cases.json +++ b/evals/stories/cases.json @@ -293,7 +293,7 @@ "claims": [ {"field": "headline", "text": "Pine corrects its team plan limit", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}], "mutable": true}, {"field": "stage", "text": "announcement", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The company says the earlier number counted archived projects and has updated the pricing table."}], "mutable": true}, - {"field": "fact", "text": "The team plan permits 60 active projects.", "citations": [{"source": 1, "quote": "Active projects beyond 60 require the enterprise plan."}], "mutable": true}, + {"field": "fact", "text": "The team plan permits 60 active projects.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}], "mutable": true}, {"field": "why", "text": "The reported active-project limit was corrected; storage, users, and launch date did not change.", "citations": [{"source": 1, "quote": "Correction: Pine's team plan supports 60 active projects, not the 100 projects stated in the October 7 launch post."}, {"source": 1, "quote": "The correction does not change the storage allowance, user limit, or launch date."}], "mutable": false} ], "unresolved_questions": [], @@ -451,7 +451,7 @@ }, "reference_decision": { "action": "hold", - "reason": {"text": "The fresh item repeats confirmed coverage, while the mobile speculation is stale and unsupported.", "citations": [{"source": 1, "quote": "The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction."}, {"source": 2, "quote": "An undated forum summary speculates that Atlas might add mobile packages someday, but it cites no maintainer statement, repository change, release artifact, or schedule."}]}, + "reason": {"text": "The fresh item repeats confirmed coverage, while the mobile speculation is stale and unsupported.", "citations": [{"source": 1, "quote": "The author is a community member and does not cite a new maintainer release, availability change, evaluation, or correction."}, {"source": 1, "quote": "The guide uses the package checksums and commands from the October 7 maintainer documentation. It presents no measured result, new compatibility claim, or later product event."}, {"source": 2, "quote": "An undated forum summary speculates that Atlas might add mobile packages someday, but it cites no maintainer statement, repository change, release artifact, or schedule."}]}, "change": {"text": "", "citations": []}, "story_ids": ["url:a00ed055ce4af9fc637a4760311c56eaf1106cd8c85a480586dbbbfe15b9f863"], "claims": [],