From 94697000af739cfe5ae2e4852ff7ad9d2bdd7c56 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 14 Oct 2026 22:36:41 -0700 Subject: [PATCH 01/10] Bill GitHub Copilot CLI sessions from relayhistory evidence relayhistory's `copilot` source (ai-hist 0.45) maps to `copilot-cli` turns: one per model call with its tool calls, Copilot's tool vocabulary for targets, edit hashes and touched files, sidechain subagent turns, and the per-message `outputTokens`. Each `session.shutdown` adds one per-session-aggregate turn per model billing what Copilot's running `modelMetrics` totals grew by (cache split out of `inputTokens`), less the output the messages already billed, so a session's turns sum to its final totals. Turns price under provider `github-copilot` with reasoning inside output; no launcher stamp resolves onto a Copilot session. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- README.md | 19 +- crates/relayburn-sdk/Cargo.toml | 2 +- crates/relayburn-sdk/src/analyze/cost.rs | 5 +- crates/relayburn-sdk/src/analyze/provider.rs | 1 + crates/relayburn-sdk/src/ingest/import.rs | 4 +- .../relayburn-sdk/src/ingest/ingest_tests.rs | 59 ++ crates/relayburn-sdk/src/reader/classifier.rs | 5 + crates/relayburn-sdk/src/reader/types.rs | 4 + .../relayburn-sdk/src/source/parity_tests.rs | 10 + .../relayburn-sdk/src/source/relayhistory.rs | 4 + .../src/source/relayhistory/copilot.rs | 362 +++++++++ crates/relayburn-sdk/src/source/usage.rs | 66 ++ docs/migrating-to-5.md | 5 +- tests/fixtures/copilot/cli-usage.jsonl | 37 + .../fixtures/copilot/official-importer.jsonl | 6 + tests/fixtures/sourcing-snapshots/README.md | 8 +- .../sourcing-snapshots/copilot-cli-usage.json | 727 ++++++++++++++++++ .../copilot-official-importer.json | 176 +++++ 19 files changed, 1492 insertions(+), 13 deletions(-) create mode 100644 crates/relayburn-sdk/src/source/relayhistory/copilot.rs create mode 100644 tests/fixtures/copilot/cli-usage.jsonl create mode 100644 tests/fixtures/copilot/official-importer.jsonl create mode 100644 tests/fixtures/sourcing-snapshots/copilot-cli-usage.json create mode 100644 tests/fixtures/sourcing-snapshots/copilot-official-importer.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 919a6429..0b33aad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,19 @@ Cross-package release notes for relayburn. Package changelogs contain package-le ## [Unreleased] -5.0.0: burn reads every session through relayhistory (`ai-hist` 0.41) and no longer parses harness logs. See `docs/migrating-to-5.md`. +5.0.0: burn reads every session through relayhistory (`ai-hist` 0.45) and no longer parses harness logs. See `docs/migrating-to-5.md`. ### Breaking - `burn ingest` syncs the relayhistory store (`$AI_HIST_DB`, else `~/.local/share/ai-hist/ai-history.db`, created on first use; the `ai-hist` CLI is not required) and appends the sessions its change feed reports since the ledger's watermark. The first run on a 4.x ledger rebuilds every session once (minutes on a large history); matching turn identities keep it from duplicating any. Later runs with nothing new finish in well under a second. -- `BURN_CLAUDE_PROJECTS_DIR` is removed; harness locations follow relayhistory (`HOME`, `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `OPENCODE_DB`). +- `BURN_CLAUDE_PROJECTS_DIR` is removed; harness locations follow relayhistory (`HOME`, `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `OPENCODE_DB`, `COPILOT_HOME`). - `relayburn-sdk`: `IngestRoots` / `IngestOptions.roots` are replaced by `IngestOptions.store: HistoryStoreOptions`; `start_watch_loop` and its option/sink types by `watch_ingest` + `WatchIngestOptions`; `RawIngestOptions` is `IngestOptions`; `ingest_claude_session`, `ingest_codex_sessions`, `ingest_opencode_sessions`, `default_session_roots`, `count_subagents_under`, `discover_subagents` and `pair_subagents_to_main` are removed. `SubagentTranscript` drops `records`. - `@relayburn/sdk` `ingest()` takes `{ ledgerHome, storeDbPath, home }`; the ignored `sessionId` / `harness` fields are removed. - `burn state rebuild` clears the ingest watermark with the derived rows, so the next ingest rebuilds every session from the store. ### Added +- GitHub Copilot CLI sessions (`~/.copilot/session-state/`, or `$COPILOT_HOME`) are ingested as `copilot-cli` turns, retroactively and with no opt-in: one turn per model call with its tool calls, activity and per-message output tokens, plus one `per-session-aggregate` turn per `session.shutdown` and model billing the input, cache-read, cache-write and remaining output tokens Copilot's running totals grew by. Subagent turns are flagged sidechain; turns price under provider `github-copilot`, with reasoning counted inside output. - Claude Code subagent spend is billed: every turn of a subagent transcript, nested subagents included, is a turn of the session that spawned it. Ledger totals for sessions that delegated work go up by exactly that spend. - OpenCode sessions in the SQLite store (`opencode.db`, current OpenCode releases) are ingested, as well as the `storage/` JSON tree. - A Codex subagent thread is ingested as a session of its own. diff --git a/README.md b/README.md index fc211b3e..8056d67b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![relayburn](./burn-readme-banner.png) Understand how you're spending tokens in agent CLIs. Burn reads Claude Code, -Codex, and OpenCode sessions through +Codex, OpenCode, and GitHub Copilot CLI sessions through [relayhistory](https://github.com/AgentWorkforce/relayhistory) into a local ledger, then shows cost by model, provider, tool, file, workflow, agent, session, and overhead file. Subagent spend is billed to the session that @@ -336,9 +336,18 @@ Run `burn summary --by-provider` to discover model IDs present in your ledger. ## `burn ingest` -`burn ingest` syncs the relayhistory store (Claude Code, Codex, and OpenCode -sessions) and appends every session it changed since the ledger's last ingest. -Reporting commands that refresh the ledger run the same step. +`burn ingest` syncs the relayhistory store (Claude Code, Codex, OpenCode, and +GitHub Copilot CLI sessions) and appends every session it changed since the +ledger's last ingest. Reporting commands that refresh the ledger run the same +step. + +GitHub Copilot CLI needs no setup: its session journals under +`~/.copilot/session-state/` (or `$COPILOT_HOME`) are read retroactively. Copilot +records output tokens per model call but input and cache tokens only as +per-model running totals when a session shuts down, so each `copilot-cli` turn +carries its output tokens and each shutdown adds one `per-session-aggregate` +turn per model with the input, cache, and remaining output it accounts for. A +session that has not shut down yet shows output tokens only. | Option | What it does | |---|---| @@ -660,7 +669,7 @@ session. ### What does `burn ingest` do? -Each harness (Claude Code, Codex, OpenCode) writes its own session transcripts +Each harness (Claude Code, Codex, OpenCode, GitHub Copilot CLI) writes its own session transcripts to disk in its own format. relayhistory reads those into one session store; `burn ingest` syncs that store, derives burn's turns, costs, and activity from what changed, and writes them into burn's local SQLite ledger so the query diff --git a/crates/relayburn-sdk/Cargo.toml b/crates/relayburn-sdk/Cargo.toml index cb66ad6f..fe1fdf20 100644 --- a/crates/relayburn-sdk/Cargo.toml +++ b/crates/relayburn-sdk/Cargo.toml @@ -26,7 +26,7 @@ rusqlite = { workspace = true } # Session sourcing: relayhistory captures harness evidence; burn owns what # the counters mean. burn depends on a semver-compatible ai-hist release; # adopting a new minor is a deliberate bump that re-runs the parity suite. -ai-hist = "0.41" +ai-hist = "0.45" # Path-mode session analysis stages one transcript into a throwaway # provider root and relayhistory store. tempfile = "3" diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index 699c7561..6b604843 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -184,11 +184,12 @@ fn reasoning_cost( /// Per-source reasoning-billing semantics override. Returning `None` means /// "defer to the model's `reasoning_mode`". /// -/// - Codex: `output_tokens` already includes reasoning; never bill it on top. +/// - Codex and Copilot CLI: `output_tokens` / `outputTokens` already include +/// reasoning; never bill it on top. /// - Everyone else: defer to the model. pub(crate) fn reasoning_mode_for_source(source: SourceKind) -> Option { match source { - SourceKind::Codex => Some(ReasoningMode::IncludedInOutput), + SourceKind::Codex | SourceKind::CopilotCli => Some(ReasoningMode::IncludedInOutput), _ => None, } } diff --git a/crates/relayburn-sdk/src/analyze/provider.rs b/crates/relayburn-sdk/src/analyze/provider.rs index 57d37b12..7dcca930 100644 --- a/crates/relayburn-sdk/src/analyze/provider.rs +++ b/crates/relayburn-sdk/src/analyze/provider.rs @@ -339,6 +339,7 @@ fn provider_from_source(source: SourceKind) -> String { SourceKind::Codex | SourceKind::OpenaiApi => "openai".into(), SourceKind::GeminiApi => "google".into(), SourceKind::Opencode => "opencode".into(), + SourceKind::CopilotCli => "github-copilot".into(), } } diff --git a/crates/relayburn-sdk/src/ingest/import.rs b/crates/relayburn-sdk/src/ingest/import.rs index 5d6aa726..3556a547 100644 --- a/crates/relayburn-sdk/src/ingest/import.rs +++ b/crates/relayburn-sdk/src/ingest/import.rs @@ -198,7 +198,9 @@ fn resolve_stamps( let harness = match evidence.session.source { Source::Claude => PendingStampHarness::Claude, Source::Codex => PendingStampHarness::Codex, - _ => PendingStampHarness::Opencode, + Source::OpenCode => PendingStampHarness::Opencode, + // No launcher stamps a session it cannot name before spawn. + _ => return 0, }; let candidate = PendingStampSessionCandidate { harness, diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests.rs b/crates/relayburn-sdk/src/ingest/ingest_tests.rs index 2fc9f0e3..4ac6858c 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests.rs @@ -803,3 +803,62 @@ fn ingest_reports_each_phase_to_its_progress_sink() { assert!(seen.iter().any(|p| p == phase), "{phase}: {seen:?}"); } } + +#[test] +fn a_copilot_session_bills_its_calls_and_its_shutdown_totals_once() { + let sandbox = Sandbox::new(); + let sid = "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81"; + sandbox.place( + "copilot/cli-usage.jsonl", + &format!(".copilot/session-state/{sid}/events.jsonl"), + ); + // No launcher stamps a Copilot session, so another harness's stamp for + // the same project stays pending. + write_pending_stamp(WriteOptions { + harness: PendingStampHarness::Opencode, + ledger_home: Some(sandbox.ledger_home()), + cwd: "/tmp/project".into(), + enrichment: Enrichment::new(), + ..Default::default() + }) + .unwrap(); + let mut ledger = sandbox.ledger(); + let report = sandbox.ingest(&mut ledger); + assert_eq!(report.ingested_sessions, 1); + assert_eq!(report.applied_pending_stamps, 0); + let turns = ledger.query_turns(&Query::for_session(sid)).unwrap(); + assert_eq!(turns.len(), 11); + assert!(turns + .iter() + .all(|t| t.turn.source == crate::reader::SourceKind::CopilotCli)); + let sum = + |f: fn(&crate::reader::Usage) -> u64| turns.iter().map(|t| f(&t.turn.usage)).sum::(); + // The final session.shutdown modelMetrics, summed over models. + assert_eq!( + sum(|u| u.input + u.cache_read + u.cache_create_5m), + 30_000 + 25_000 + 4_000 + ); + assert_eq!(sum(|u| u.output), 82 + 65 + 11); + assert_eq!(sandbox.ingest(&mut ledger).appended_turns, 0); +} + +#[test] +fn an_opencode_launcher_stamp_lands_on_the_opencode_session_it_spawned() { + let sandbox = Sandbox::new(); + let mut enrichment = Enrichment::new(); + enrichment.insert("persona".to_string(), "code-reviewer".to_string()); + write_pending_stamp(WriteOptions { + harness: PendingStampHarness::Opencode, + ledger_home: Some(sandbox.ledger_home()), + cwd: "/tmp/project".into(), + enrichment, + ..Default::default() + }) + .unwrap(); + let storage = sandbox.place("opencode/simple/storage", ".local/share/opencode/storage"); + // Written after the spawn, as a session the launcher started is. + let session = storage.join("session/global/ses_simple.json"); + fs::write(&session, fs::read(&session).unwrap()).unwrap(); + let mut ledger = sandbox.ledger(); + assert_eq!(sandbox.ingest(&mut ledger).applied_pending_stamps, 1); +} diff --git a/crates/relayburn-sdk/src/reader/classifier.rs b/crates/relayburn-sdk/src/reader/classifier.rs index d8a8c39a..d884ed0a 100644 --- a/crates/relayburn-sdk/src/reader/classifier.rs +++ b/crates/relayburn-sdk/src/reader/classifier.rs @@ -70,6 +70,11 @@ static TOOL_ALIASES: phf::Map<&'static str, &'static str> = phf_map! { "glob" => "Glob", "webfetch" => "WebFetch", "task" => "Task", + // Copilot CLI (shares OpenCode's bash/edit/grep/glob/task names) + "view" => "Read", + "create" => "Write", + "web_fetch" => "WebFetch", + "read_agent" => "Task", }; pub fn normalize_tool_name(name: &str) -> &str { diff --git a/crates/relayburn-sdk/src/reader/types.rs b/crates/relayburn-sdk/src/reader/types.rs index 693a1354..4e2024d8 100644 --- a/crates/relayburn-sdk/src/reader/types.rs +++ b/crates/relayburn-sdk/src/reader/types.rs @@ -24,6 +24,7 @@ pub enum SourceKind { ClaudeCode, Codex, Opencode, + CopilotCli, AnthropicApi, OpenaiApi, GeminiApi, @@ -36,6 +37,7 @@ impl SourceKind { Self::ClaudeCode => "claude-code", Self::Codex => "codex", Self::Opencode => "opencode", + Self::CopilotCli => "copilot-cli", Self::AnthropicApi => "anthropic-api", Self::OpenaiApi => "openai-api", Self::GeminiApi => "gemini-api", @@ -55,6 +57,7 @@ pub enum RelationshipSourceKind { ClaudeCode, Codex, Opencode, + CopilotCli, AnthropicApi, OpenaiApi, GeminiApi, @@ -70,6 +73,7 @@ impl RelationshipSourceKind { Self::ClaudeCode => "claude-code", Self::Codex => "codex", Self::Opencode => "opencode", + Self::CopilotCli => "copilot-cli", Self::AnthropicApi => "anthropic-api", Self::OpenaiApi => "openai-api", Self::GeminiApi => "gemini-api", diff --git a/crates/relayburn-sdk/src/source/parity_tests.rs b/crates/relayburn-sdk/src/source/parity_tests.rs index 10cffef8..94e19897 100644 --- a/crates/relayburn-sdk/src/source/parity_tests.rs +++ b/crates/relayburn-sdk/src/source/parity_tests.rs @@ -121,6 +121,16 @@ fn stage_corpus() -> Vec { ); staged.push(fixture); } + for path in sorted(&root.join("copilot")) { + let fixture = Staged::new(format!("copilot-{}", name_of(&path))); + let journal = std::fs::read_to_string(&path).unwrap(); + let header: Value = serde_json::from_str(journal.lines().next().unwrap()).unwrap(); + let sid = header["data"]["sessionId"].as_str().unwrap(); + let dir = fixture.home.path().join(".copilot/session-state").join(sid); + std::fs::create_dir_all(&dir).unwrap(); + fixture.copy(&path, &dir.join("events.jsonl")); + staged.push(fixture); + } staged } diff --git a/crates/relayburn-sdk/src/source/relayhistory.rs b/crates/relayburn-sdk/src/source/relayhistory.rs index ce93c004..444a9e3f 100644 --- a/crates/relayburn-sdk/src/source/relayhistory.rs +++ b/crates/relayburn-sdk/src/source/relayhistory.rs @@ -25,6 +25,7 @@ use crate::util::time::format_iso_ms; mod claude; mod codex; mod content; +mod copilot; mod opencode; mod tools; @@ -33,6 +34,7 @@ pub(crate) fn source_kind(source: Source) -> Option { Source::Claude => Some(SourceKind::ClaudeCode), Source::Codex => Some(SourceKind::Codex), Source::OpenCode => Some(SourceKind::Opencode), + Source::Copilot => Some(SourceKind::CopilotCli), _ => None, } } @@ -76,6 +78,7 @@ pub(crate) fn records_from_evidence(ev: &SessionEvidence, billing: &Billing) -> records.request_id_lookup = ctx.request_ids(&records.turns); match source { SourceKind::Opencode => opencode::refine(&ctx, &mut records), + SourceKind::CopilotCli => copilot::refine(&ctx, &mut records), _ => claude::refine(&ctx, &mut records), } records @@ -84,6 +87,7 @@ pub(crate) fn records_from_evidence(ev: &SessionEvidence, billing: &Billing) -> fn relationship_source(source: SourceKind) -> RelationshipSourceKind { match source { SourceKind::Opencode => RelationshipSourceKind::Opencode, + SourceKind::CopilotCli => RelationshipSourceKind::CopilotCli, _ => RelationshipSourceKind::ClaudeCode, } } diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs new file mode 100644 index 00000000..b533c2cd --- /dev/null +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -0,0 +1,362 @@ +//! copilot-specific refinements of the shared evidence mapping. +//! +//! Copilot CLI writes one `assistant.message` per model call, so a burn turn +//! is one assistant message. The only per-call counter it persists is +//! `outputTokens`; input and cache usage exist only as the per-model running +//! totals each `session.shutdown` records. Those totals are billed here as +//! one per-session-aggregate turn per shutdown and model: what the totals +//! grew by since the previous shutdown, less the output the session's +//! messages already billed. + +use std::collections::{BTreeSet, HashMap}; + +use ai_hist::{Marker, Message}; +use serde_json::Value; + +use super::Context; +use crate::reader::hash::content_hash; +use crate::reader::resolve_project; +use crate::reader::types::{ + Coverage, Fidelity, Subagent, ToolCall, TurnRecord, Usage, UsageGranularity, +}; +use crate::source::usage::{copilot_totals, usage_from_raw}; +use crate::source::SessionRecords; +use crate::util::time::format_iso_ms; + +/// Derive what the shared mapping cannot for copilot sessions. +pub(super) fn refine(ctx: &Context<'_>, records: &mut SessionRecords) { + for turn in &mut records.turns { + refine_turn(ctx, turn); + } + records.turns.extend(shutdown_turns(ctx)); + // A shutdown's turn sits where the journal wrote it, so a session that + // resumes appends turns after it and earlier indices never move. + records.turns.sort_by(|a, b| a.ts.cmp(&b.ts)); + for (index, turn) in records.turns.iter_mut().enumerate() { + turn.turn_index = index as u64; + } +} + +/// Copilot's tool vocabulary for targets, edit hashes and touched files; +/// subagent turns flagged; then the activity classification over them. +fn refine_turn(ctx: &Context<'_>, turn: &mut TurnRecord) { + let mut files = BTreeSet::new(); + for call in &mut turn.tool_calls { + let input = ctx + .ev + .tool_calls + .iter() + .find(|c| c.tool_use_id == call.id) + .and_then(|c| c.args.clone()) + .unwrap_or(Value::Null); + apply_tool(call, &input); + if matches!(call.name.as_str(), "view" | "edit" | "create") { + if let Some(target) = &call.target { + files.insert(target.clone()); + } + } + } + turn.files_touched = (!files.is_empty()).then(|| files.into_iter().collect()); + let messages: Vec<&Message> = ctx + .units + .iter() + .filter(|u| u.id == turn.message_id) + .flat_map(|u| &u.request.message_ids) + .filter_map(|id| ctx.by_id.get(id.as_str()).copied()) + .collect(); + if messages.iter().any(|m| m.is_sidechain == Some(true)) { + turn.subagent = Some(Subagent { + is_sidechain: true, + parent_tool_use_id: None, + agent_id: None, + parent_agent_id: None, + subagent_type: None, + description: None, + }); + } + ctx.classify(turn, &messages); +} + +fn apply_tool(call: &mut ToolCall, input: &Value) { + let s = |k: &str| input.get(k).and_then(Value::as_str).map(str::to_string); + // `view`/`edit`/`create` name a `path`, `web_fetch` a `url`, `bash` a + // `command`; the rest name what they act on differently. + call.target = match call.name.as_str() { + "grep" | "glob" => s("pattern"), + "task" => s("agent_type").or_else(|| s("description")), + _ => s("path").or_else(|| s("url")).or_else(|| s("command")), + }; + let hash = |k: &str| input.get(k).and_then(Value::as_str).map(content_hash); + match call.name.as_str() { + "edit" => { + call.edit_pre_hash = hash("old_str"); + call.edit_post_hash = hash("new_str"); + } + "create" => call.edit_post_hash = hash("file_text"), + _ => {} + } +} + +/// One `session.shutdown`: when it was written and each model's totals. +struct Shutdown<'a> { + marker: &'a Marker, + ts_ms: i64, + totals: Vec<(String, Usage)>, +} + +fn shutdowns<'a>(ctx: &Context<'a>) -> Vec> { + let mut out: Vec = ctx + .ev + .markers + .iter() + .filter(|m| m.kind == "session.shutdown") + .filter_map(|marker| { + let metrics = marker.payload.as_ref()?.pointer("/data/modelMetrics")?; + let totals = metrics + .as_object()? + .iter() + .filter_map(|(model, metric)| { + Some((model.clone(), copilot_totals(&metric["usage"])?)) + }) + .collect(); + Some(Shutdown { + marker, + ts_ms: marker.ts_ms?, + totals, + }) + }) + .collect(); + out.sort_by_key(|s| s.ts_ms); + out +} + +/// Output the session's messages billed per model, in `(after, through]`. +/// Only assistant messages carry Copilot's `outputTokens`. +fn message_output(ctx: &Context<'_>, after: i64, through: i64) -> HashMap { + let mut out = HashMap::new(); + for message in &ctx.ev.messages { + if message.ts_ms <= after || message.ts_ms > through { + continue; + } + let (usage, coverage) = usage_from_raw(ctx.source, message.raw_usage()); + if let (true, Some(model)) = (coverage.has_output_tokens, message.model.as_ref()) { + *out.entry(model.clone()).or_default() += usage.output; + } + } + out +} + +/// What a running total grew by. A total lower than the previous one +/// restarted, and counts whole. +fn growth(now: &Usage, before: Option<&Usage>) -> Usage { + let Some(before) = before else { + return now.clone(); + }; + let pairs = [ + (now.input, before.input), + (now.output, before.output), + (now.reasoning, before.reasoning), + (now.cache_read, before.cache_read), + (now.cache_create_5m, before.cache_create_5m), + ]; + if pairs.iter().any(|(n, b)| n < b) { + return now.clone(); + } + Usage { + input: now.input - before.input, + output: now.output - before.output, + reasoning: now.reasoning - before.reasoning, + cache_read: now.cache_read - before.cache_read, + cache_create_5m: now.cache_create_5m - before.cache_create_5m, + cache_create_1h: 0, + } +} + +fn shutdown_turns(ctx: &Context<'_>) -> Vec { + let mut turns = Vec::new(); + let mut previous: HashMap = HashMap::new(); + let mut after = i64::MIN; + for shutdown in shutdowns(ctx) { + let billed = message_output(ctx, after, shutdown.ts_ms); + for (model, totals) in &shutdown.totals { + let mut usage = growth(totals, previous.get(model)); + usage.output = usage + .output + .saturating_sub(billed.get(model).copied().unwrap_or(0)); + if usage != Usage::default() { + turns.push(aggregate_turn(ctx, &shutdown, model, usage)); + } + previous.insert(model.to_string(), totals.clone()); + } + after = shutdown.ts_ms; + } + turns +} + +fn aggregate_turn( + ctx: &Context<'_>, + shutdown: &Shutdown<'_>, + model: &str, + usage: Usage, +) -> TurnRecord { + let resolved = ctx.ev.session.cwd.as_deref().map(resolve_project); + TurnRecord { + v: 1, + source: ctx.source, + session_id: ctx.session_id().to_string(), + session_path: ctx + .ev + .session + .raw_path + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + message_id: format!("{}:{model}", shutdown.marker.marker_uid), + turn_index: 0, + ts: format_iso_ms(shutdown.ts_ms), + model: model.to_string(), + project: resolved.as_ref().map(|r| r.project.clone()), + project_key: resolved.and_then(|r| r.project_key), + usage, + tool_calls: Vec::new(), + files_touched: None, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: Some(Fidelity::new( + UsageGranularity::PerSessionAggregate, + Coverage { + has_input_tokens: true, + has_output_tokens: true, + has_reasoning_tokens: true, + has_cache_read_tokens: true, + has_cache_create_tokens: true, + ..Coverage::default() + }, + )), + reasoning: None, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::reader::classifier::normalize_tool_name; + + fn usage(input: u64, output: u64, cache_read: u64) -> Usage { + Usage { + input, + output, + cache_read, + ..Usage::default() + } + } + + /// Records for one Copilot journal, read through a throwaway store. + fn records_for(journal: &str) -> SessionRecords { + let home = tempfile::tempdir().unwrap(); + let sid = "boundary-session"; + let dir = home.path().join(".copilot/session-state").join(sid); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("events.jsonl"), journal).unwrap(); + let mut options = ai_hist::StoreOptions::default(); + options.db_path = Some(home.path().join("ai-history.db")); + options.roots = Some(ai_hist::ProviderRoots::from_home( + home.path().to_path_buf(), + home.path().join("opencode.db"), + )); + let store = ai_hist::SessionStore::open(options).unwrap(); + store.sync(Default::default()).unwrap(); + let evidence = store + .session( + &ai_hist::SessionRef::id(ai_hist::Source::Copilot, sid), + ai_hist::SessionQuery::default(), + ) + .unwrap() + .unwrap(); + super::super::records_from_evidence(&evidence, &crate::source::Billing::settled()) + } + + #[test] + fn a_message_written_with_its_shutdown_is_billed_once() { + let at = "2026-09-18T21:30:05.000Z"; + let journal = [ + json!({"type":"session.start","data":{"version":1,"sessionId":"boundary-session","context":{"cwd":"/tmp/project"}},"id":"start","timestamp":"2026-09-18T21:30:00.000Z","parentId":null}), + json!({"type":"assistant.message","data":{"messageId":"m1","model":"gpt-5.4","content":"done","outputTokens":5},"id":"a1","timestamp":at,"parentId":"start"}), + json!({"type":"session.shutdown","data":{"modelMetrics":{"gpt-5.4":{"usage":{"inputTokens":100,"outputTokens":7,"cacheReadTokens":0,"cacheWriteTokens":0}}}},"id":"shut","timestamp":at,"parentId":"a1"}), + ] + .iter() + .map(|v| format!("{v}\n")) + .collect::(); + let records = records_for(&journal); + let output: u64 = records.turns.iter().map(|t| t.usage.output).sum(); + let input: u64 = records.turns.iter().map(|t| t.usage.input).sum(); + assert_eq!((input, output), (100, 7)); + } + + #[test] + fn running_totals_bill_their_growth_and_a_restart_whole() { + let first = usage(100, 10, 50); + assert_eq!(growth(&first, None), first); + assert_eq!(growth(&usage(160, 15, 90), Some(&first)), usage(60, 5, 40)); + let restarted = usage(20, 2, 0); + assert_eq!(growth(&restarted, Some(&first)), restarted); + } + + fn call(name: &str) -> ToolCall { + ToolCall { + id: "id".into(), + name: name.into(), + target: None, + args_hash: String::new(), + is_error: None, + edit_pre_hash: None, + edit_post_hash: None, + skill_name: None, + replaced_tools: None, + collapsed_calls: None, + } + } + + #[test] + fn copilot_tools_name_their_targets_and_edit_hashes() { + let mut edit = call("edit"); + apply_tool( + &mut edit, + &json!({"path": "/src/lib.rs", "old_str": "a", "new_str": "b"}), + ); + assert_eq!(edit.target.as_deref(), Some("/src/lib.rs")); + assert_eq!(edit.edit_pre_hash, Some(content_hash("a"))); + assert_eq!(edit.edit_post_hash, Some(content_hash("b"))); + let mut create = call("create"); + apply_tool(&mut create, &json!({"path": "/new.rs", "file_text": "c"})); + assert_eq!(create.edit_post_hash, Some(content_hash("c"))); + for (name, input, target) in [ + ("bash", json!({"command": "ls"}), "ls"), + ("grep", json!({"pattern": "p"}), "p"), + ("web_fetch", json!({"url": "u"}), "u"), + ( + "task", + json!({"agent_type": "explore", "description": "d"}), + "explore", + ), + ] { + let mut c = call(name); + apply_tool(&mut c, &input); + assert_eq!(c.target.as_deref(), Some(target), "{name}"); + } + for (name, canonical) in [ + ("view", "Read"), + ("create", "Write"), + ("edit", "Edit"), + ("bash", "Bash"), + ("web_fetch", "WebFetch"), + ("read_agent", "Task"), + ] { + assert_eq!(normalize_tool_name(name), canonical, "{name}"); + } + } +} diff --git a/crates/relayburn-sdk/src/source/usage.rs b/crates/relayburn-sdk/src/source/usage.rs index 67b9db0f..6afa7691 100644 --- a/crates/relayburn-sdk/src/source/usage.rs +++ b/crates/relayburn-sdk/src/source/usage.rs @@ -15,6 +15,7 @@ pub(crate) fn usage_from_raw(source: SourceKind, raw: Option<&str>) -> (Usage, C }; match source { SourceKind::Opencode => opencode(obj), + SourceKind::CopilotCli => copilot_message(obj), _ => anthropic(obj), } } @@ -95,6 +96,38 @@ fn opencode(obj: &Map) -> (Usage, Coverage) { (usage, coverage) } +/// Copilot CLI's per-message counter: `assistant.message.outputTokens`, the +/// only usage it persists per request. Input and cache stay unmeasured. +fn copilot_message(obj: &Map) -> (Usage, Coverage) { + let usage = Usage { + output: n(obj, "outputTokens"), + ..Usage::default() + }; + let coverage = Coverage { + has_output_tokens: obj.get("outputTokens").is_some_and(Value::is_u64), + ..Coverage::default() + }; + (usage, coverage) +} + +/// One model's running totals from a Copilot `session.shutdown` +/// `modelMetrics[model].usage`: `inputTokens` includes cache reads and +/// writes, `outputTokens` includes reasoning. Cache writes carry no TTL and +/// bill at the 5-minute rate. `None` when a counter is missing or not a +/// non-negative integer. +pub(crate) fn copilot_totals(usage: &Value) -> Option { + let counter = |key: &str| usage.get(key)?.as_u64(); + let (read, write) = (counter("cacheReadTokens")?, counter("cacheWriteTokens")?); + Some(Usage { + input: counter("inputTokens")?.checked_sub(read + write)?, + output: counter("outputTokens")?, + reasoning: counter("reasoningTokens").unwrap_or(0), + cache_read: read, + cache_create_5m: write, + cache_create_1h: 0, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -121,4 +154,37 @@ mod tests { assert!(!coverage.has_cache_create_tokens); assert!(coverage.has_input_tokens); } + + #[test] + fn copilot_messages_measure_output_only() { + let (usage, coverage) = + usage_from_raw(SourceKind::CopilotCli, Some(r#"{"outputTokens": 42}"#)); + assert_eq!(usage.output, 42); + assert!(coverage.has_output_tokens && !coverage.has_input_tokens); + let (_, coverage) = usage_from_raw(SourceKind::CopilotCli, Some(r#"{"outputTokens": -1}"#)); + assert!(!coverage.has_output_tokens); + } + + #[test] + fn copilot_totals_split_cache_out_of_input() { + let usage = serde_json::json!({"inputTokens": 30000, "outputTokens": 82, + "cacheReadTokens": 20000, "cacheWriteTokens": 9000, "reasoningTokens": 4}); + let totals = copilot_totals(&usage).unwrap(); + assert_eq!( + ( + totals.input, + totals.cache_read, + totals.cache_create_5m, + totals.output, + totals.reasoning + ), + (1000, 20000, 9000, 82, 4) + ); + assert!( + copilot_totals(&serde_json::json!({"inputTokens": 1, "outputTokens": 1, + "cacheReadTokens": 2, "cacheWriteTokens": 0})) + .is_none() + ); + assert!(copilot_totals(&serde_json::json!({"inputTokens": 1})).is_none()); + } } diff --git a/docs/migrating-to-5.md b/docs/migrating-to-5.md index df0a8f02..bf72071d 100644 --- a/docs/migrating-to-5.md +++ b/docs/migrating-to-5.md @@ -16,6 +16,9 @@ few numbers that were wrong before. - **OpenCode's SQLite store is read.** Current OpenCode releases keep sessions in `~/.local/share/opencode/opencode.db`; burn reads it (and the older `storage/` JSON tree). +- **GitHub Copilot CLI sessions are read.** Copilot's session journals + (`~/.copilot/session-state//events.jsonl`) are ingested retroactively as + `copilot-cli` turns, with no exporter to enable. - **Archived Codex threads are read.** Rollouts Codex moved to `~/.codex/archived_sessions/` are ingested like live ones. - **Codex usage comes from each response's raw token snapshot**, and a Codex @@ -87,7 +90,7 @@ names. | Variable | In burn 5 | |---|---| | `AI_HIST_DB` | The relayhistory store burn reads. | -| `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `OPENCODE_DB` | Harness store locations, resolved by relayhistory. | +| `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `OPENCODE_DB`, `COPILOT_HOME` | Harness store locations, resolved by relayhistory. | | `BURN_CLAUDE_PROJECTS_DIR` | Removed. Point `CLAUDE_CONFIG_DIR` (or `HOME`) at the Claude install instead. | | `RELAYBURN_HOME`, `RELAYBURN_*` ledger settings | Unchanged. | diff --git a/tests/fixtures/copilot/cli-usage.jsonl b/tests/fixtures/copilot/cli-usage.jsonl new file mode 100644 index 00000000..0839a82e --- /dev/null +++ b/tests/fixtures/copilot/cli-usage.jsonl @@ -0,0 +1,37 @@ +{"type":"session.start","data":{"sessionId":"5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81","version":1,"producer":"copilot-agent","copilotVersion":"1.0.86","startTime":"2026-09-18T21:30:00.000Z","selectedModel":"claude-sonnet-4.6","contextTier":null,"context":{"cwd":"/tmp/project","gitRoot":"/tmp/project","branch":"main","headCommit":"1111111111111111111111111111111111111111","baseCommit":"1111111111111111111111111111111111111111","repository":"fixture/copilot-usage","hostType":"github","repositoryHost":"github.com"},"alreadyInUse":false,"remoteSteerable":false},"id":"2e802722-1ca7-5000-a0cd-87f77f27839c","timestamp":"2026-09-18T21:30:01.037Z","parentId":null} +{"type":"system.message","data":{"role":"system","content":"Synthetic system instructions.","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650"},"id":"b9b9f7c9-425e-5378-916f-cbc87bc9ea9f","timestamp":"2026-09-18T21:30:02.074Z","parentId":"2e802722-1ca7-5000-a0cd-87f77f27839c"} +{"type":"user.message","data":{"content":"Fix the failing fixture test.","transformedContent":"2026-09-18T21:30:00Z\n\nFix the failing fixture test.","attachments":[],"interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","messageId":"1564ff31-b476-509b-96c8-6fd67664ce83","turnId":"0"},"id":"875652b8-0f90-5d2a-84b2-6d9cf1b64b6a","timestamp":"2026-09-18T21:30:03.111Z","parentId":"b9b9f7c9-425e-5378-916f-cbc87bc9ea9f"} +{"type":"assistant.turn_start","data":{"turnId":"0","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650"},"id":"9b1f1fcb-695e-50be-b067-82371828b8cc","timestamp":"2026-09-18T21:30:04.148Z","parentId":"875652b8-0f90-5d2a-84b2-6d9cf1b64b6a"} +{"type":"assistant.message","data":{"messageId":"b14d26db-ba18-5214-8347-f994214c56f7","model":"claude-sonnet-4.6","content":"I will read the test first.","toolRequests":[{"toolCallId":"toolu_fixture_view","name":"view","arguments":{"path":"/tmp/project/tests/fixture.rs"},"type":"function","intentionSummary":"view the test"}],"interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","turnId":"0","reasoningText":"Inspect the test before changing it.","outputTokens":42,"requestId":"req-fixture-1","apiCallId":"a588514c-8a66-5568-8492-12766827fc42"},"id":"96587a9a-64d6-5de5-8308-c956ae2bf3bb","timestamp":"2026-09-18T21:30:05.185Z","parentId":"9b1f1fcb-695e-50be-b067-82371828b8cc"} +{"type":"tool.execution_start","data":{"toolCallId":"toolu_fixture_view","toolName":"view","arguments":{"path":"/tmp/project/tests/fixture.rs"},"turnId":"0","model":"claude-sonnet-4.6"},"id":"c17408c0-1286-5b65-8890-6b4ed47b6ad2","timestamp":"2026-09-18T21:30:06.222Z","parentId":"96587a9a-64d6-5de5-8308-c956ae2bf3bb"} +{"type":"tool.execution_complete","data":{"toolCallId":"toolu_fixture_view","success":true,"model":"claude-sonnet-4.6","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","turnId":"0","toolTelemetry":{},"result":{"content":"fn fixture() { assert_eq!(1, 2); }","detailedContent":"1. fn fixture() { assert_eq!(1, 2); }"}},"id":"f8f362fe-d064-5088-9721-f6e741fc2013","timestamp":"2026-09-18T21:30:07.259Z","parentId":"c17408c0-1286-5b65-8890-6b4ed47b6ad2"} +{"type":"assistant.turn_end","data":{"turnId":"0"},"id":"fda524fd-c318-55a7-bc30-b1a693c80a56","timestamp":"2026-09-18T21:30:08.296Z","parentId":"f8f362fe-d064-5088-9721-f6e741fc2013"} +{"type":"assistant.turn_start","data":{"turnId":"1","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650"},"id":"9fcb6c29-8b2d-527a-a632-c6bc9db1dbf9","timestamp":"2026-09-18T21:30:09.333Z","parentId":"fda524fd-c318-55a7-bc30-b1a693c80a56"} +{"type":"assistant.message","data":{"messageId":"47852f90-5c76-5d70-b1c0-e3afa4e1ab4e","model":"claude-sonnet-4.6","content":"","toolRequests":[{"toolCallId":"bash_1","name":"bash","arguments":{"command":"cargo test fixture","description":"Run the fixture test"},"type":"function"}],"interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","turnId":"1","outputTokens":17,"apiCallId":"9e981797-9daa-5320-bb47-39402851caef"},"id":"b196e98c-55a3-5596-842b-0188776a77e6","timestamp":"2026-09-18T21:30:10.370Z","parentId":"9fcb6c29-8b2d-527a-a632-c6bc9db1dbf9"} +{"type":"tool.execution_start","data":{"toolCallId":"bash_1","toolName":"bash","arguments":{"command":"cargo test fixture","description":"Run the fixture test"},"turnId":"1","model":"claude-sonnet-4.6"},"id":"82d0ecdb-45a3-57e6-ae68-185c60b69516","timestamp":"2026-09-18T21:30:11.407Z","parentId":"b196e98c-55a3-5596-842b-0188776a77e6"} +{"type":"tool.execution_complete","data":{"toolCallId":"bash_1","success":false,"model":"claude-sonnet-4.6","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","turnId":"1","toolTelemetry":{},"error":{"code":"failure","message":"test fixture ... FAILED; exit status 101"}},"id":"7815bb27-eb11-56cb-9543-6f89ba4aa4da","timestamp":"2026-09-18T21:30:12.444Z","parentId":"82d0ecdb-45a3-57e6-ae68-185c60b69516"} +{"type":"assistant.turn_end","data":{"turnId":"1"},"id":"6d393a88-14ac-58c8-a903-b242c127b5e9","timestamp":"2026-09-18T21:30:13.481Z","parentId":"7815bb27-eb11-56cb-9543-6f89ba4aa4da"} +{"type":"assistant.turn_start","data":{"turnId":"2","interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650"},"id":"2df822bb-2385-5a05-aaa0-5b7fc1e387e6","timestamp":"2026-09-18T21:30:14.518Z","parentId":"6d393a88-14ac-58c8-a903-b242c127b5e9"} +{"type":"assistant.message","data":{"messageId":"12c7b239-8581-501f-947a-60d2a17ec0ea","model":"claude-sonnet-4.6","content":"The fixture test fails with exit status 101.","toolRequests":[],"interactionId":"3de6d466-1896-5d0b-bcf2-09d20c3ac650","turnId":"2","phase":"final_answer","outputTokens":23,"apiCallId":"a8801d12-c836-5e75-92d9-2b91ee2963fb"},"id":"3ba731e1-7684-54b1-8549-b3ca7a4ccfc1","timestamp":"2026-09-18T21:30:15.555Z","parentId":"2df822bb-2385-5a05-aaa0-5b7fc1e387e6"} +{"type":"assistant.turn_end","data":{"turnId":"2"},"id":"d35e2cb1-8bca-5f26-abf2-bf7281f77cbb","timestamp":"2026-09-18T21:30:16.592Z","parentId":"3ba731e1-7684-54b1-8549-b3ca7a4ccfc1"} +{"type":"session.model_change","data":{"previousModel":"claude-sonnet-4.6","newModel":"gpt-5.4","source":"user"},"id":"c6467fbf-0e37-5db9-95a6-1cdd80feac82","timestamp":"2026-09-18T21:30:17.629Z","parentId":"d35e2cb1-8bca-5f26-abf2-bf7281f77cbb"} +{"type":"user.message","data":{"content":"Explore why it fails, then summarize.","attachments":[],"interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","messageId":"38c8444c-39d0-522e-9333-48c47ed721bd","turnId":"3"},"id":"003d05ab-bbfd-5192-831b-43918e74da61","timestamp":"2026-09-18T21:30:18.666Z","parentId":"c6467fbf-0e37-5db9-95a6-1cdd80feac82"} +{"type":"assistant.turn_start","data":{"turnId":"3","interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759"},"id":"bf157f55-250d-50d4-bb6a-efec3d631dbf","timestamp":"2026-09-18T21:30:19.703Z","parentId":"003d05ab-bbfd-5192-831b-43918e74da61"} +{"type":"assistant.message","data":{"messageId":"00183503-c6e5-5f9c-82fe-60c1190b312f","model":"gpt-5.4","content":"Delegating exploration.","toolRequests":[{"toolCallId":"call_fixture_task","name":"task","arguments":{"agent_type":"explore","prompt":"Find the failing assertion."},"type":"function"}],"interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","turnId":"3","apiCallId":"197fcb23-6847-557c-9caf-36babaf37cd3"},"id":"c68b0900-7390-5862-bd82-d275f0c95703","timestamp":"2026-09-18T21:30:20.740Z","parentId":"bf157f55-250d-50d4-bb6a-efec3d631dbf"} +{"type":"tool.execution_start","data":{"toolCallId":"call_fixture_task","toolName":"task","arguments":{"agent_type":"explore","prompt":"Find the failing assertion."},"turnId":"3","model":"gpt-5.4"},"id":"29f42ed2-05a0-5c6a-99ee-8d6b92014871","timestamp":"2026-09-18T21:30:21.777Z","parentId":"c68b0900-7390-5862-bd82-d275f0c95703"} +{"type":"subagent.started","data":{"toolCallId":"call_fixture_task","agentName":"explore","agentDisplayName":"Explore","agentDescription":"Read-only exploration","agentType":"explore","model":"gpt-5.4-mini"},"id":"c967caed-f17c-5e78-8657-050eeb09cabc","timestamp":"2026-09-18T21:30:22.814Z","parentId":"29f42ed2-05a0-5c6a-99ee-8d6b92014871"} +{"type":"assistant.message","data":{"messageId":"ad48d783-9ce7-5135-915d-7155250c45a3","model":"gpt-5.4-mini","content":"","toolRequests":[{"toolCallId":"bash_1","name":"bash","arguments":{"command":"grep -n assert_eq tests"},"type":"function"}],"interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","turnId":"3","outputTokens":11,"apiCallId":"61d89d44-630b-5365-978b-8c9d77b6b7f2","parentToolCallId":"call_fixture_task"},"id":"a4e048d6-9849-5de7-b219-8f82011fda96","timestamp":"2026-09-18T21:30:23.851Z","parentId":"c967caed-f17c-5e78-8657-050eeb09cabc"} +{"type":"tool.execution_start","data":{"toolCallId":"bash_1","toolName":"bash","arguments":{"command":"grep -n assert_eq tests"},"turnId":"3","model":"gpt-5.4-mini","parentToolCallId":"call_fixture_task"},"id":"dfdf093c-fc11-5637-879d-88f25e56de46","timestamp":"2026-09-18T21:30:24.888Z","parentId":"a4e048d6-9849-5de7-b219-8f82011fda96"} +{"type":"tool.execution_complete","data":{"toolCallId":"bash_1","success":true,"model":"gpt-5.4-mini","interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","turnId":"3","toolTelemetry":{},"result":{"content":"tests/fixture.rs:1: assert_eq!(1, 2)"},"parentToolCallId":"call_fixture_task"},"id":"fbbc0152-f87c-5712-816d-395156bff3bd","timestamp":"2026-09-18T21:30:25.925Z","parentId":"dfdf093c-fc11-5637-879d-88f25e56de46"} +{"type":"subagent.completed","data":{"toolCallId":"call_fixture_task","agentName":"explore","agentDisplayName":"Explore","model":"gpt-5.4-mini","totalToolCalls":1,"totalTokens":4011,"durationMs":1200},"id":"fa7dbdaf-caa2-5d36-a968-c5c27717b08b","timestamp":"2026-09-18T21:30:26.962Z","parentId":"fbbc0152-f87c-5712-816d-395156bff3bd"} +{"type":"tool.execution_complete","data":{"toolCallId":"call_fixture_task","success":true,"model":"gpt-5.4","interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","turnId":"3","toolTelemetry":{},"result":{"content":"The assertion compares 1 with 2."}},"id":"f994f3d2-7c9d-5fc9-896c-68c91719664c","timestamp":"2026-09-18T21:30:27.999Z","parentId":"fa7dbdaf-caa2-5d36-a968-c5c27717b08b"} +{"type":"assistant.message","data":{"messageId":"665ba8d7-b0c0-5087-9e8d-499f0a97d65c","model":"gpt-5.4","content":"Summary: the assertion compares 1 with 2.","toolRequests":[],"interactionId":"f0cd6bb3-bc6b-5e4e-86fc-5cab2416f759","turnId":"3","phase":"final_answer","outputTokens":31,"apiCallId":"ac1791f7-65f5-55c1-8062-ea89bc3011da"},"id":"b8b6bf6f-7b3c-5a11-a33e-fbe4d43a13e5","timestamp":"2026-09-18T21:30:28.036Z","parentId":"f994f3d2-7c9d-5fc9-896c-68c91719664c"} +{"type":"assistant.turn_end","data":{"turnId":"3"},"id":"d3fe6914-e863-5cba-8d0c-8605bdfe3cc9","timestamp":"2026-09-18T21:30:29.073Z","parentId":"b8b6bf6f-7b3c-5a11-a33e-fbe4d43a13e5"} +{"type":"session.usage_checkpoint","data":{"totalNanoAiu":120000000,"totalPremiumRequests":2,"modelCacheState":[],"promptCacheBreakState":[]},"id":"28c40cf2-159e-5268-a9f2-4c51607c789b","timestamp":"2026-09-18T21:30:30.110Z","parentId":"d3fe6914-e863-5cba-8d0c-8605bdfe3cc9"} +{"type":"session.shutdown","data":{"shutdownType":"routine","totalPremiumRequests":2,"totalNanoAiu":150000000,"tokenDetails":{"input":{"tokenCount":13000},"cache_read":{"tokenCount":24000},"cache_write":{"tokenCount":9000},"output":{"tokenCount":153}},"totalApiDurationMs":9100,"sessionStartTime":1789767000000,"codeChanges":{"linesAdded":0,"linesRemoved":0,"filesModified":[]},"modelMetrics":{"claude-sonnet-4.6":{"requests":{"count":3,"cost":1},"usage":{"inputTokens":30000,"outputTokens":82,"cacheReadTokens":20000,"cacheWriteTokens":9000,"reasoningTokens":0},"totalNanoAiu":90000000,"tokenDetails":{"input":{"tokenCount":1000},"cache_read":{"tokenCount":20000},"cache_write":{"tokenCount":9000},"output":{"tokenCount":82}}},"gpt-5.4":{"requests":{"count":2,"cost":1},"usage":{"inputTokens":12000,"outputTokens":60,"cacheReadTokens":4000,"cacheWriteTokens":0,"reasoningTokens":20},"totalNanoAiu":50000000,"tokenDetails":{"input":{"tokenCount":8000},"cache_read":{"tokenCount":4000},"cache_write":{"tokenCount":0},"output":{"tokenCount":60}}},"gpt-5.4-mini":{"requests":{"count":1,"cost":1},"usage":{"inputTokens":4000,"outputTokens":11,"cacheReadTokens":0,"cacheWriteTokens":0,"reasoningTokens":5},"totalNanoAiu":10000000,"tokenDetails":{"input":{"tokenCount":4000},"cache_read":{"tokenCount":0},"cache_write":{"tokenCount":0},"output":{"tokenCount":11}}}},"currentModel":"gpt-5.4","currentTokens":21000,"systemTokens":4000,"conversationTokens":16000,"toolDefinitionsTokens":1000},"id":"fc9ac928-bb14-5990-9d7f-43fe18ed7ae6","timestamp":"2026-09-18T21:30:31.147Z","parentId":"28c40cf2-159e-5268-a9f2-4c51607c789b"} +{"type":"session.resume","data":{"resumeTime":"2026-09-18T22:00:00.000Z","eventCount":31,"selectedModel":"gpt-5.4","context":{"cwd":"/tmp/project","gitRoot":"/tmp/project","branch":"main","headCommit":"1111111111111111111111111111111111111111","baseCommit":"1111111111111111111111111111111111111111","repository":"fixture/copilot-usage","hostType":"github","repositoryHost":"github.com"},"alreadyInUse":false,"sessionWasActive":false},"id":"c5fb3e7b-9b77-51e7-844f-5ea155d20e42","timestamp":"2026-09-18T21:30:32.184Z","parentId":"fc9ac928-bb14-5990-9d7f-43fe18ed7ae6"} +{"type":"user.message","data":{"content":"Thanks, that is enough.","attachments":[],"interactionId":"8fd9c144-a932-57ae-a37c-1ef875408fc8","messageId":"79819aee-37b4-56f9-a8be-efe240865e3e","turnId":"4"},"id":"171c4789-018a-5061-8ce1-e3e407ebe8fb","timestamp":"2026-09-18T21:30:33.221Z","parentId":"c5fb3e7b-9b77-51e7-844f-5ea155d20e42"} +{"type":"assistant.turn_start","data":{"turnId":"4","interactionId":"8fd9c144-a932-57ae-a37c-1ef875408fc8"},"id":"b4c210ce-bf3b-589b-b1ba-41b6bd0a8706","timestamp":"2026-09-18T21:30:34.258Z","parentId":"171c4789-018a-5061-8ce1-e3e407ebe8fb"} +{"type":"assistant.message","data":{"messageId":"04ec1c7f-9108-5f20-aa61-5a10cda9de23","model":"gpt-5.4","content":"You're welcome.","toolRequests":[],"interactionId":"8fd9c144-a932-57ae-a37c-1ef875408fc8","turnId":"4","phase":"final_answer","outputTokens":5,"apiCallId":"f3e6c90c-bb88-5fe8-b4b4-e67683992096"},"id":"adcaddae-fdce-568a-9958-a9aaf6fd03a4","timestamp":"2026-09-18T21:30:35.295Z","parentId":"b4c210ce-bf3b-589b-b1ba-41b6bd0a8706"} +{"type":"assistant.turn_end","data":{"turnId":"4"},"id":"16868d3d-b294-5764-8fde-67364566e997","timestamp":"2026-09-18T21:30:36.332Z","parentId":"adcaddae-fdce-568a-9958-a9aaf6fd03a4"} +{"type":"session.shutdown","data":{"shutdownType":"routine","totalPremiumRequests":2,"totalNanoAiu":150000000,"tokenDetails":{"input":{"tokenCount":16000},"cache_read":{"tokenCount":34000},"cache_write":{"tokenCount":9000},"output":{"tokenCount":158}},"totalApiDurationMs":9100,"sessionStartTime":1789767000000,"codeChanges":{"linesAdded":0,"linesRemoved":0,"filesModified":[]},"modelMetrics":{"claude-sonnet-4.6":{"requests":{"count":3,"cost":1},"usage":{"inputTokens":30000,"outputTokens":82,"cacheReadTokens":20000,"cacheWriteTokens":9000,"reasoningTokens":0},"totalNanoAiu":90000000,"tokenDetails":{"input":{"tokenCount":1000},"cache_read":{"tokenCount":20000},"cache_write":{"tokenCount":9000},"output":{"tokenCount":82}}},"gpt-5.4":{"requests":{"count":3,"cost":1},"usage":{"inputTokens":25000,"outputTokens":65,"cacheReadTokens":14000,"cacheWriteTokens":0,"reasoningTokens":22},"totalNanoAiu":60000000,"tokenDetails":{"input":{"tokenCount":11000},"cache_read":{"tokenCount":14000},"cache_write":{"tokenCount":0},"output":{"tokenCount":65}}},"gpt-5.4-mini":{"requests":{"count":1,"cost":1},"usage":{"inputTokens":4000,"outputTokens":11,"cacheReadTokens":0,"cacheWriteTokens":0,"reasoningTokens":5},"totalNanoAiu":10000000,"tokenDetails":{"input":{"tokenCount":4000},"cache_read":{"tokenCount":0},"cache_write":{"tokenCount":0},"output":{"tokenCount":11}}}},"currentModel":"gpt-5.4","currentTokens":21000,"systemTokens":4000,"conversationTokens":16000,"toolDefinitionsTokens":1000},"id":"3bd66697-92d9-568d-b7c4-dede0a476f3c","timestamp":"2026-09-18T21:30:37.369Z","parentId":"16868d3d-b294-5764-8fde-67364566e997"} diff --git a/tests/fixtures/copilot/official-importer.jsonl b/tests/fixtures/copilot/official-importer.jsonl new file mode 100644 index 00000000..8184afb8 --- /dev/null +++ b/tests/fixtures/copilot/official-importer.jsonl @@ -0,0 +1,6 @@ +{"type":"session.start","data":{"sessionId":"b3576354-cf9a-4912-8bd8-e969c9b94323","version":1,"producer":"copilot-agent","copilotVersion":"1.0.95","startTime":"2026-01-01T00:00:00.000Z","contextTier":null,"context":{"cwd":"/tmp/project"},"alreadyInUse":false},"id":"6010aa8f-4647-4b34-b4b1-4b2b6eef1b1d","timestamp":"2026-10-11T19:18:38.671Z","parentId":null} +{"type":"user.message","data":{"content":"Inspect the synthetic greeting."},"id":"23a7f17a-cf80-4bf0-84ce-b943957f3439","timestamp":"2026-10-11T19:18:38.672Z","parentId":"6010aa8f-4647-4b34-b4b1-4b2b6eef1b1d"} +{"type":"assistant.message","data":{"messageId":"28e4dc43-6d27-415f-9889-7a018434e36c","content":"I will read the synthetic greeting.","toolRequests":[{"name":"read","toolCallId":"2bd062f7-b79a-4158-a23f-7b9a076aa23c","arguments":{"path":"greeting.txt"}}]},"id":"64bb7028-928b-46c6-9156-2faaf8a50ba3","timestamp":"2026-10-11T19:18:38.672Z","parentId":"23a7f17a-cf80-4bf0-84ce-b943957f3439"} +{"type":"tool.execution_start","data":{"toolCallId":"2bd062f7-b79a-4158-a23f-7b9a076aa23c","toolName":"read","arguments":{"path":"greeting.txt"}},"id":"07e59fa3-ebd3-480b-8ddb-e62fd0bcfc77","timestamp":"2026-10-11T19:18:38.672Z","parentId":"64bb7028-928b-46c6-9156-2faaf8a50ba3"} +{"type":"tool.execution_complete","data":{"toolCallId":"2bd062f7-b79a-4158-a23f-7b9a076aa23c","success":true,"result":{"content":"Hello synthetic fixture"}},"id":"63f4f99c-6c46-4a58-b8a8-5c0f3becfdec","timestamp":"2026-10-11T19:18:38.672Z","parentId":"07e59fa3-ebd3-480b-8ddb-e62fd0bcfc77"} +{"type":"assistant.message","data":{"messageId":"5fd0999e-2bb7-4296-bddd-3b3fd418526d","content":"The greeting says Hello synthetic fixture."},"id":"4bfa2dcb-14d6-4dda-aa35-1d77fa486947","timestamp":"2026-10-11T19:18:38.672Z","parentId":"63f4f99c-6c46-4a58-b8a8-5c0f3becfdec"} diff --git a/tests/fixtures/sourcing-snapshots/README.md b/tests/fixtures/sourcing-snapshots/README.md index db22fd84..9cb02e0e 100644 --- a/tests/fixtures/sourcing-snapshots/README.md +++ b/tests/fixtures/sourcing-snapshots/README.md @@ -1,7 +1,7 @@ # Sourcing snapshots The `SessionRecords` burn derives from each session fixture under -`tests/fixtures/{claude,codex,opencode}`, one JSON file per session. They are +`tests/fixtures/{claude,codex,opencode,copilot}`, one JSON file per session. They are the frozen spec for mapping relayhistory evidence onto burn records: `source::parity_tests` stages each fixture in a throwaway HOME (its `/tmp/project` cwd fields pointed at an empty directory there, so nothing on @@ -20,4 +20,10 @@ differ: progress (no `stop_reason` yet) is not emitted; it becomes a turn once its response settles. +The `copilot-*` snapshots have no original reader behind them: they are burn's +mapping of the Copilot CLI journals relayhistory captures, reviewed when the +source was added. `copilot-cli-usage.json` bills each `session.shutdown`'s +growth in Copilot's per-model running totals as `per-session-aggregate` turns, +so its turns sum to the final `modelMetrics`. + The files change only by hand, alongside the mapping change they describe. diff --git a/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json b/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json new file mode 100644 index 00000000..76aa1eea --- /dev/null +++ b/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json @@ -0,0 +1,727 @@ +{ + "turns": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "b14d26db-ba18-5214-8347-f994214c56f7", + "turnIndex": 0, + "ts": "2026-09-18T21:30:05.185Z", + "model": "claude-sonnet-4.6", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 42, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [ + { + "id": "toolu_fixture_view", + "name": "view", + "target": "/tmp/project/tests/fixture.rs", + "argsHash": "657d3e56bf6bee66" + } + ], + "filesTouched": [ + "/tmp/project/tests/fixture.rs" + ], + "activity": "debugging", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "47852f90-5c76-5d70-b1c0-e3afa4e1ab4e", + "turnIndex": 1, + "ts": "2026-09-18T21:30:10.370Z", + "model": "claude-sonnet-4.6", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 17, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [ + { + "id": "bash_1", + "name": "bash", + "target": "cargo test fixture", + "argsHash": "03d9abe8e9377c39", + "isError": true + } + ], + "activity": "debugging", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "12c7b239-8581-501f-947a-60d2a17ec0ea", + "turnIndex": 2, + "ts": "2026-09-18T21:30:15.555Z", + "model": "claude-sonnet-4.6", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 23, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [], + "activity": "debugging", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": false, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "00183503-c6e5-5f9c-82fe-60c1190b312f", + "turnIndex": 3, + "ts": "2026-09-18T21:30:20.740Z", + "model": "gpt-5.4", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [ + { + "id": "call_fixture_task", + "name": "task", + "target": "explore", + "argsHash": "4ae367a468ed01af" + } + ], + "activity": "delegation", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "ad48d783-9ce7-5135-915d-7155250c45a3", + "turnIndex": 4, + "ts": "2026-09-18T21:30:23.851Z", + "model": "gpt-5.4-mini", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 11, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [ + { + "id": "call_fixture_task/bash_1", + "name": "bash", + "target": "grep -n assert_eq tests", + "argsHash": "ddc5bcdefb280246" + } + ], + "subagent": { + "isSidechain": true + }, + "activity": "exploration", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "665ba8d7-b0c0-5087-9e8d-499f0a97d65c", + "turnIndex": 5, + "ts": "2026-09-18T21:30:28.036Z", + "model": "gpt-5.4", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 31, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [], + "activity": "brainstorming", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-session-aggregate", + "coverage": { + "hasInputTokens": true, + "hasOutputTokens": true, + "hasReasoningTokens": true, + "hasCacheReadTokens": true, + "hasCacheCreateTokens": true, + "hasToolCalls": false, + "hasToolResultEvents": false, + "hasSessionRelationships": false, + "hasRawContent": false + }, + "class": "aggregate-only" + }, + "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:claude-sonnet-4.6", + "turnIndex": 6, + "ts": "2026-09-18T21:30:31.147Z", + "model": "claude-sonnet-4.6", + "project": "/tmp/project", + "usage": { + "input": 1000, + "output": 0, + "reasoning": 0, + "cacheRead": 20000, + "cacheCreate5m": 9000, + "cacheCreate1h": 0 + }, + "toolCalls": [] + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-session-aggregate", + "coverage": { + "hasInputTokens": true, + "hasOutputTokens": true, + "hasReasoningTokens": true, + "hasCacheReadTokens": true, + "hasCacheCreateTokens": true, + "hasToolCalls": false, + "hasToolResultEvents": false, + "hasSessionRelationships": false, + "hasRawContent": false + }, + "class": "aggregate-only" + }, + "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:gpt-5.4", + "turnIndex": 7, + "ts": "2026-09-18T21:30:31.147Z", + "model": "gpt-5.4", + "project": "/tmp/project", + "usage": { + "input": 8000, + "output": 29, + "reasoning": 20, + "cacheRead": 4000, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [] + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-session-aggregate", + "coverage": { + "hasInputTokens": true, + "hasOutputTokens": true, + "hasReasoningTokens": true, + "hasCacheReadTokens": true, + "hasCacheCreateTokens": true, + "hasToolCalls": false, + "hasToolResultEvents": false, + "hasSessionRelationships": false, + "hasRawContent": false + }, + "class": "aggregate-only" + }, + "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:gpt-5.4-mini", + "turnIndex": 8, + "ts": "2026-09-18T21:30:31.147Z", + "model": "gpt-5.4-mini", + "project": "/tmp/project", + "usage": { + "input": 4000, + "output": 0, + "reasoning": 5, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [] + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": true, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "04ec1c7f-9108-5f20-aa61-5a10cda9de23", + "turnIndex": 9, + "ts": "2026-09-18T21:30:35.295Z", + "model": "gpt-5.4", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 5, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [], + "activity": "conversation", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "fidelity": { + "granularity": "per-session-aggregate", + "coverage": { + "hasInputTokens": true, + "hasOutputTokens": true, + "hasReasoningTokens": true, + "hasCacheReadTokens": true, + "hasCacheCreateTokens": true, + "hasToolCalls": false, + "hasToolResultEvents": false, + "hasSessionRelationships": false, + "hasRawContent": false + }, + "class": "aggregate-only" + }, + "messageId": "3bd66697-92d9-568d-b7c4-dede0a476f3c:gpt-5.4", + "turnIndex": 10, + "ts": "2026-09-18T21:30:37.369Z", + "model": "gpt-5.4", + "project": "/tmp/project", + "usage": { + "input": 3000, + "output": 0, + "reasoning": 2, + "cacheRead": 10000, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [] + } + ], + "content": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "875652b8-0f90-5d2a-84b2-6d9cf1b64b6a", + "ts": "2026-09-18T21:30:03.111Z", + "role": "user", + "kind": "text", + "text": "Fix the failing fixture test." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "b14d26db-ba18-5214-8347-f994214c56f7", + "ts": "2026-09-18T21:30:05.185Z", + "role": "assistant", + "kind": "thinking", + "text": "Inspect the test before changing it." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "b14d26db-ba18-5214-8347-f994214c56f7", + "ts": "2026-09-18T21:30:05.185Z", + "role": "assistant", + "kind": "text", + "text": "I will read the test first." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "b14d26db-ba18-5214-8347-f994214c56f7", + "ts": "2026-09-18T21:30:05.185Z", + "role": "assistant", + "kind": "tool_use", + "toolUse": { + "id": "toolu_fixture_view", + "name": "view", + "input": { + "path": "/tmp/project/tests/fixture.rs" + } + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "f8f362fe-d064-5088-9721-f6e741fc2013", + "ts": "2026-09-18T21:30:07.259Z", + "role": "tool_result", + "kind": "tool_result", + "toolResult": { + "toolUseId": "toolu_fixture_view", + "content": "fn fixture() { assert_eq!(1, 2); }" + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "47852f90-5c76-5d70-b1c0-e3afa4e1ab4e", + "ts": "2026-09-18T21:30:10.370Z", + "role": "assistant", + "kind": "tool_use", + "toolUse": { + "id": "bash_1", + "name": "bash", + "input": { + "command": "cargo test fixture", + "description": "Run the fixture test" + } + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "7815bb27-eb11-56cb-9543-6f89ba4aa4da", + "ts": "2026-09-18T21:30:12.444Z", + "role": "tool_result", + "kind": "tool_result", + "toolResult": { + "toolUseId": "bash_1", + "content": "test fixture ... FAILED; exit status 101", + "isError": true + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "12c7b239-8581-501f-947a-60d2a17ec0ea", + "ts": "2026-09-18T21:30:15.555Z", + "role": "assistant", + "kind": "text", + "text": "The fixture test fails with exit status 101." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "003d05ab-bbfd-5192-831b-43918e74da61", + "ts": "2026-09-18T21:30:18.666Z", + "role": "user", + "kind": "text", + "text": "Explore why it fails, then summarize." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "00183503-c6e5-5f9c-82fe-60c1190b312f", + "ts": "2026-09-18T21:30:20.740Z", + "role": "assistant", + "kind": "text", + "text": "Delegating exploration." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "00183503-c6e5-5f9c-82fe-60c1190b312f", + "ts": "2026-09-18T21:30:20.740Z", + "role": "assistant", + "kind": "tool_use", + "toolUse": { + "id": "call_fixture_task", + "name": "task", + "input": { + "agent_type": "explore", + "prompt": "Find the failing assertion." + } + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "ad48d783-9ce7-5135-915d-7155250c45a3", + "ts": "2026-09-18T21:30:23.851Z", + "role": "assistant", + "kind": "tool_use", + "toolUse": { + "id": "call_fixture_task/bash_1", + "name": "bash", + "input": { + "command": "grep -n assert_eq tests" + } + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "fbbc0152-f87c-5712-816d-395156bff3bd", + "ts": "2026-09-18T21:30:25.925Z", + "role": "tool_result", + "kind": "tool_result", + "toolResult": { + "toolUseId": "call_fixture_task/bash_1", + "content": "tests/fixture.rs:1: assert_eq!(1, 2)" + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "f994f3d2-7c9d-5fc9-896c-68c91719664c", + "ts": "2026-09-18T21:30:27.999Z", + "role": "tool_result", + "kind": "tool_result", + "toolResult": { + "toolUseId": "call_fixture_task", + "content": "The assertion compares 1 with 2." + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "665ba8d7-b0c0-5087-9e8d-499f0a97d65c", + "ts": "2026-09-18T21:30:28.036Z", + "role": "assistant", + "kind": "text", + "text": "Summary: the assertion compares 1 with 2." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "171c4789-018a-5061-8ce1-e3e407ebe8fb", + "ts": "2026-09-18T21:30:33.221Z", + "role": "user", + "kind": "text", + "text": "Thanks, that is enough." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "04ec1c7f-9108-5f20-aa61-5a10cda9de23", + "ts": "2026-09-18T21:30:35.295Z", + "role": "assistant", + "kind": "text", + "text": "You're welcome." + } + ], + "compactions": [], + "relationships": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "relationshipType": "root", + "ts": "2026-09-18T21:30:01.037Z", + "sourceVersion": "1.0.86" + } + ], + "toolResultEvents": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "f8f362fe-d064-5088-9721-f6e741fc2013", + "toolUseId": "toolu_fixture_view", + "eventIndex": 0, + "ts": "2026-09-18T21:30:07.259Z", + "status": "completed", + "eventSource": "tool_result", + "contentLength": 34, + "outputBytes": 34, + "outputTruncated": false, + "contentHash": "d23d0b2f38196865" + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "7815bb27-eb11-56cb-9543-6f89ba4aa4da", + "toolUseId": "bash_1", + "eventIndex": 0, + "ts": "2026-09-18T21:30:12.444Z", + "status": "errored", + "eventSource": "tool_result", + "contentLength": 40, + "outputBytes": 40, + "outputTruncated": false, + "contentHash": "afee4e556d283629", + "isError": true + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "fbbc0152-f87c-5712-816d-395156bff3bd", + "toolUseId": "call_fixture_task/bash_1", + "eventIndex": 0, + "ts": "2026-09-18T21:30:25.925Z", + "status": "completed", + "eventSource": "tool_result", + "contentLength": 36, + "outputBytes": 36, + "outputTruncated": false, + "contentHash": "63b76b0dd332dd0a" + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "messageId": "f994f3d2-7c9d-5fc9-896c-68c91719664c", + "toolUseId": "call_fixture_task", + "eventIndex": 0, + "ts": "2026-09-18T21:30:27.999Z", + "status": "completed", + "eventSource": "tool_result", + "contentLength": 32, + "outputBytes": 32, + "outputTruncated": false, + "contentHash": "226a25c9587ad927" + } + ], + "userTurns": [], + "requestIdLookup": [ + [ + "5f0c2a8e-1d4b-4c6e-9a7f-3b2d1e0c9a81", + "b14d26db-ba18-5214-8347-f994214c56f7", + "req-fixture-1" + ] + ] +} diff --git a/tests/fixtures/sourcing-snapshots/copilot-official-importer.json b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json new file mode 100644 index 00000000..3bdc6618 --- /dev/null +++ b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json @@ -0,0 +1,176 @@ +{ + "turns": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": false, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "28e4dc43-6d27-415f-9889-7a018434e36c", + "turnIndex": 0, + "ts": "2026-10-11T19:18:38.672Z", + "model": "", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [ + { + "id": "2bd062f7-b79a-4158-a23f-7b9a076aa23c", + "name": "read", + "target": "greeting.txt", + "argsHash": "6b3640bd99e591a8" + } + ], + "activity": "review", + "retries": 0, + "hasEdits": false + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "fidelity": { + "granularity": "per-turn", + "coverage": { + "hasInputTokens": false, + "hasOutputTokens": false, + "hasReasoningTokens": false, + "hasCacheReadTokens": false, + "hasCacheCreateTokens": false, + "hasToolCalls": true, + "hasToolResultEvents": true, + "hasSessionRelationships": true, + "hasRawContent": true + }, + "class": "partial" + }, + "messageId": "5fd0999e-2bb7-4296-bddd-3b3fd418526d", + "turnIndex": 1, + "ts": "2026-10-11T19:18:38.672Z", + "model": "", + "project": "/tmp/project", + "usage": { + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "toolCalls": [], + "activity": "review", + "retries": 0, + "hasEdits": false + } + ], + "content": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "23a7f17a-cf80-4bf0-84ce-b943957f3439", + "ts": "2026-10-11T19:18:38.672Z", + "role": "user", + "kind": "text", + "text": "Inspect the synthetic greeting." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "28e4dc43-6d27-415f-9889-7a018434e36c", + "ts": "2026-10-11T19:18:38.672Z", + "role": "assistant", + "kind": "text", + "text": "I will read the synthetic greeting." + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "28e4dc43-6d27-415f-9889-7a018434e36c", + "ts": "2026-10-11T19:18:38.672Z", + "role": "assistant", + "kind": "tool_use", + "toolUse": { + "id": "2bd062f7-b79a-4158-a23f-7b9a076aa23c", + "name": "read", + "input": { + "path": "greeting.txt" + } + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "63f4f99c-6c46-4a58-b8a8-5c0f3becfdec", + "ts": "2026-10-11T19:18:38.672Z", + "role": "tool_result", + "kind": "tool_result", + "toolResult": { + "toolUseId": "2bd062f7-b79a-4158-a23f-7b9a076aa23c", + "content": "Hello synthetic fixture" + } + }, + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "5fd0999e-2bb7-4296-bddd-3b3fd418526d", + "ts": "2026-10-11T19:18:38.672Z", + "role": "assistant", + "kind": "text", + "text": "The greeting says Hello synthetic fixture." + } + ], + "compactions": [], + "relationships": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "relationshipType": "root", + "ts": "2026-10-11T19:18:38.671Z", + "sourceVersion": "1.0.95" + } + ], + "toolResultEvents": [ + { + "v": 1, + "source": "copilot-cli", + "sessionId": "b3576354-cf9a-4912-8bd8-e969c9b94323", + "messageId": "63f4f99c-6c46-4a58-b8a8-5c0f3becfdec", + "toolUseId": "2bd062f7-b79a-4158-a23f-7b9a076aa23c", + "eventIndex": 0, + "ts": "2026-10-11T19:18:38.672Z", + "status": "completed", + "eventSource": "tool_result", + "contentLength": 23, + "outputBytes": 23, + "outputTruncated": false, + "contentHash": "d48af42aafa94f3c" + } + ], + "userTurns": [], + "requestIdLookup": [] +} From 594598a504c7a08c950c5656c29c7fbad9ea6c3f Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 00:52:57 -0700 Subject: [PATCH 02/10] Take main's lockfile resolution Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24abc60b..90e14c55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -431,7 +431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -977,7 +977,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1344,7 +1344,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1569,7 +1569,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2024,7 +2024,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From 4a6a704597884948f5172f642e9e30ae7b31c0ad Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 00:57:56 -0700 Subject: [PATCH 03/10] Keep reader types within the file-length target Co-Authored-By: Claude Opus 5.5 --- crates/relayburn-sdk/src/reader/types.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/relayburn-sdk/src/reader/types.rs b/crates/relayburn-sdk/src/reader/types.rs index 0540d56e..5f26ce15 100644 --- a/crates/relayburn-sdk/src/reader/types.rs +++ b/crates/relayburn-sdk/src/reader/types.rs @@ -7,8 +7,7 @@ //! continues to mean "absent" rather than "explicit null" — the ledger is //! append-only JSONL and we want byte-identical output to the TS writer. -use std::collections::BTreeMap; -use std::fmt; +use std::{collections::BTreeMap, fmt}; use serde::{Deserialize, Serialize}; use serde_json::Value; From ddb14006be7e5723387487c206d3a6a80c5fa926 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 01:53:30 -0700 Subject: [PATCH 04/10] Price and count Copilot shutdown totals as aggregates; read every file tool - A shutdown's aggregate turn records no model call (request count 0) and prices at the model's base rates: it sums requests of unknown size, so none is known to cross a context tier. - Total-token counts leave reasoning out wherever the source counts it inside output (Codex, Copilot CLI), not only for Codex. - Copilot's `read` and `write` touch files like `view`, `edit` and `create`; the hotspot file rollup reads every harness's name for a file tool through the classifier's aliases. - A turn's tool call is read from the evidence its own message made, so a reused Copilot call id never lends one turn another's target. Co-Authored-By: Claude Opus 5.5 --- crates/relayburn-sdk/src/analyze/cost.rs | 51 +++++++- crates/relayburn-sdk/src/analyze/findings.rs | 81 ++++++------ crates/relayburn-sdk/src/analyze/hotspots.rs | 8 +- .../src/analyze/hotspots_tests.rs | 28 +++++ .../relayburn-sdk/src/ingest/ingest_tests.rs | 9 ++ .../src/ingest/ingest_tests/copilot.rs | 117 ++++++++++++++++++ crates/relayburn-sdk/src/session_metrics.rs | 49 +++++++- .../src/source/relayhistory/copilot.rs | 31 +++-- .../sourcing-snapshots/copilot-cli-usage.json | 4 + .../copilot-official-importer.json | 3 + 10 files changed, 324 insertions(+), 57 deletions(-) create mode 100644 crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index de6dc522..408857a9 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -10,7 +10,7 @@ use std::borrow::Cow; use serde::{Deserialize, Serialize}; -use crate::reader::{SourceKind, TurnRecord, Usage}; +use crate::reader::{SourceKind, TurnRecord, Usage, UsageGranularity}; use crate::analyze::pricing::{ModelCost, ModelCostTier, PricingTable, ReasoningMode}; use crate::analyze::provider_reattribution::{resolve_provider, strip_provider_prefix}; @@ -39,6 +39,10 @@ pub struct CostBreakdown { #[derive(Debug, Clone, Copy, Default)] pub struct CostForUsageOptions { pub reasoning_mode: Option, + /// The usage sums requests whose sizes are unknown (a per-session + /// aggregate), so no request is known to cross a context tier: the + /// model's base rates apply. + pub base_tier: bool, } /// Tokens-per-million divisor shared by every per-token cost calc in the @@ -70,7 +74,12 @@ pub(crate) fn cost_for_usage( options: CostForUsageOptions, ) -> Option { let rate = lookup_model_rate(model, pricing)?; - let effective = effective_model_rate(usage, rate); + let tier = if options.base_tier { + None + } else { + context_tier_for_usage(usage, rate) + }; + let effective = rate_at_tier(rate, tier); let mode = options.reasoning_mode.unwrap_or(rate.reasoning_mode); let input = (usage.input as f64 / PER_MILLION) * effective.input; let output = (usage.output as f64 / PER_MILLION) * effective.output; @@ -105,6 +114,10 @@ pub fn cost_for_turn(turn: &TurnRecord, pricing: &PricingTable) -> Option(usage: &Usage, rate: &'a ModelCost) -> Option<&'a } pub(crate) fn effective_model_rate(usage: &Usage, rate: &ModelCost) -> EffectiveModelRate { - let tier = context_tier_for_usage(usage, rate); + rate_at_tier(rate, context_tier_for_usage(usage, rate)) +} + +fn rate_at_tier(rate: &ModelCost, tier: Option<&ModelCostTier>) -> EffectiveModelRate { EffectiveModelRate { input: tier.map_or(rate.input, |tier| tier.input), output: tier.map_or(rate.output, |tier| tier.output), @@ -194,6 +210,12 @@ pub(crate) fn reasoning_mode_for_source(source: SourceKind) -> Option bool { + reasoning_mode_for_source(source) == Some(ReasoningMode::IncludedInOutput) +} + /// Shared lookup: direct match → synthetic reattribution → generic /// `provider/model` strip. Used by `cost_for_usage` and (later) by the /// hotspots / claude-md attribution paths so synthetic-routed turns price @@ -652,6 +674,7 @@ mod tests { &p, CostForUsageOptions { reasoning_mode: Some(ReasoningMode::IncludedInOutput), + ..Default::default() }, ) .expect("skipped priced"); @@ -893,6 +916,28 @@ mod tests { t } + /// A per-session aggregate sums requests whose sizes are unknown: three + /// 100k-token gpt-5.4 calls total 300k, past the 272k tier, yet none of + /// them crossed it, so the aggregate prices at the base rates. + #[test] + fn a_session_aggregate_prices_at_the_base_tier() { + let p = load_builtin_pricing(); + let usage = Usage { + input: 300_000, + ..Usage::default() + }; + let mut aggregate = turn("gpt-5.4", usage.clone(), SourceKind::CopilotCli); + aggregate.fidelity = Some(Fidelity::new( + UsageGranularity::PerSessionAggregate, + Coverage::EMPTY, + )); + let per_turn = with_coverage(turn("gpt-5.4", usage, SourceKind::CopilotCli), true, true); + let base = cost_for_turn(&aggregate, &p).expect("priced").input; + let tiered = cost_for_turn(&per_turn, &p).expect("priced").input; + assert!((base - 0.75).abs() < 1e-9, "{base}"); + assert!((tiered - 1.50).abs() < 1e-9, "{tiered}"); + } + #[test] fn measured_zero_token_turns_cost_nothing_even_without_a_tariff() { let p = load_builtin_pricing(); diff --git a/crates/relayburn-sdk/src/analyze/findings.rs b/crates/relayburn-sdk/src/analyze/findings.rs index 4c9ac46e..7999cab8 100644 --- a/crates/relayburn-sdk/src/analyze/findings.rs +++ b/crates/relayburn-sdk/src/analyze/findings.rs @@ -418,7 +418,7 @@ fn unpriced_usage_by_session<'a>( } let usage = by_session.entry(&turn.session_id).or_default(); usage.turns += 1; - let separately_reported_reasoning = if matches!(turn.source, SourceKind::Codex) { + let separately_reported_reasoning = if super::cost::reasoning_in_output(turn.source) { 0 } else { turn.usage.reasoning @@ -1211,44 +1211,47 @@ mod tests { } #[test] - fn unpriced_codex_volume_does_not_double_count_reasoning_inside_output() { - let turn = TurnRecord { - v: 1, - source: SourceKind::Codex, - session_id: "codex-unknown".to_string(), - session_path: None, - message_id: "m1".to_string(), - turn_index: 0, - request_count: None, - ts: "2026-08-01T00:00:00.000Z".to_string(), - model: "future-unpriced-model".to_string(), - project: None, - project_key: None, - usage: crate::reader::Usage { - input: 400_000, - output: 20_000, - reasoning: 10_000, - cache_read: 13_000, - cache_create_5m: 0, - cache_create_1h: 0, - }, - tool_calls: Vec::new(), - files_touched: None, - subagent: None, - stop_reason: None, - activity: None, - retries: None, - has_edits: None, - fidelity: None, - reasoning: None, - }; - - let findings = unpriced_usage_findings(&[turn], &PricingTable::new()); - - assert_eq!( - findings[0].estimated_savings.tokens_per_session, - Some(433_000) - ); + fn unpriced_volume_does_not_double_count_reasoning_inside_output() { + for source in [SourceKind::Codex, SourceKind::CopilotCli] { + let turn = TurnRecord { + v: 1, + source, + session_id: "codex-unknown".to_string(), + session_path: None, + message_id: "m1".to_string(), + turn_index: 0, + request_count: None, + ts: "2026-08-01T00:00:00.000Z".to_string(), + model: "future-unpriced-model".to_string(), + project: None, + project_key: None, + usage: crate::reader::Usage { + input: 400_000, + output: 20_000, + reasoning: 10_000, + cache_read: 13_000, + cache_create_5m: 0, + cache_create_1h: 0, + }, + tool_calls: Vec::new(), + files_touched: None, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: None, + reasoning: None, + }; + + let findings = unpriced_usage_findings(&[turn], &PricingTable::new()); + + assert_eq!( + findings[0].estimated_savings.tokens_per_session, + Some(433_000), + "{source:?}" + ); + } } #[test] diff --git a/crates/relayburn-sdk/src/analyze/hotspots.rs b/crates/relayburn-sdk/src/analyze/hotspots.rs index 86ed8ffe..c048399d 100644 --- a/crates/relayburn-sdk/src/analyze/hotspots.rs +++ b/crates/relayburn-sdk/src/analyze/hotspots.rs @@ -10,8 +10,8 @@ use std::collections::HashMap; use crate::reader::{ - BashParse, ContentKind, ContentRecord, ToolResultEventRecord, TurnRecord, UserTurnBlockKind, - UserTurnRecord, + normalize_tool_name, BashParse, ContentKind, ContentRecord, ToolResultEventRecord, TurnRecord, + UserTurnBlockKind, UserTurnRecord, }; use indexmap::IndexMap; use phf::phf_set; @@ -620,13 +620,13 @@ fn accumulate_output_bytes( } /// Roll up file-touching tool attributions (`Read | Edit | Write | -/// NotebookEdit`) by their target path. Rows missing or with an empty target +/// NotebookEdit`, under any harness's name for them) by their target path. Rows missing or with an empty target /// are skipped. Output is sorted by `total_cost` descending. pub(crate) fn aggregate_by_file(attributions: &[ToolAttribution]) -> Vec { aggregate( attributions, |a| { - if !FILE_TOOLS.contains(a.tool_name.as_str()) { + if !FILE_TOOLS.contains(normalize_tool_name(&a.tool_name)) { return None; } match a.target.as_ref() { diff --git a/crates/relayburn-sdk/src/analyze/hotspots_tests.rs b/crates/relayburn-sdk/src/analyze/hotspots_tests.rs index 6d4f368d..1c23b956 100644 --- a/crates/relayburn-sdk/src/analyze/hotspots_tests.rs +++ b/crates/relayburn-sdk/src/analyze/hotspots_tests.rs @@ -1598,3 +1598,31 @@ fn persistence_rides_only_non_empty_results_the_cache_read_covers() { .collect(); assert_eq!(rides, [(0, 0.0), (1, 100.0), (0, 0.0)]); } + +/// A harness's own name for a file tool (Copilot's `view`, `read`, `edit`, +/// `create`) rolls up by file like the canonical one; a non-file tool does +/// not. +#[test] +fn file_rollup_reads_every_harness_name_for_a_file_tool() { + let named = |tool: &str, path: &str| ToolAttribution { + tool_name: tool.into(), + ..bash_attribution(path, tool, 1.0, 0.0, 0.0, 0) + }; + let attributions = [ + named("view", "a.rs"), + named("read", "a.rs"), + named("edit", "b.rs"), + named("create", "c.rs"), + named("Read", "c.rs"), + named("bash", "ls"), + ]; + let mut files: Vec<(String, u64)> = aggregate_by_file(&attributions) + .into_iter() + .map(|f| (f.path, f.tool_call_count)) + .collect(); + files.sort(); + assert_eq!( + files, + [("a.rs".into(), 2), ("b.rs".into(), 1), ("c.rs".into(), 2)] + ); +} diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests.rs b/crates/relayburn-sdk/src/ingest/ingest_tests.rs index d4c0e0a2..3a62772d 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests.rs @@ -839,6 +839,14 @@ fn a_copilot_session_bills_its_calls_and_its_shutdown_totals_once() { 30_000 + 25_000 + 4_000 ); assert_eq!(sum(|u| u.output), 82 + 65 + 11); + // A shutdown's running totals are no model call. + let calls: Vec<_> = turns.iter().map(|t| t.turn.request_count).collect(); + assert_eq!( + calls.iter().filter(|c| **c == Some(0)).count(), + 4, + "{calls:?}" + ); + assert_eq!(calls.iter().filter(|c| c.is_none()).count(), 7, "{calls:?}"); assert_eq!(sandbox.ingest(&mut ledger).appended_turns, 0); } @@ -863,5 +871,6 @@ fn an_opencode_launcher_stamp_lands_on_the_opencode_session_it_spawned() { assert_eq!(sandbox.ingest(&mut ledger).applied_pending_stamps, 1); } +mod copilot; mod fork_history; mod request_accounting; diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs new file mode 100644 index 00000000..fb36fd87 --- /dev/null +++ b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs @@ -0,0 +1,117 @@ +//! Copilot CLI journals: tool calls are read from the message that made +//! them, and shutdown totals are not model calls. + +use super::*; + +const SID: &str = "7a1c3e5f-2b4d-4f6a-8c0e-1d3f5a7b9c2e"; + +/// A journal: `session.start`, then one prompt and one assistant message +/// per `(tool, call id, path)`, each followed by its tool execution. +fn journal(calls: &[(&str, &str, &str)]) -> String { + let mut lines = Vec::new(); + let mut parent = serde_json::Value::Null; + let mut push = |kind: &str, data: serde_json::Value, lines: &mut Vec| { + let n = lines.len(); + let id = format!("00000000-0000-4000-8000-{n:012}"); + lines.push( + serde_json::json!({ + "type": kind, "data": data, "id": id, "parentId": parent, + "timestamp": format!("2026-09-18T21:30:{:02}.000Z", n), + }) + .to_string(), + ); + parent = serde_json::json!(id); + }; + push( + "session.start", + serde_json::json!({"sessionId": SID, "version": 1, "producer": "copilot-agent", + "copilotVersion": "1.0.86", "context": {"cwd": "/tmp/project"}}), + &mut lines, + ); + for (i, (tool, call, path)) in calls.iter().enumerate() { + push( + "user.message", + serde_json::json!({"content": format!("step {i}"), "turnId": i.to_string()}), + &mut lines, + ); + push( + "assistant.message", + serde_json::json!({"messageId": format!("msg-{i}"), "model": "gpt-5.4", + "content": "on it", "turnId": i.to_string(), "outputTokens": 10, + "toolRequests": [{"toolCallId": call, "name": tool, + "arguments": {"path": path}, "type": "function"}]}), + &mut lines, + ); + push( + "tool.execution_start", + serde_json::json!({"toolCallId": call, "toolName": tool, + "arguments": {"path": path}, "turnId": i.to_string()}), + &mut lines, + ); + push( + "tool.execution_complete", + serde_json::json!({"toolCallId": call, "success": true, + "turnId": i.to_string(), "result": {"content": "ok"}}), + &mut lines, + ); + } + lines.join("\n") + "\n" +} + +fn ingest_journal(sandbox: &Sandbox, text: &str) -> Vec { + let dir = sandbox.home().join(format!(".copilot/session-state/{SID}")); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("events.jsonl"), text).unwrap(); + let mut ledger = sandbox.ledger(); + sandbox.ingest(&mut ledger); + let mut turns: Vec<_> = ledger + .query_turns(&Query::for_session(SID)) + .unwrap() + .into_iter() + .map(|t| t.turn) + .collect(); + turns.sort_by_key(|t| t.turn_index); + turns +} + +/// Copilot numbers some tool calls per model session (`bash_14`), so two +/// turns can carry one id. A turn's call is read only from the evidence its +/// own message made; relayhistory keeps one call per id, so the earlier +/// turn shows none rather than the later one's. +#[test] +fn a_reused_tool_call_id_is_read_from_its_own_message() { + let sandbox = Sandbox::new(); + let turns = ingest_journal( + &sandbox, + &journal(&[("view", "call_1", "a.rs"), ("view", "call_1", "b.rs")]), + ); + let calls: Vec>> = turns + .iter() + .map(|t| t.tool_calls.iter().map(|c| c.target.clone()).collect()) + .collect(); + assert_eq!(calls, [vec![], vec![Some("b.rs".to_string())]]); + let touched: Vec<_> = turns.iter().map(|t| t.files_touched.clone()).collect(); + assert_eq!(touched, [None, Some(vec!["b.rs".to_string()])]); +} + +#[test] +fn every_copilot_file_tool_name_touches_its_file() { + let sandbox = Sandbox::new(); + let tools = ["view", "read", "edit", "create", "write"]; + let calls: Vec<(&str, String, String)> = tools + .iter() + .enumerate() + .map(|(i, tool)| (*tool, format!("call_{i}"), format!("{tool}.txt"))) + .collect(); + let calls: Vec<(&str, &str, &str)> = calls + .iter() + .map(|(t, c, p)| (*t, c.as_str(), p.as_str())) + .collect(); + let turns = ingest_journal(&sandbox, &journal(&calls)); + let touched: Vec<_> = turns.iter().map(|t| t.files_touched.clone()).collect(); + let want: Vec<_> = tools + .iter() + .map(|tool| Some(vec![format!("{tool}.txt")])) + .collect(); + assert_eq!(touched, want); +} diff --git a/crates/relayburn-sdk/src/session_metrics.rs b/crates/relayburn-sdk/src/session_metrics.rs index 5bf5ce14..2bc0b535 100644 --- a/crates/relayburn-sdk/src/session_metrics.rs +++ b/crates/relayburn-sdk/src/session_metrics.rs @@ -64,9 +64,9 @@ pub(crate) fn turn_total_tokens(turn: &TurnRecord) -> u64 { usage .input .saturating_add(usage.output) - // Codex includes reasoning in output; the other supported sources - // expose a separate, billable reasoning bucket. - .saturating_add(if matches!(turn.source, crate::reader::SourceKind::Codex) { + // A source that counts reasoning inside output has no separate + // reasoning bucket to add. + .saturating_add(if crate::analyze::cost::reasoning_in_output(turn.source) { 0 } else { usage.reasoning @@ -243,6 +243,49 @@ mod tests { assert_eq!(report.models[0].model, "claude-sonnet-4-6"); } + /// Codex and Copilot CLI count reasoning inside output; every other + /// source reports it as a bucket of its own. + #[test] + fn total_tokens_count_reasoning_once() { + use crate::reader::SourceKind; + let turn = |source| TurnRecord { + v: 1, + source, + session_id: "s".to_string(), + session_path: None, + message_id: "m".to_string(), + turn_index: 0, + request_count: None, + ts: "2026-08-01T00:00:00.000Z".to_string(), + model: "m".to_string(), + project: None, + project_key: None, + usage: crate::reader::Usage { + input: 100, + output: 20, + reasoning: 5, + cache_read: 1_000, + ..Default::default() + }, + tool_calls: Vec::new(), + files_touched: None, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: None, + reasoning: None, + }; + for (source, total) in [ + (SourceKind::Codex, 1_120), + (SourceKind::CopilotCli, 1_120), + (SourceKind::ClaudeCode, 1_125), + ] { + assert_eq!(turn_total_tokens(&turn(source)), total, "{source:?}"); + } + } + #[test] fn measures_one_codex_transcript_with_reasoning_bucket() { let report = measure_session(MeasureSessionOptions { diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs index 3b1a90cd..99a7abcd 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -40,29 +40,43 @@ pub(super) fn refine(ctx: &Context<'_>, records: &mut SessionRecords) { /// Copilot's tool vocabulary for targets, edit hashes and touched files; /// subagent turns flagged; then the activity classification over them. fn refine_turn(ctx: &Context<'_>, turn: &mut TurnRecord) { + let message_ids: Vec<&str> = ctx + .units + .iter() + .filter(|u| u.id == turn.message_id) + .flat_map(|u| &u.request.message_ids) + .map(String::as_str) + .collect(); let mut files = BTreeSet::new(); for call in &mut turn.tool_calls { + // Copilot reuses a tool call id across turns: a call is the one its + // own messages made. let input = ctx .ev .tool_calls .iter() - .find(|c| c.tool_use_id == call.id) + .find(|c| { + c.tool_use_id == call.id + && c.message_id + .as_deref() + .is_some_and(|m| message_ids.contains(&m)) + }) .and_then(|c| c.args.clone()) .unwrap_or(Value::Null); apply_tool(call, &input); - if matches!(call.name.as_str(), "view" | "edit" | "create") { + if matches!( + call.name.as_str(), + "view" | "read" | "edit" | "create" | "write" + ) { if let Some(target) = &call.target { files.insert(target.clone()); } } } turn.files_touched = (!files.is_empty()).then(|| files.into_iter().collect()); - let messages: Vec<&Message> = ctx - .units + let messages: Vec<&Message> = message_ids .iter() - .filter(|u| u.id == turn.message_id) - .flat_map(|u| &u.request.message_ids) - .filter_map(|id| ctx.by_id.get(id.as_str()).copied()) + .filter_map(|id| ctx.by_id.get(id).copied()) .collect(); if messages.iter().any(|m| m.is_sidechain == Some(true)) { turn.subagent = Some(Subagent { @@ -236,7 +250,8 @@ fn aggregate_turn( }, )), reasoning: None, - request_count: None, + // Running totals, not a model call. + request_count: Some(0), } } diff --git a/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json b/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json index 76aa1eea..9e650a11 100644 --- a/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json +++ b/tests/fixtures/sourcing-snapshots/copilot-cli-usage.json @@ -278,6 +278,7 @@ }, "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:claude-sonnet-4.6", "turnIndex": 6, + "requestCount": 0, "ts": "2026-09-18T21:30:31.147Z", "model": "claude-sonnet-4.6", "project": "/tmp/project", @@ -312,6 +313,7 @@ }, "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:gpt-5.4", "turnIndex": 7, + "requestCount": 0, "ts": "2026-09-18T21:30:31.147Z", "model": "gpt-5.4", "project": "/tmp/project", @@ -346,6 +348,7 @@ }, "messageId": "fc9ac928-bb14-5990-9d7f-43fe18ed7ae6:gpt-5.4-mini", "turnIndex": 8, + "requestCount": 0, "ts": "2026-09-18T21:30:31.147Z", "model": "gpt-5.4-mini", "project": "/tmp/project", @@ -417,6 +420,7 @@ }, "messageId": "3bd66697-92d9-568d-b7c4-dede0a476f3c:gpt-5.4", "turnIndex": 10, + "requestCount": 0, "ts": "2026-09-18T21:30:37.369Z", "model": "gpt-5.4", "project": "/tmp/project", diff --git a/tests/fixtures/sourcing-snapshots/copilot-official-importer.json b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json index 3bdc6618..b2c4aa29 100644 --- a/tests/fixtures/sourcing-snapshots/copilot-official-importer.json +++ b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json @@ -40,6 +40,9 @@ "argsHash": "6b3640bd99e591a8" } ], + "filesTouched": [ + "greeting.txt" + ], "activity": "review", "retries": 0, "hasEdits": false From e06ebca43bec151443c5a18720793fed657dbfd2 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 02:10:21 -0700 Subject: [PATCH 05/10] Pin the unpriced-usage turn count Co-Authored-By: Claude Opus 5.5 --- crates/relayburn-sdk/src/analyze/findings.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/relayburn-sdk/src/analyze/findings.rs b/crates/relayburn-sdk/src/analyze/findings.rs index 7999cab8..45e96594 100644 --- a/crates/relayburn-sdk/src/analyze/findings.rs +++ b/crates/relayburn-sdk/src/analyze/findings.rs @@ -1251,6 +1251,10 @@ mod tests { Some(433_000), "{source:?}" ); + assert_eq!( + findings[0].title, "Unpriced usage: 433,000 tokens across 1 turn", + "{source:?}" + ); } } From 2fb164e82daa6b9fe14e585a7a81368b2ed1f562 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 02:14:55 -0700 Subject: [PATCH 06/10] Bill a shutdown only for output its message turns did not; reject malformed totals - A shutdown's aggregate leaves out exactly the output its interval's message turns billed, including a turn billed under no model. - `copilot_totals` rejects a reported `reasoningTokens` that is not a counter and cache counters whose sum overflows; an absent reasoning count is zero. - The changelog entry for Copilot CLI is one impact line; the README keeps the accounting. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- .../src/ingest/ingest_tests/copilot.rs | 20 +++++++++++ .../src/source/relayhistory/copilot.rs | 34 +++++++++++-------- crates/relayburn-sdk/src/source/usage.rs | 30 ++++++++++++++-- 4 files changed, 69 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed671011..310bf617 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Cross-package release notes for relayburn. Package changelogs contain package-le ### Added -- GitHub Copilot CLI sessions (`~/.copilot/session-state/`, or `$COPILOT_HOME`) are ingested as `copilot-cli` turns, retroactively and with no opt-in: one turn per model call with its tool calls, activity and per-message output tokens, plus one `per-session-aggregate` turn per `session.shutdown` and model billing the input, cache-read, cache-write and remaining output tokens Copilot's running totals grew by. Subagent turns are flagged sidechain; turns price under provider `github-copilot`, with reasoning counted inside output. +- GitHub Copilot CLI sessions (`~/.copilot/session-state/`, or `$COPILOT_HOME`) are ingested and billed, retroactively and with no setup. - Claude Code subagent spend is billed: every turn of a subagent transcript, nested subagents included, is a turn of the session that spawned it. Ledger totals for sessions that delegated work go up by exactly that spend. - OpenCode sessions in the SQLite store (`opencode.db`, current OpenCode releases) are ingested, as well as the `storage/` JSON tree. - A Codex subagent thread is ingested as a session of its own. diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs index fb36fd87..1293032f 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs @@ -115,3 +115,23 @@ fn every_copilot_file_tool_name_touches_its_file() { .collect(); assert_eq!(touched, want); } + +/// The output a message turn bills is what a shutdown's totals leave out, +/// whatever model the turn resolved to. +#[test] +fn a_shutdown_bills_only_the_output_its_message_turns_did_not() { + let sandbox = Sandbox::new(); + let mut text = journal(&[("view", "call_1", "a.rs")]).replace(r#""model":"gpt-5.4","#, ""); + text.push_str( + &serde_json::json!({"type": "session.shutdown", "id": "00000000-0000-4000-8000-0000000000ff", + "parentId": null, "timestamp": "2026-09-18T21:31:00.000Z", + "data": {"modelMetrics": {"gpt-5.4": {"usage": {"inputTokens": 100, + "outputTokens": 10, "cacheReadTokens": 0, "cacheWriteTokens": 0}}}}}) + .to_string(), + ); + text.push('\n'); + let turns = ingest_journal(&sandbox, &text); + let output: u64 = turns.iter().map(|t| t.usage.output).sum(); + let models: Vec<&str> = turns.iter().map(|t| t.model.as_str()).collect(); + assert_eq!(output, 10, "{models:?}"); +} diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs index 99a7abcd..e50a8575 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -19,16 +19,17 @@ use crate::reader::resolve_project; use crate::reader::types::{ Coverage, Fidelity, Subagent, ToolCall, TurnRecord, Usage, UsageGranularity, }; -use crate::source::usage::{copilot_totals, usage_from_raw}; +use crate::source::usage::copilot_totals; use crate::source::SessionRecords; -use crate::util::time::format_iso_ms; +use crate::util::time::{format_iso_ms, parse_iso_ms}; /// Derive what the shared mapping cannot for copilot sessions. pub(super) fn refine(ctx: &Context<'_>, records: &mut SessionRecords) { for turn in &mut records.turns { refine_turn(ctx, turn); } - records.turns.extend(shutdown_turns(ctx)); + let aggregates = shutdown_turns(ctx, &records.turns); + records.turns.extend(aggregates); // A shutdown's turn sits where the journal wrote it, so a session that // resumes appends turns after it and earlier indices never move. records.turns.sort_by(|a, b| a.ts.cmp(&b.ts)); @@ -144,17 +145,16 @@ fn shutdowns<'a>(ctx: &Context<'a>) -> Vec> { out } -/// Output the session's messages billed per model, in `(after, through]`. -/// Only assistant messages carry Copilot's `outputTokens`. -fn message_output(ctx: &Context<'_>, after: i64, through: i64) -> HashMap { +/// The output the session's message turns bill per model, in `(after, +/// through]`: exactly what a shutdown's totals must not bill again. +fn message_output(turns: &[TurnRecord], after: i64, through: i64) -> HashMap { let mut out = HashMap::new(); - for message in &ctx.ev.messages { - if message.ts_ms <= after || message.ts_ms > through { + for turn in turns { + let Some(ts) = parse_iso_ms(&turn.ts) else { continue; - } - let (usage, coverage) = usage_from_raw(ctx.source, message.raw_usage()); - if let (true, Some(model)) = (coverage.has_output_tokens, message.model.as_ref()) { - *out.entry(model.clone()).or_default() += usage.output; + }; + if ts > after && ts <= through { + *out.entry(turn.model.clone()).or_default() += turn.usage.output; } } out @@ -186,17 +186,23 @@ fn growth(now: &Usage, before: Option<&Usage>) -> Usage { } } -fn shutdown_turns(ctx: &Context<'_>) -> Vec { +fn shutdown_turns(ctx: &Context<'_>, message_turns: &[TurnRecord]) -> Vec { let mut turns = Vec::new(); let mut previous: HashMap = HashMap::new(); let mut after = i64::MIN; for shutdown in shutdowns(ctx) { - let billed = message_output(ctx, after, shutdown.ts_ms); + let billed = message_output(message_turns, after, shutdown.ts_ms); + // Output a turn billed under no model is in some model's totals: + // each model in turn leaves out what of it remains. + let mut unattributed = billed.get("").copied().unwrap_or(0); for (model, totals) in &shutdown.totals { let mut usage = growth(totals, previous.get(model)); usage.output = usage .output .saturating_sub(billed.get(model).copied().unwrap_or(0)); + let covered = usage.output.min(unattributed); + usage.output -= covered; + unattributed -= covered; if usage != Usage::default() { turns.push(aggregate_turn(ctx, &shutdown, model, usage)); } diff --git a/crates/relayburn-sdk/src/source/usage.rs b/crates/relayburn-sdk/src/source/usage.rs index 6afa7691..1f87ebd8 100644 --- a/crates/relayburn-sdk/src/source/usage.rs +++ b/crates/relayburn-sdk/src/source/usage.rs @@ -118,10 +118,15 @@ fn copilot_message(obj: &Map) -> (Usage, Coverage) { pub(crate) fn copilot_totals(usage: &Value) -> Option { let counter = |key: &str| usage.get(key)?.as_u64(); let (read, write) = (counter("cacheReadTokens")?, counter("cacheWriteTokens")?); + // Reasoning is optional, but one that is reported must be a counter. + let reasoning = match usage.get("reasoningTokens") { + None => 0, + Some(value) => value.as_u64()?, + }; Some(Usage { - input: counter("inputTokens")?.checked_sub(read + write)?, + input: counter("inputTokens")?.checked_sub(read.checked_add(write)?)?, output: counter("outputTokens")?, - reasoning: counter("reasoningTokens").unwrap_or(0), + reasoning, cache_read: read, cache_create_5m: write, cache_create_1h: 0, @@ -187,4 +192,25 @@ mod tests { ); assert!(copilot_totals(&serde_json::json!({"inputTokens": 1})).is_none()); } + + /// Reasoning may be absent (zero), but a reported one that is no + /// counter, or cache counters whose sum overflows, reject the totals. + #[test] + fn copilot_totals_reject_malformed_counters() { + let totals = |extra: serde_json::Value| { + let mut usage = serde_json::json!({"inputTokens": 10, "outputTokens": 1, + "cacheReadTokens": 2, "cacheWriteTokens": 3}); + usage + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + copilot_totals(&usage) + }; + assert_eq!(totals(serde_json::json!({})).unwrap().reasoning, 0); + assert!(totals(serde_json::json!({"reasoningTokens": "4"})).is_none()); + assert!(totals(serde_json::json!({"reasoningTokens": -1})).is_none()); + assert!(totals(serde_json::json!({"inputTokens": u64::MAX, + "cacheReadTokens": u64::MAX, "cacheWriteTokens": 1})) + .is_none()); + } } From 45bcc285a3b4a9c987395f1fcf3c7f5b3358639d Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 02:27:50 -0700 Subject: [PATCH 07/10] Bill a model-less message's output under the model its shutdown names A message turn that names no model takes the model whose growth over the shutdown interval still holds its output, so the turn is priced and the shutdown leaves those tokens out once: every token has one priced home. Co-Authored-By: Claude Opus 5.5 --- .../src/ingest/ingest_tests/copilot.rs | 27 +++++++--- .../src/source/relayhistory/copilot.rs | 50 ++++++++++++++----- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs index 1293032f..2644f6a0 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs @@ -116,10 +116,11 @@ fn every_copilot_file_tool_name_touches_its_file() { assert_eq!(touched, want); } -/// The output a message turn bills is what a shutdown's totals leave out, -/// whatever model the turn resolved to. +/// A message that names no model bills its output under the model whose +/// shutdown totals hold it, and that shutdown leaves it out: every token +/// is priced once. #[test] -fn a_shutdown_bills_only_the_output_its_message_turns_did_not() { +fn a_message_naming_no_model_bills_under_the_model_its_shutdown_names() { let sandbox = Sandbox::new(); let mut text = journal(&[("view", "call_1", "a.rs")]).replace(r#""model":"gpt-5.4","#, ""); text.push_str( @@ -131,7 +132,21 @@ fn a_shutdown_bills_only_the_output_its_message_turns_did_not() { ); text.push('\n'); let turns = ingest_journal(&sandbox, &text); - let output: u64 = turns.iter().map(|t| t.usage.output).sum(); - let models: Vec<&str> = turns.iter().map(|t| t.model.as_str()).collect(); - assert_eq!(output, 10, "{models:?}"); + let billed: Vec<(&str, u64, u64)> = turns + .iter() + .map(|t| (t.model.as_str(), t.usage.input, t.usage.output)) + .collect(); + assert_eq!(billed, [("gpt-5.4", 0, 10), ("gpt-5.4", 100, 0)]); + let pricing = crate::analyze::pricing::load_builtin_pricing(); + let rate = crate::analyze::cost::lookup_model_rate("gpt-5.4", &pricing).unwrap(); + let cost: f64 = turns + .iter() + .map(|t| { + crate::analyze::cost::cost_for_turn(t, &pricing) + .expect("priced") + .total + }) + .sum(); + let want = (100.0 * rate.input + 10.0 * rate.output) / 1_000_000.0; + assert!((cost - want).abs() < 1e-12, "{cost} vs {want}"); } diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs index e50a8575..28bb58a4 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -28,7 +28,7 @@ pub(super) fn refine(ctx: &Context<'_>, records: &mut SessionRecords) { for turn in &mut records.turns { refine_turn(ctx, turn); } - let aggregates = shutdown_turns(ctx, &records.turns); + let aggregates = shutdown_turns(ctx, &mut records.turns); records.turns.extend(aggregates); // A shutdown's turn sits where the journal wrote it, so a session that // resumes appends turns after it and earlier indices never move. @@ -186,26 +186,27 @@ fn growth(now: &Usage, before: Option<&Usage>) -> Usage { } } -fn shutdown_turns(ctx: &Context<'_>, message_turns: &[TurnRecord]) -> Vec { +fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec { let mut turns = Vec::new(); let mut previous: HashMap = HashMap::new(); let mut after = i64::MIN; for shutdown in shutdowns(ctx) { + let mut growths: Vec<(&str, Usage)> = shutdown + .totals + .iter() + .map(|(model, totals)| (model.as_str(), growth(totals, previous.get(model)))) + .collect(); + assign_models(message_turns, after, shutdown.ts_ms, &growths); let billed = message_output(message_turns, after, shutdown.ts_ms); - // Output a turn billed under no model is in some model's totals: - // each model in turn leaves out what of it remains. - let mut unattributed = billed.get("").copied().unwrap_or(0); - for (model, totals) in &shutdown.totals { - let mut usage = growth(totals, previous.get(model)); + for (model, usage) in &mut growths { usage.output = usage .output - .saturating_sub(billed.get(model).copied().unwrap_or(0)); - let covered = usage.output.min(unattributed); - usage.output -= covered; - unattributed -= covered; - if usage != Usage::default() { - turns.push(aggregate_turn(ctx, &shutdown, model, usage)); + .saturating_sub(billed.get(*model).copied().unwrap_or(0)); + if *usage != Usage::default() { + turns.push(aggregate_turn(ctx, &shutdown, model, usage.clone())); } + } + for (model, totals) in &shutdown.totals { previous.insert(model.to_string(), totals.clone()); } after = shutdown.ts_ms; @@ -213,6 +214,29 @@ fn shutdown_turns(ctx: &Context<'_>, message_turns: &[TurnRecord]) -> Vec = growths + .iter() + .map(|(model, usage)| { + let named = billed.get(*model).copied().unwrap_or(0); + (*model, usage.output.saturating_sub(named)) + }) + .collect(); + for turn in turns.iter_mut().filter(|t| t.model.is_empty()) { + if !parse_iso_ms(&turn.ts).is_some_and(|ts| ts > after && ts <= through) { + continue; + } + if let Some((model, left)) = room.iter_mut().find(|(_, left)| *left >= turn.usage.output) { + turn.model = model.to_string(); + *left -= turn.usage.output; + } + } +} + fn aggregate_turn( ctx: &Context<'_>, shutdown: &Shutdown<'_>, From 7fa2e143a95b1032e67a0185d5271ed7811665dd Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 02:36:50 -0700 Subject: [PATCH 08/10] Settle each shutdown interval so every output token is billed once A shutdown interval's message turns and its per-model growth settle by one rule: under a single model, a message that names none is that model's; under several, it keeps its tools and content but bills no output; each model's aggregate is its growth less what its own message turns billed. Per model, the interval bills exactly the growth, and no output sits on a turn without a model. A seeded property test checks that over 2,000 generated intervals. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- .../src/source/relayhistory/copilot.rs | 87 ++++++----- .../relayhistory/copilot/settle_tests.rs | 135 ++++++++++++++++++ 3 files changed, 180 insertions(+), 47 deletions(-) create mode 100644 crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs diff --git a/README.md b/README.md index 6dcc2252..8d3d0e54 100644 --- a/README.md +++ b/README.md @@ -352,7 +352,10 @@ records output tokens per model call but input and cache tokens only as per-model running totals when a session shuts down, so each `copilot-cli` turn carries its output tokens and each shutdown adds one `per-session-aggregate` turn per model with the input, cache, and remaining output it accounts for. A -session that has not shut down yet shows output tokens only. +model call that names no model is billed under the shutdown's model when the +shutdown names one, and otherwise leaves its output to the per-model +aggregates, so every token is billed once under a known model. A session that +has not shut down yet shows output tokens only. | Option | What it does | |---|---| diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs index 28bb58a4..fc0db8d9 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -145,21 +145,6 @@ fn shutdowns<'a>(ctx: &Context<'a>) -> Vec> { out } -/// The output the session's message turns bill per model, in `(after, -/// through]`: exactly what a shutdown's totals must not bill again. -fn message_output(turns: &[TurnRecord], after: i64, through: i64) -> HashMap { - let mut out = HashMap::new(); - for turn in turns { - let Some(ts) = parse_iso_ms(&turn.ts) else { - continue; - }; - if ts > after && ts <= through { - *out.entry(turn.model.clone()).or_default() += turn.usage.output; - } - } - out -} - /// What a running total grew by. A total lower than the previous one /// restarted, and counts whole. fn growth(now: &Usage, before: Option<&Usage>) -> Usage { @@ -191,20 +176,17 @@ fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec = HashMap::new(); let mut after = i64::MIN; for shutdown in shutdowns(ctx) { - let mut growths: Vec<(&str, Usage)> = shutdown + let growths = shutdown .totals .iter() - .map(|(model, totals)| (model.as_str(), growth(totals, previous.get(model)))) + .map(|(model, totals)| (model.clone(), growth(totals, previous.get(model)))) .collect(); - assign_models(message_turns, after, shutdown.ts_ms, &growths); - let billed = message_output(message_turns, after, shutdown.ts_ms); - for (model, usage) in &mut growths { - usage.output = usage - .output - .saturating_sub(billed.get(*model).copied().unwrap_or(0)); - if *usage != Usage::default() { - turns.push(aggregate_turn(ctx, &shutdown, model, usage.clone())); - } + let mut messages: Vec<&mut TurnRecord> = message_turns + .iter_mut() + .filter(|t| parse_iso_ms(&t.ts).is_some_and(|ts| ts > after && ts <= shutdown.ts_ms)) + .collect(); + for (model, usage) in settle(&mut messages, growths) { + turns.push(aggregate_turn(ctx, &shutdown, &model, usage)); } for (model, totals) in &shutdown.totals { previous.insert(model.to_string(), totals.clone()); @@ -214,27 +196,37 @@ fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec = growths - .iter() - .map(|(model, usage)| { - let named = billed.get(*model).copied().unwrap_or(0); - (*model, usage.output.saturating_sub(named)) - }) - .collect(); - for turn in turns.iter_mut().filter(|t| t.model.is_empty()) { - if !parse_iso_ms(&turn.ts).is_some_and(|ts| ts > after && ts <= through) { - continue; - } - if let Some((model, left)) = room.iter_mut().find(|(_, left)| *left >= turn.usage.output) { - turn.model = model.to_string(); - *left -= turn.usage.output; - } +/// Settle one shutdown interval: the message turns written in it, and what +/// the shutdown's running totals grew by per model. Every output token +/// lands on exactly one turn under a model the shutdown names: +/// +/// - with one model in the totals, a message turn that names no model is +/// that model's; +/// - with several, such a turn keeps its tools and content but bills no +/// output: which model produced it is unknown, so the aggregates carry it; +/// - each model's aggregate is its growth less the output its own message +/// turns billed. +/// +/// Returns the aggregates that bill anything. +fn settle(messages: &mut [&mut TurnRecord], growths: Vec<(String, Usage)>) -> Vec<(String, Usage)> { + let unnamed = messages.iter_mut().filter(|t| t.model.is_empty()); + match growths.as_slice() { + [] => {} + [(only, _)] => unnamed.for_each(|t| t.model = only.clone()), + _ => unnamed.for_each(|t| t.usage.output = 0), } + growths + .into_iter() + .filter_map(|(model, mut usage)| { + let billed: u64 = messages + .iter() + .filter(|t| t.model == model) + .map(|t| t.usage.output) + .sum(); + usage.output = usage.output.saturating_sub(billed); + (usage != Usage::default()).then_some((model, usage)) + }) + .collect() } fn aggregate_turn( @@ -285,6 +277,9 @@ fn aggregate_turn( } } +#[cfg(test)] +mod settle_tests; + #[cfg(test)] mod tests { use serde_json::json; diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs new file mode 100644 index 00000000..633e80fe --- /dev/null +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs @@ -0,0 +1,135 @@ +//! A shutdown interval settles so that every output token sits on exactly +//! one turn, under a model the shutdown names. + +use std::collections::HashMap; + +use super::settle; +use crate::reader::types::{TurnRecord, Usage}; + +fn message(model: &str, output: u64) -> TurnRecord { + let mut turn: TurnRecord = serde_json::from_value(serde_json::json!({ + "v": 1, "source": "copilot-cli", "sessionId": "s", "messageId": "m", + "turnIndex": 0, "ts": "2026-09-18T21:30:00.000Z", "model": model, + "usage": {"input": 0, "output": 0, "reasoning": 0, "cacheRead": 0, + "cacheCreate5m": 0, "cacheCreate1h": 0}, + "toolCalls": [], + })) + .unwrap(); + turn.usage.output = output; + turn +} + +fn grew(output: u64) -> Usage { + Usage { + input: 1, + output, + ..Usage::default() + } +} + +/// Output billed per model across the message turns and the aggregates. +fn billed(messages: &[TurnRecord], aggregates: &[(String, Usage)]) -> HashMap { + let mut out = HashMap::new(); + for turn in messages { + *out.entry(turn.model.clone()).or_default() += turn.usage.output; + } + for (model, usage) in aggregates { + *out.entry(model.clone()).or_default() += usage.output; + } + out.retain(|_, n| *n > 0); + out +} + +fn settled( + mut messages: Vec, + growths: &[(&str, u64)], +) -> (Vec, Vec<(String, Usage)>) { + let growths = growths + .iter() + .map(|(model, output)| (model.to_string(), grew(*output))) + .collect(); + let mut refs: Vec<&mut TurnRecord> = messages.iter_mut().collect(); + let aggregates = settle(&mut refs, growths); + (messages, aggregates) +} + +/// Two models with 6 and 4 output tokens left, and unnamed messages of 4 +/// then 6: neither message can be placed by size alone, so the aggregates +/// carry all 10 and the messages bill none. +#[test] +fn unnamed_messages_under_several_models_leave_their_output_to_the_aggregates() { + let (messages, aggregates) = settled( + vec![message("", 4), message("", 6)], + &[("gpt-5.4", 6), ("kimi-k3", 4)], + ); + let want = HashMap::from([("gpt-5.4".to_string(), 6), ("kimi-k3".to_string(), 4)]); + assert_eq!(billed(&messages, &aggregates), want); + assert!(messages.iter().all(|t| t.usage.output == 0)); +} + +#[test] +fn an_unnamed_message_under_one_model_is_that_models() { + let (messages, aggregates) = settled(vec![message("", 4)], &[("gpt-5.4", 10)]); + assert_eq!(messages[0].model, "gpt-5.4"); + assert_eq!(messages[0].usage.output, 4); + assert_eq!(aggregates[0].1.output, 6); +} + +#[test] +fn an_interval_no_shutdown_names_a_model_for_keeps_its_messages() { + let (messages, aggregates) = settled(vec![message("", 4)], &[]); + assert_eq!( + (messages[0].model.as_str(), messages[0].usage.output), + ("", 4) + ); + assert!(aggregates.is_empty()); +} + +/// xorshift: a fixed, dependency-free sequence of cases. +struct Cases(u64); + +impl Cases { + fn below(&mut self, n: u64) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % n + } +} + +/// For any interval whose message turns bill no more than the shutdown's +/// growth: per model, billed output equals the growth; no output is left +/// on a turn without a model; no aggregate exceeds its growth. +#[test] +fn every_interval_bills_each_models_growth_exactly_once() { + let models = ["claude-sonnet-4.6", "gpt-5.4", "kimi-k3"]; + let mut cases = Cases(0x9e37_79b9_7f4a_7c15); + for _ in 0..2_000 { + let count = 1 + cases.below(models.len() as u64) as usize; + let named = &models[..count]; + let mut left: Vec = named.iter().map(|_| cases.below(500)).collect(); + let growth: Vec<(&str, u64)> = named.iter().copied().zip(left.clone()).collect(); + let mut messages = Vec::new(); + for _ in 0..cases.below(8) { + let pick = cases.below(count as u64) as usize; + let output = cases.below(left[pick] + 1); + left[pick] -= output; + let model = if cases.below(3) == 0 { "" } else { named[pick] }; + messages.push(message(model, output)); + } + let (messages, aggregates) = settled(messages, &growth); + let want: HashMap = growth + .iter() + .filter(|(_, n)| *n > 0) + .map(|(m, n)| (m.to_string(), *n)) + .collect(); + assert_eq!(billed(&messages, &aggregates), want, "{growth:?}"); + assert!(messages + .iter() + .all(|t| !t.model.is_empty() || t.usage.output == 0)); + for (model, usage) in &aggregates { + let grown = growth.iter().find(|(m, _)| m == model).unwrap().1; + assert!(usage.output <= grown, "{model}"); + } + } +} From 42ea481ff770b6f89ebcfabffac8ea98524a3765 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 02:52:06 -0700 Subject: [PATCH 09/10] Bill a model-less Copilot message nothing; its shutdown carries the output A message turn that names no model bills a measured zero (priced $0, not unpriced) from the moment it is ingested, and the shutdown's per-model aggregates carry its output. A message turn therefore bills the same before and after its session shuts down, so a ledger that ingested the session mid-run needs no rewrite: per model, the interval bills exactly the shutdown's growth. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- .../src/ingest/ingest_tests/copilot.rs | 90 +++++++++++++------ .../src/source/relayhistory/copilot.rs | 54 ++++++----- .../relayhistory/copilot/settle_tests.rs | 31 ++++--- .../copilot-official-importer.json | 12 +-- 5 files changed, 122 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 8d3d0e54..eb98c41b 100644 --- a/README.md +++ b/README.md @@ -352,10 +352,10 @@ records output tokens per model call but input and cache tokens only as per-model running totals when a session shuts down, so each `copilot-cli` turn carries its output tokens and each shutdown adds one `per-session-aggregate` turn per model with the input, cache, and remaining output it accounts for. A -model call that names no model is billed under the shutdown's model when the -shutdown names one, and otherwise leaves its output to the per-model -aggregates, so every token is billed once under a known model. A session that -has not shut down yet shows output tokens only. +model call that names no model bills nothing itself ($0); the shutdown's +per-model aggregates carry its output, so every token is billed once under a +known model. A session that has not shut down yet shows the output tokens of +the model calls that name their model. | Option | What it does | |---|---| diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs index 2644f6a0..4b1598b0 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs @@ -116,37 +116,69 @@ fn every_copilot_file_tool_name_touches_its_file() { assert_eq!(touched, want); } -/// A message that names no model bills its output under the model whose -/// shutdown totals hold it, and that shutdown leaves it out: every token -/// is priced once. -#[test] -fn a_message_naming_no_model_bills_under_the_model_its_shutdown_names() { - let sandbox = Sandbox::new(); - let mut text = journal(&[("view", "call_1", "a.rs")]).replace(r#""model":"gpt-5.4","#, ""); - text.push_str( - &serde_json::json!({"type": "session.shutdown", "id": "00000000-0000-4000-8000-0000000000ff", - "parentId": null, "timestamp": "2026-09-18T21:31:00.000Z", - "data": {"modelMetrics": {"gpt-5.4": {"usage": {"inputTokens": 100, - "outputTokens": 10, "cacheReadTokens": 0, "cacheWriteTokens": 0}}}}}) - .to_string(), - ); - text.push('\n'); - let turns = ingest_journal(&sandbox, &text); - let billed: Vec<(&str, u64, u64)> = turns - .iter() - .map(|t| (t.model.as_str(), t.usage.input, t.usage.output)) - .collect(); - assert_eq!(billed, [("gpt-5.4", 0, 10), ("gpt-5.4", 100, 0)]); +/// The journal of [`journal`] with no model on its messages, and the +/// `session.shutdown` that closes it: gpt-5.4 grew 100 input, 10 output. +fn unnamed_journal() -> (String, String) { + let text = journal(&[("view", "call_1", "a.rs")]).replace(r#""model":"gpt-5.4","#, ""); + let shutdown = serde_json::json!({"type": "session.shutdown", + "id": "00000000-0000-4000-8000-0000000000ff", "parentId": null, + "timestamp": "2026-09-18T21:31:00.000Z", + "data": {"modelMetrics": {"gpt-5.4": {"usage": {"inputTokens": 100, + "outputTokens": 10, "cacheReadTokens": 0, "cacheWriteTokens": 0}}}}}); + (text, format!("{shutdown}\n")) +} + +/// Model, input, output and dollar cost per turn, in turn order. +fn priced(turns: &[crate::reader::TurnRecord]) -> Vec<(String, u64, u64, f64)> { let pricing = crate::analyze::pricing::load_builtin_pricing(); - let rate = crate::analyze::cost::lookup_model_rate("gpt-5.4", &pricing).unwrap(); - let cost: f64 = turns + turns .iter() .map(|t| { - crate::analyze::cost::cost_for_turn(t, &pricing) - .expect("priced") - .total + let cost = crate::analyze::cost::cost_for_turn(t, &pricing).expect("priced"); + (t.model.clone(), t.usage.input, t.usage.output, cost.total) }) - .sum(); - let want = (100.0 * rate.input + 10.0 * rate.output) / 1_000_000.0; - assert!((cost - want).abs() < 1e-12, "{cost} vs {want}"); + .collect() +} + +/// A message that names no model bills nothing, priced at \$0; its output +/// is on the shutdown's aggregate, under the model the shutdown names. +#[test] +fn a_message_naming_no_model_leaves_its_output_to_the_shutdown() { + let sandbox = Sandbox::new(); + let (text, shutdown) = unnamed_journal(); + let turns = ingest_journal(&sandbox, &(text + &shutdown)); + let pricing = crate::analyze::pricing::load_builtin_pricing(); + let rate = crate::analyze::cost::lookup_model_rate("gpt-5.4", &pricing).unwrap(); + let aggregate = (100.0 * rate.input + 10.0 * rate.output) / 1_000_000.0; + assert_eq!( + priced(&turns), + [ + (String::new(), 0, 0, 0.0), + ("gpt-5.4".to_string(), 100, 10, aggregate) + ] + ); + assert!(crate::analyze::findings::unpriced_usage_findings(&turns, &pricing).is_empty()); +} + +/// Ingested while running and again after it shuts down, a session bills +/// exactly the shutdown's totals: the message turn the ledger already holds +/// bills the same either way. +#[test] +fn a_session_ingested_before_and_after_its_shutdown_bills_its_totals_once() { + let sandbox = Sandbox::new(); + let (text, shutdown) = unnamed_journal(); + let running = ingest_journal(&sandbox, &text); + assert_eq!(priced(&running), [(String::new(), 0, 0, 0.0)]); + + let settled = ingest_journal(&sandbox, &(text + &shutdown)); + let billed: Vec<_> = priced(&settled) + .into_iter() + .map(|(model, input, output, _)| (model, input, output)) + .collect(); + assert_eq!( + billed, + [(String::new(), 0, 0), ("gpt-5.4".to_string(), 100, 10)] + ); + let mut ledger = sandbox.ledger(); + assert_eq!(sandbox.ingest(&mut ledger), IngestReport::empty()); } diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs index fc0db8d9..05ed7590 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot.rs @@ -27,8 +27,9 @@ use crate::util::time::{format_iso_ms, parse_iso_ms}; pub(super) fn refine(ctx: &Context<'_>, records: &mut SessionRecords) { for turn in &mut records.turns { refine_turn(ctx, turn); + leave_unnamed_output_to_aggregates(turn); } - let aggregates = shutdown_turns(ctx, &mut records.turns); + let aggregates = shutdown_turns(ctx, &records.turns); records.turns.extend(aggregates); // A shutdown's turn sits where the journal wrote it, so a session that // resumes appends turns after it and earlier indices never move. @@ -171,7 +172,7 @@ fn growth(now: &Usage, before: Option<&Usage>) -> Usage { } } -fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec { +fn shutdown_turns(ctx: &Context<'_>, message_turns: &[TurnRecord]) -> Vec { let mut turns = Vec::new(); let mut previous: HashMap = HashMap::new(); let mut after = i64::MIN; @@ -181,11 +182,11 @@ fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec = message_turns - .iter_mut() + let messages: Vec<&TurnRecord> = message_turns + .iter() .filter(|t| parse_iso_ms(&t.ts).is_some_and(|ts| ts > after && ts <= shutdown.ts_ms)) .collect(); - for (model, usage) in settle(&mut messages, growths) { + for (model, usage) in settle(&messages, growths) { turns.push(aggregate_turn(ctx, &shutdown, &model, usage)); } for (model, totals) in &shutdown.totals { @@ -196,25 +197,34 @@ fn shutdown_turns(ctx: &Context<'_>, message_turns: &mut [TurnRecord]) -> Vec) -> Vec<(String, Usage)> { - let unnamed = messages.iter_mut().filter(|t| t.model.is_empty()); - match growths.as_slice() { - [] => {} - [(only, _)] => unnamed.for_each(|t| t.model = only.clone()), - _ => unnamed.for_each(|t| t.usage.output = 0), - } +fn settle(messages: &[&TurnRecord], growths: Vec<(String, Usage)>) -> Vec<(String, Usage)> { growths .into_iter() .filter_map(|(model, mut usage)| { diff --git a/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs b/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs index 633e80fe..149db68d 100644 --- a/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs +++ b/crates/relayburn-sdk/src/source/relayhistory/copilot/settle_tests.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; -use super::settle; +use super::{leave_unnamed_output_to_aggregates, settle}; use crate::reader::types::{TurnRecord, Usage}; fn message(model: &str, output: u64) -> TurnRecord { @@ -40,16 +40,20 @@ fn billed(messages: &[TurnRecord], aggregates: &[(String, Usage)]) -> HashMap, growths: &[(&str, u64)], ) -> (Vec, Vec<(String, Usage)>) { + messages + .iter_mut() + .for_each(leave_unnamed_output_to_aggregates); let growths = growths .iter() .map(|(model, output)| (model.to_string(), grew(*output))) .collect(); - let mut refs: Vec<&mut TurnRecord> = messages.iter_mut().collect(); - let aggregates = settle(&mut refs, growths); + let refs: Vec<&TurnRecord> = messages.iter().collect(); + let aggregates = settle(&refs, growths); (messages, aggregates) } @@ -67,21 +71,24 @@ fn unnamed_messages_under_several_models_leave_their_output_to_the_aggregates() assert!(messages.iter().all(|t| t.usage.output == 0)); } +/// Under one model too, an unnamed message bills nothing and that model's +/// aggregate carries its output. #[test] -fn an_unnamed_message_under_one_model_is_that_models() { +fn an_unnamed_message_under_one_model_leaves_its_output_to_that_models_aggregate() { let (messages, aggregates) = settled(vec![message("", 4)], &[("gpt-5.4", 10)]); - assert_eq!(messages[0].model, "gpt-5.4"); - assert_eq!(messages[0].usage.output, 4); - assert_eq!(aggregates[0].1.output, 6); + assert_eq!( + (messages[0].model.as_str(), messages[0].usage.output), + ("", 0) + ); + assert_eq!(aggregates, [("gpt-5.4".to_string(), grew(10))]); } +/// Before any shutdown names a model, an unnamed message still bills +/// nothing: it bills the same once a shutdown lands. #[test] -fn an_interval_no_shutdown_names_a_model_for_keeps_its_messages() { +fn an_unnamed_message_bills_nothing_before_a_shutdown() { let (messages, aggregates) = settled(vec![message("", 4)], &[]); - assert_eq!( - (messages[0].model.as_str(), messages[0].usage.output), - ("", 4) - ); + assert_eq!(messages[0].usage.output, 0); assert!(aggregates.is_empty()); } diff --git a/tests/fixtures/sourcing-snapshots/copilot-official-importer.json b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json index b2c4aa29..41dbf187 100644 --- a/tests/fixtures/sourcing-snapshots/copilot-official-importer.json +++ b/tests/fixtures/sourcing-snapshots/copilot-official-importer.json @@ -7,8 +7,8 @@ "fidelity": { "granularity": "per-turn", "coverage": { - "hasInputTokens": false, - "hasOutputTokens": false, + "hasInputTokens": true, + "hasOutputTokens": true, "hasReasoningTokens": false, "hasCacheReadTokens": false, "hasCacheCreateTokens": false, @@ -17,7 +17,7 @@ "hasSessionRelationships": true, "hasRawContent": true }, - "class": "partial" + "class": "usage-only" }, "messageId": "28e4dc43-6d27-415f-9889-7a018434e36c", "turnIndex": 0, @@ -54,8 +54,8 @@ "fidelity": { "granularity": "per-turn", "coverage": { - "hasInputTokens": false, - "hasOutputTokens": false, + "hasInputTokens": true, + "hasOutputTokens": true, "hasReasoningTokens": false, "hasCacheReadTokens": false, "hasCacheCreateTokens": false, @@ -64,7 +64,7 @@ "hasSessionRelationships": true, "hasRawContent": true }, - "class": "partial" + "class": "usage-only" }, "messageId": "5fd0999e-2bb7-4296-bddd-3b3fd418526d", "turnIndex": 1, From 7973765557dd8fd15953d9d44887bd991276f279 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Wed, 21 Oct 2026 03:25:23 -0700 Subject: [PATCH 10/10] Pin per-call argument lookup and shutdown interval boundaries Co-Authored-By: Claude Opus 5.5 --- .../src/ingest/ingest_tests/copilot.rs | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs index 4b1598b0..f94ba740 100644 --- a/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs +++ b/crates/relayburn-sdk/src/ingest/ingest_tests/copilot.rs @@ -182,3 +182,81 @@ fn a_session_ingested_before_and_after_its_shutdown_bills_its_totals_once() { let mut ledger = sandbox.ledger(); assert_eq!(sandbox.ingest(&mut ledger), IngestReport::empty()); } + +/// Each tool call of one message reads its own arguments. +#[test] +fn each_call_of_a_message_reads_its_own_arguments() { + let sandbox = Sandbox::new(); + let second = + r#"{"toolCallId":"call_2","name":"view","arguments":{"path":"b.rs"},"type":"function"}"#; + let text = journal(&[("view", "call_1", "a.rs")]).replace( + r#""type":"function"}]"#, + &format!(r#""type":"function"}},{second}]"#), + ); + assert!(text.contains("call_2")); + let turns = ingest_journal(&sandbox, &text); + let mut targets: Vec<_> = turns[0] + .tool_calls + .iter() + .map(|c| (c.id.clone(), c.target.clone())) + .collect(); + targets.sort(); + assert_eq!( + targets, + [ + ("call_1".to_string(), Some("a.rs".to_string())), + ("call_2".to_string(), Some("b.rs".to_string())) + ] + ); +} + +/// A message written in its shutdown's millisecond belongs to that +/// shutdown's interval alone: the next shutdown bills its own growth whole. +#[test] +fn a_message_on_a_shutdown_boundary_counts_in_one_interval() { + let sandbox = Sandbox::new(); + let at = "2026-09-18T21:30:05.000Z"; + let event = |kind: &str, n: u32, ts: &str, data: serde_json::Value| { + serde_json::json!({"type": kind, "id": format!("00000000-0000-4000-8000-{n:012}"), + "parentId": null, "timestamp": ts, "data": data}) + .to_string() + }; + let totals = |output: u64| { + serde_json::json!({"modelMetrics": {"gpt-5.4": {"usage": {"inputTokens": 100, + "outputTokens": output, "cacheReadTokens": 0, "cacheWriteTokens": 0}}}}) + }; + let text = [ + event( + "session.start", + 0, + "2026-09-18T21:30:00.000Z", + serde_json::json!({"sessionId": SID, "version": 1, "producer": "copilot-agent", + "copilotVersion": "1.0.86", "context": {"cwd": "/tmp/project"}}), + ), + event( + "user.message", + 1, + "2026-09-18T21:30:01.000Z", + serde_json::json!({"content": "go", "turnId": "0"}), + ), + event( + "assistant.message", + 2, + at, + serde_json::json!({"messageId": "msg-0", "model": "gpt-5.4", "content": "done", + "turnId": "0", "outputTokens": 10}), + ), + event("session.shutdown", 3, at, totals(10)), + event( + "session.shutdown", + 4, + "2026-09-18T21:31:00.000Z", + totals(25), + ), + ] + .join("\n") + + "\n"; + let turns = ingest_journal(&sandbox, &text); + let output: u64 = turns.iter().map(|t| t.usage.output).sum(); + assert_eq!(output, 25); +}