From 83c93e3ec6401dd95851f394bd6e67d4b427be09 Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Sat, 14 Nov 2026 02:21:11 +0530 Subject: [PATCH] Forward prompt-cache token usage for every agent that records it Droid keeps its running token totals (with cache reads and writes) only in the .settings.json next to its transcript, and Continue CLI keeps them at the top level of the session file, outside the history autter streams. Both readers now append one `autter_usage_snapshot` event (disjoint counts, running session totals, the agent's own cost when it has one) to the batch that reaches the end of the file; it does not move the watermark. Copilot Chat's OTel traces database keeps cache writes only in the `gen_ai.usage.cache_creation.input_tokens` span attribute; the span JSON now carries them as `cache_creation_tokens` next to `cached_tokens`. The other agents already carry their cache counts in the raw events autter forwards; tests now pin that for Codex, Gemini CLI, Pi, Amp, Copilot CLI and OpenCode with real-shaped events. docs/agent-usage.md lists what each agent records and the snapshot convention. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + docs/agent-usage.md | 76 +++++++++ docs/agents.md | 3 + src/streams/agents/amp.rs | 25 +++ src/streams/agents/codex.rs | 23 +++ src/streams/agents/continue_cli.rs | 85 +++++++++- src/streams/agents/copilot_cli.rs | 22 +++ src/streams/agents/copilot_otel.rs | 73 +++++++++ src/streams/agents/droid.rs | 83 +++++++++- src/streams/agents/gemini.rs | 22 +++ src/streams/agents/opencode.rs | 36 +++++ src/streams/agents/pi.rs | 23 +++ src/streams/mod.rs | 1 + src/streams/usage_snapshot.rs | 252 +++++++++++++++++++++++++++++ tests/integration/continue_cli.rs | 13 +- tests/integration/droid.rs | 15 +- 16 files changed, 747 insertions(+), 9 deletions(-) create mode 100644 docs/agent-usage.md create mode 100644 src/streams/usage_snapshot.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f792a24..1a5b217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ This changelog is automatically generated by AI when releases are published. ## [Unreleased] +### Fixed + +- **AI spend includes prompt caching for Droid, Continue CLI and Copilot in VS Code.** These agents' cache reads and writes did not reach the backend, so their spend was priced as if nothing was cached. Droid keeps its token totals only in the `.settings.json` file next to the transcript, and Continue CLI keeps them at the top level of the session file, outside the history autter streams; autter now sends a snapshot of each total (input, output, cache reads, cache writes and, for Continue, its own cost) with the session's events. Copilot Chat's traces database stores cache writes only as a span attribute; autter now sends them as `cache_creation_tokens` next to `cached_tokens`. The other agents already carried their cache counts in the events autter forwards. [docs/agent-usage.md](docs/agent-usage.md) lists what each agent records and the convention the snapshot uses: disjoint counts, with input excluding cache reads and writes, and running session totals that are diffed, not summed. + ## [3.9.2] - 2026-11-14 ### Fixed diff --git a/docs/agent-usage.md b/docs/agent-usage.md new file mode 100644 index 0000000..90c4ee0 --- /dev/null +++ b/docs/agent-usage.md @@ -0,0 +1,76 @@ +# Agent token usage and prompt caching + +autter's background service streams each agent's session events to the +backend, which turns the token usage in them into the estimated AI spend shown +on the dashboard. Every estimate prices four token classes separately: input, +output, cache reads and cache writes. This page lists what each agent records +and how autter forwards it. + +## How usage travels + +Most agents put usage inside their own session events, and autter forwards +those events as written (secrets redacted, numbers untouched). The backend +reads each agent's shape and normalizes it there. autter adds data only where +the agent keeps usage outside the events it streams: + +| Agent | Where usage comes from | Convention in the source | What autter adds | +| --- | --- | --- | --- | +| Claude Code | `message.usage` on assistant lines | Disjoint: `input_tokens` excludes `cache_read_input_tokens` and `cache_creation_input_tokens`; `cache_creation.ephemeral_5m_input_tokens` / `ephemeral_1h_input_tokens` split writes by TTL | Nothing | +| Codex | `token_count` events, `info.total_token_usage` (running total) | OpenAI: `input_tokens` includes `cached_input_tokens` and `cache_write_input_tokens`; `output_tokens` includes `reasoning_output_tokens` | Nothing | +| GitHub Copilot (VS Code, OTel) | Copilot Chat's OTel traces database | OpenAI: `input_tokens` includes `cached_tokens` (reads) and the cache writes | `span.cache_creation_tokens`, read from the `gen_ai.usage.cache_creation.input_tokens` span attribute, which the database does not copy into a column | +| GitHub Copilot (VS Code chat sessions) | `result.usage` | `promptTokens` / `completionTokens` only, no cache counts | Nothing (cache counts come from OTel) | +| Copilot CLI | `session.shutdown` event, `data.modelMetrics..usage` (per-model running total) | `inputTokens` includes `cacheReadTokens` and `cacheWriteTokens` | Nothing. Per-request `assistant.usage` events are not written to disk by Copilot CLI | +| Gemini CLI | `tokens` on `gemini` records | Google: `input` (`promptTokenCount`) includes `cached`; `thoughts` and `tool` are counted separately; no cache writes | Nothing | +| OpenCode | `tokens` and `cost` on assistant messages | Disjoint: `input`, `cache.read`, `cache.write`; `reasoning` separate from `output`; `cost` in USD | Nothing | +| Pi | `message.usage` on assistant messages | Disjoint: `input`, `cacheRead`, `cacheWrite` (`cacheWrite1h` is a subset); `output` includes `reasoning`; `cost.total` in USD | Nothing | +| Amp | `usage` on assistant messages | Disjoint (Anthropic style, camelCase): `inputTokens`, `cacheReadInputTokens`, `cacheCreationInputTokens` | Nothing | +| Continue CLI | Newer versions: `message.usage` on assistant history items. All versions: a session-level `usage` total | Per message: `prompt_tokens` with `prompt_tokens_details`; the session total is disjoint with `totalCost` | A usage snapshot of the session-level total (below) | +| Droid | `tokenUsage` in `.settings.json`, next to the transcript (running total) | Disjoint: `inputTokens`, `cacheReadTokens`, `cacheCreationTokens`; `thinkingTokens` | A usage snapshot of the settings total (below) | +| Cursor, Windsurf, Firebender, Antigravity, Kiro | Not recorded in their transcripts or hook payloads | | Nothing; their sessions show as "no usage data" rather than $0 | + +## Usage snapshot events + +For Droid and Continue CLI, the reader appends one synthetic event to a batch +that read new records (only to the batch that reaches the end of the file, so +one per read-through). It does not count toward the stream's watermark. + +```json +{ + "type": "autter_usage_snapshot", + "timestamp": "2026-01-28T16:57:30.000Z", + "autter_usage_snapshot": { + "version": 1, + "source": "droid_settings", + "scope": "session_total", + "model": "claude-sonnet-4-6", + "input_tokens": 6743, + "output_tokens": 1450, + "cache_read_tokens": 42496, + "cache_write_tokens": 0, + "reasoning_tokens": 1088, + "cost_usd": null, + "input_includes_cache": false + } +} +``` + +- The counts are disjoint: `input_tokens` excludes cache reads and writes, and + `input_includes_cache` is always `false`. +- `output_tokens` includes reasoning; `reasoning_tokens`, when present, is a + subset of it. +- `scope` is `session_total`: each count is the session's running total when + the snapshot was taken. Consumers diff successive snapshots of a session and + never sum them. +- `cost_usd` is the cost the agent reported itself (Continue's `totalCost`), + or `null`. +- `source` is `droid_settings` or `continue_session`. `version` changes only + for a breaking change; new fields may be added. + +## Limits + +- Droid writes its settings file after a turn; usage from a session's last turn + arrives only when the transcript grows again. +- Codex records cache writes without their TTL (30-minute or 12-hour), so they + are priced at the 30-minute rate. +- Copilot reports cost in AI units (`copilot_usage_nano_aiu`), not dollars; it + is forwarded but not used as a dollar cost. diff --git a/docs/agents.md b/docs/agents.md index 547b792..73fdf84 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -37,6 +37,9 @@ Every failed step prints a `fix:` line. | JetBrains IDEs | IDE plugin (build 252 or newer) | Edits by GitHub Copilot for JetBrains and Junie | The model and prompts; other JetBrains AI plugins | `autter doctor --agent jetbrains` | | Visual Studio (Windows) | IDE extension (VSIX, Visual Studio 2022) | Copilot edits | The model and prompts | `autter doctor --agent visual-studio` | +Token usage, including prompt-cache reads and writes, is described per agent in +[agent-usage.md](agent-usage.md). + Shell commands an agent runs (Bash, `run_in_terminal`, ...) are captured by comparing the files before and after the command. Gitignored files and deletions are not attributed; see [shell-edit-capture.md](shell-edit-capture.md). diff --git a/src/streams/agents/amp.rs b/src/streams/agents/amp.rs index b778d8c..e4a84f5 100644 --- a/src/streams/agents/amp.rs +++ b/src/streams/agents/amp.rs @@ -447,4 +447,29 @@ mod tests { assert_eq!(content[1]["type"], "text"); assert_eq!(content[2]["type"], "tool_use"); } + + /// Amp assistant messages carry Anthropic-style disjoint usage in camelCase + /// (`inputTokens` excludes `cacheCreationInputTokens` / `cacheReadInputTokens`; + /// `totalInputTokens` is their sum). Real thread fixture, forwarded verbatim. + #[test] + fn test_assistant_usage_cache_fields_are_forwarded() { + let fixture = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/amp-threads/T-019ca1ce-3ae2-7686-a41e-ccc078837f8a.json"); + let (events, _) = drain_all(&AmpAgent::new(), &fixture); + let first = events + .iter() + .find(|e| e["role"] == "assistant") + .map(|e| crate::daemon::transcript_redaction::redact_json_secrets(e.clone())) + .unwrap(); + let usage = &first["usage"]; + assert_eq!(usage["model"], "claude-opus-4-6"); + assert_eq!(usage["inputTokens"], 3); + assert_eq!(usage["cacheCreationInputTokens"], 1619); + assert_eq!(usage["cacheReadInputTokens"], 13248); + assert_eq!( + usage["totalInputTokens"].as_u64().unwrap(), + 3 + 1619 + 13248, + "inputTokens is disjoint from the cache counts" + ); + } } diff --git a/src/streams/agents/codex.rs b/src/streams/agents/codex.rs index ac20293..73662d9 100644 --- a/src/streams/agents/codex.rs +++ b/src/streams/agents/codex.rs @@ -592,4 +592,27 @@ mod tests { CodexAgent::detect_subagent_parent(Path::new("/nonexistent/path/rollout.jsonl")); assert_eq!(result, None); } + + /// A Codex `token_count` rollout event (codex-rs/protocol `TokenUsage`): + /// OpenAI convention, `input_tokens` includes `cached_input_tokens` and + /// `cache_write_input_tokens`. Forwarded verbatim, cache fields intact. + #[test] + fn test_token_count_cache_fields_are_forwarded() { + use std::io::Write; + use tempfile::NamedTempFile; + + let line = r#"{"timestamp":"2026-11-10T10:00:05.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":24210,"cached_input_tokens":18944,"cache_write_input_tokens":4096,"output_tokens":812,"reasoning_output_tokens":448,"total_tokens":25022},"last_token_usage":{"input_tokens":12105,"cached_input_tokens":9472,"cache_write_input_tokens":2048,"output_tokens":406,"reasoning_output_tokens":224,"total_tokens":12511},"model_context_window":272000},"rate_limits":null}}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "{}", line).unwrap(); + file.flush().unwrap(); + + let (events, _) = drain_all(&CodexAgent::new(), file.path()); + let event = crate::daemon::transcript_redaction::redact_json_secrets(events[0].clone()); + let total = &event["payload"]["info"]["total_token_usage"]; + assert_eq!(total["input_tokens"], 24210); + assert_eq!(total["cached_input_tokens"], 18944); + assert_eq!(total["cache_write_input_tokens"], 4096); + assert_eq!(total["output_tokens"], 812); + assert_eq!(total["reasoning_output_tokens"], 448); + } } diff --git a/src/streams/agents/continue_cli.rs b/src/streams/agents/continue_cli.rs index 88a8083..04bc267 100644 --- a/src/streams/agents/continue_cli.rs +++ b/src/streams/agents/continue_cli.rs @@ -149,6 +149,12 @@ impl Agent for ContinueAgent { message: format!("Invalid JSON in {}: {}", path.display(), e), })?; + // Continue keeps a running session total (with prompt-cache reads and + // writes and its own cost) at the top level, outside `history`. + let session_usage = parsed + .get("usage") + .and_then(crate::streams::usage_snapshot::from_continue_session_usage); + let history = match parsed.as_object_mut().and_then(|obj| obj.remove("history")) { Some(serde_json::Value::Array(arr)) => arr, _ => { @@ -162,16 +168,25 @@ impl Agent for ContinueAgent { }; let batch_limit = self.batch_size_hint(); + let history_len = history.len() as u64; - let events: Vec = history + let mut events: Vec = history .into_iter() .skip(already_processed as usize) .take(batch_limit) .collect(); - let new_watermark = Box::new(RecordIndexWatermark::new( - already_processed + events.len() as u64, - )); + let next_index = already_processed + events.len() as u64; + let new_watermark = Box::new(RecordIndexWatermark::new(next_index)); + + // Append the session total to the last batch that read new history + // items. It is not a history record, so the watermark ignores it. + if !events.is_empty() + && next_index >= history_len + && let Some(snapshot) = session_usage.and_then(|usage| usage.to_event(None)) + { + events.push(snapshot); + } Ok(StreamBatch { events, @@ -401,4 +416,66 @@ mod tests { assert_eq!(result.events[0]["message"]["content"], "Let me check"); assert!(result.events[0]["contextItems"].is_array()); } + + fn snapshots(events: &[serde_json::Value]) -> Vec<&serde_json::Value> { + events + .iter() + .filter(|e| e["type"] == crate::streams::usage_snapshot::USAGE_SNAPSHOT_EVENT_TYPE) + .collect() + } + + #[test] + fn test_session_usage_with_cache_tokens_is_forwarded_once() { + let fixture = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/continue-cli-session-simple.json"); + let history_len = { + let parsed: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&fixture).unwrap()).unwrap(); + parsed["history"].as_array().unwrap().len() + }; + + // A small batch size: the snapshot rides only on the batch that reaches + // the end of the history, so it is sent once per read-through. + let agent = ContinueAgent::with_batch_size(2); + let (events, wm) = drain_all(&agent, &fixture); + assert_eq!(events.len(), history_len + 1); + let snaps = snapshots(&events); + assert_eq!(snaps.len(), 1); + assert!(std::ptr::eq(snaps[0], events.last().unwrap())); + + let usage = &snaps[0]["autter_usage_snapshot"]; + assert_eq!(usage["source"], "continue_session"); + assert_eq!(usage["scope"], "session_total"); + assert_eq!(usage["input_includes_cache"], false); + assert_eq!(usage["input_tokens"], 1813); + assert_eq!(usage["output_tokens"], 305); + assert_eq!(usage["cache_read_tokens"], 4366); + assert_eq!(usage["cache_write_tokens"], 2183); + assert_eq!(usage["cost_usd"], 0.002423); + + // The snapshot is not a history record: the watermark counts history only. + let wm = wm.as_any().downcast_ref::().unwrap(); + assert_eq!(wm.0, history_len as u64); + + // Nothing new: no events and no repeated snapshot. + let again = agent + .read_incremental(&fixture, Box::new(RecordIndexWatermark::new(wm.0)), "test") + .unwrap(); + assert!(again.events.is_empty()); + } + + #[test] + fn test_session_without_usage_adds_no_snapshot() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut file = NamedTempFile::new().unwrap(); + write!(file, "{}", make_continue_json(2)).unwrap(); + file.flush().unwrap(); + + let agent = ContinueAgent::new(); + let (events, _) = drain_all(&agent, file.path()); + assert_eq!(events.len(), 2); + assert!(snapshots(&events).is_empty()); + } } diff --git a/src/streams/agents/copilot_cli.rs b/src/streams/agents/copilot_cli.rs index 7579fb1..ff6aaf4 100644 --- a/src/streams/agents/copilot_cli.rs +++ b/src/streams/agents/copilot_cli.rs @@ -310,4 +310,26 @@ mod tests { assert_eq!(id, Some("evt-001".to_string())); assert_eq!(parent_id, None); } + + /// Copilot CLI's persisted `session.shutdown` event (copilot-sdk + /// `ShutdownModelMetricUsage`): per-model totals where `inputTokens` + /// includes `cacheReadTokens` and `cacheWriteTokens`. + #[test] + fn test_shutdown_model_metrics_cache_fields_are_forwarded() { + use std::io::Write; + use tempfile::NamedTempFile; + + let line = r#"{"type":"session.shutdown","data":{"shutdownType":"routine","totalPremiumRequests":3,"totalApiDurationMs":41250,"sessionStartTime":1794304800000,"currentModel":"claude-sonnet-4.6","modelMetrics":{"claude-sonnet-4.6":{"requests":{"count":7,"cost":3},"usage":{"inputTokens":148200,"outputTokens":3120,"cacheReadTokens":121600,"cacheWriteTokens":18400,"reasoningTokens":640}}}},"id":"e5f6a7b8-5555-6666-7777-888899990000","timestamp":"2026-11-10T10:05:00.000Z","parentId":"d4e5f6a7-4444-5555-6666-777788889999"}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "{}", line).unwrap(); + file.flush().unwrap(); + + let (events, _) = drain_all(&CopilotCliAgent::new(), file.path()); + let event = crate::daemon::transcript_redaction::redact_json_secrets(events[0].clone()); + let usage = &event["data"]["modelMetrics"]["claude-sonnet-4.6"]["usage"]; + assert_eq!(usage["inputTokens"], 148200); + assert_eq!(usage["cacheReadTokens"], 121600); + assert_eq!(usage["cacheWriteTokens"], 18400); + assert_eq!(usage["reasoningTokens"], 640); + } } diff --git a/src/streams/agents/copilot_otel.rs b/src/streams/agents/copilot_otel.rs index c055982..457bf17 100644 --- a/src/streams/agents/copilot_otel.rs +++ b/src/streams/agents/copilot_otel.rs @@ -273,11 +273,28 @@ fn read_events_for_spans( Ok(result) } +/// Span attribute VS Code Copilot Chat sets to the prompt-cache write count. +/// Its traces DB keeps it only in `span_attributes`; `cached_tokens` is its +/// cache-read sibling, `gen_ai.usage.cache_read.input_tokens`. +const CACHE_CREATION_ATTR: &str = "gen_ai.usage.cache_creation.input_tokens"; + +/// Integer value of a span attribute (stored as text), if it parses. +fn attribute_count(attributes: &HashMap, key: &str) -> Option { + let raw = attributes.get(key)?.trim(); + raw.parse::() + .ok() + .or_else(|| raw.parse::().ok().map(|f| f as i64)) +} + fn build_span_event_json( span: SpanRow, attributes: HashMap, events: Vec, ) -> serde_json::Value { + // Token convention (OpenAI style, as Copilot reports it): `input_tokens` + // includes both `cached_tokens` (cache reads) and `cache_creation_tokens` + // (cache writes); `output_tokens` includes `reasoning_tokens`. + let cache_creation_tokens = attribute_count(&attributes, CACHE_CREATION_ATTR); json!({ "span": { "span_id": span.span_id, @@ -297,6 +314,7 @@ fn build_span_event_json( "input_tokens": span.input_tokens, "output_tokens": span.output_tokens, "cached_tokens": span.cached_tokens, + "cache_creation_tokens": cache_creation_tokens, "reasoning_tokens": span.reasoning_tokens, "tool_name": span.tool_name, "tool_call_id": span.tool_call_id, @@ -864,6 +882,7 @@ mod tests { assert_eq!(span["input_tokens"], 500); assert_eq!(span["output_tokens"], 200); assert_eq!(span["cached_tokens"], 100); + assert!(span["cache_creation_tokens"].is_null()); assert_eq!(span["reasoning_tokens"], 50); assert_eq!(span["tool_name"], "read_file"); assert_eq!(span["tool_call_id"], "call-xyz"); @@ -873,6 +892,60 @@ mod tests { assert_eq!(span["ttft_ms"], 125.5); } + /// A Claude request through Copilot, as VS Code Copilot Chat records it: + /// `prompt_tokens` = Anthropic input + cache writes + cache reads, the read + /// count denormalized into `cached_tokens`, the write count only in + /// `span_attributes`. + #[test] + fn test_cache_creation_tokens_are_lifted_from_span_attributes() { + let (_dir, db_path) = create_test_otel_db(); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute( + "INSERT INTO spans (span_id, trace_id, name, start_time_ms, end_time_ms, \ + operation_name, provider_name, request_model, response_model, \ + input_tokens, output_tokens, cached_tokens, reasoning_tokens, chat_session_id) \ + VALUES ('claude-span', 'trace-1', 'chat claude-sonnet-4.6', 1000, 2000, \ + 'chat', 'anthropic', 'claude-sonnet-4.6', 'claude-sonnet-4-6', \ + 15040, 412, 12800, 96, 'session-1')", + [], + ) + .unwrap(); + for (key, value) in [ + ("gen_ai.usage.input_tokens", "15040"), + ("gen_ai.usage.output_tokens", "412"), + ("gen_ai.usage.cache_read.input_tokens", "12800"), + ("gen_ai.usage.cache_creation.input_tokens", "2210"), + ] { + conn.execute( + "INSERT INTO span_attributes (span_id, key, value) VALUES ('claude-span', ?1, ?2)", + rusqlite::params![key, value], + ) + .unwrap(); + } + drop(conn); + + let watermark: Box = Box::new(TimestampCursorWatermark::initial()); + let batch = read_otel_spans_incremental(&db_path, watermark, 100).unwrap(); + let event = &batch.events[0]; + assert_eq!(event["span"]["input_tokens"], 15040); + assert_eq!(event["span"]["cached_tokens"], 12800); + assert_eq!(event["span"]["cache_creation_tokens"], 2210); + // The raw attribute still travels with the span. + assert_eq!( + event["attributes"]["gen_ai.usage.cache_creation.input_tokens"], + "2210" + ); + } + + #[test] + fn test_unparseable_cache_creation_attribute_is_null() { + let mut attrs = HashMap::new(); + attrs.insert(CACHE_CREATION_ATTR.to_string(), "n/a".to_string()); + assert_eq!(attribute_count(&attrs, CACHE_CREATION_ATTR), None); + attrs.insert(CACHE_CREATION_ATTR.to_string(), " 7.0 ".to_string()); + assert_eq!(attribute_count(&attrs, CACHE_CREATION_ATTR), Some(7)); + } + #[test] fn test_initial_watermark_uses_simple_greater_than() { let (_dir, db_path) = create_test_otel_db(); diff --git a/src/streams/agents/droid.rs b/src/streams/agents/droid.rs index 6dc37a4..3763e5b 100644 --- a/src/streams/agents/droid.rs +++ b/src/streams/agents/droid.rs @@ -65,6 +65,23 @@ impl DroidAgent { } } } + + /// The settings file Droid writes next to a session transcript. + pub fn settings_path(transcript: &Path) -> Option { + let name = transcript.file_name()?.to_str()?; + let stem = name.strip_suffix(".jsonl")?; + Some(transcript.with_file_name(format!("{stem}.settings.json"))) + } + + /// A usage snapshot event from the session's settings file, if it has usage. + fn usage_snapshot_event(transcript: &Path) -> Option { + let settings_path = Self::settings_path(transcript)?; + let settings: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&settings_path).ok()?).ok()?; + crate::streams::usage_snapshot::from_droid_settings(&settings)?.to_event( + crate::streams::usage_snapshot::file_mtime_rfc3339(&settings_path), + ) + } } impl Default for DroidAgent { @@ -172,6 +189,7 @@ impl Agent for DroidAgent { // Read lines from watermark position let mut line = String::new(); + let mut reached_end = false; loop { match crate::streams::types::read_jsonl_line(&mut reader, &mut line).map_err(|e| { StreamError::Transient { @@ -179,8 +197,11 @@ impl Agent for DroidAgent { retry_after: std::time::Duration::from_secs(5), } })? { - crate::streams::types::JsonlLineState::Eof => break, - crate::streams::types::JsonlLineState::Partial => break, + crate::streams::types::JsonlLineState::Eof + | crate::streams::types::JsonlLineState::Partial => { + reached_end = true; + break; + } crate::streams::types::JsonlLineState::Complete(bytes_read) => { line_number += 1; current_offset += bytes_read as u64; @@ -231,6 +252,16 @@ impl Agent for DroidAgent { } } + // Droid keeps token usage (with prompt-cache reads and writes) only as a + // running total in the sibling settings file. Append a snapshot of it to + // the last batch that read new messages; the watermark is unaffected. + if reached_end + && !events.is_empty() + && let Some(snapshot) = Self::usage_snapshot_event(path) + { + events.push(snapshot); + } + // Create new hybrid watermark with updated offset, record count, and timestamp let new_watermark = Box::new(HybridWatermark::new( current_offset, @@ -430,4 +461,52 @@ mod tests { assert_eq!(new_watermark.record, 2); // Two message records processed assert!(new_watermark.timestamp.is_some()); // Timestamp captured } + + #[test] + fn test_settings_token_usage_with_cache_tokens_is_forwarded() { + let fixtures = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let dir = tempfile::tempdir().unwrap(); + let transcript = dir.path().join("052cb8d0.jsonl"); + std::fs::copy(fixtures.join("droid-session.jsonl"), &transcript).unwrap(); + std::fs::copy( + fixtures.join("droid-session.settings.json"), + dir.path().join("052cb8d0.settings.json"), + ) + .unwrap(); + + let agent = DroidAgent::new(); + let (events, wm) = drain_all(&agent, &transcript); + let snapshot_type = crate::streams::usage_snapshot::USAGE_SNAPSHOT_EVENT_TYPE; + let snaps: Vec<_> = events + .iter() + .filter(|e| e["type"] == snapshot_type) + .collect(); + assert_eq!(snaps.len(), 1); + assert_eq!(events.last().unwrap()["type"], snapshot_type); + assert!(snaps[0]["timestamp"].is_string()); + + let usage = &snaps[0]["autter_usage_snapshot"]; + assert_eq!(usage["source"], "droid_settings"); + assert_eq!(usage["model"], "custom:BYOK-GPT-5-MINI-0"); + assert_eq!(usage["input_tokens"], 6743); + assert_eq!(usage["output_tokens"], 1450); + assert_eq!(usage["cache_read_tokens"], 42496); + assert_eq!(usage["cache_write_tokens"], 0); + assert_eq!(usage["reasoning_tokens"], 1088); + assert_eq!(usage["input_includes_cache"], false); + + // Only message records count toward the watermark. + let messages = events.iter().filter(|e| e["type"] == "message").count(); + let wm = wm.as_any().downcast_ref::().unwrap(); + assert_eq!(wm.record, messages as u64); + } + + #[test] + fn test_settings_path_is_sibling_of_transcript() { + assert_eq!( + DroidAgent::settings_path(Path::new("/s/abc.jsonl")), + Some(PathBuf::from("/s/abc.settings.json")) + ); + assert_eq!(DroidAgent::settings_path(Path::new("/s/abc.json")), None); + } } diff --git a/src/streams/agents/gemini.rs b/src/streams/agents/gemini.rs index 07aa9c9..65eb291 100644 --- a/src/streams/agents/gemini.rs +++ b/src/streams/agents/gemini.rs @@ -424,4 +424,26 @@ mod tests { .unwrap(); assert_eq!(result2.events.len(), 0); } + + /// A Gemini CLI `gemini` record with its `tokens` summary + /// (chatRecordingService `TokensSummary`): `input` is `promptTokenCount`, + /// which includes `cached` (`cachedContentTokenCount`). + #[test] + fn test_message_token_summary_with_cached_is_forwarded() { + use std::io::Write; + use tempfile::NamedTempFile; + + let line = r#"{"id":"0c5f2a9e-6b1d-4a54-9e1a-2f6f5c1d7a10","timestamp":"2026-11-10T10:00:05.000Z","type":"gemini","content":"Updated index.ts.","model":"gemini-3.1-pro-preview","thoughts":[],"tokens":{"input":20480,"output":310,"cached":16384,"thoughts":220,"tool":12,"total":21022}}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "{}", line).unwrap(); + file.flush().unwrap(); + + let (events, _) = drain_all(&GeminiAgent::new(), file.path()); + let event = crate::daemon::transcript_redaction::redact_json_secrets(events[0].clone()); + assert_eq!(event["model"], "gemini-3.1-pro-preview"); + assert_eq!(event["tokens"]["input"], 20480); + assert_eq!(event["tokens"]["cached"], 16384); + assert_eq!(event["tokens"]["thoughts"], 220); + assert_eq!(event["tokens"]["tool"], 12); + } } diff --git a/src/streams/agents/opencode.rs b/src/streams/agents/opencode.rs index bcb9eb1..085a6f0 100644 --- a/src/streams/agents/opencode.rs +++ b/src/streams/agents/opencode.rs @@ -690,4 +690,40 @@ mod tests { assert_eq!(pid, Some("msg_c5d3ff791001Egl5tW62x4Vgzo".to_string())); assert_eq!(tid, None); } + + /// An OpenCode assistant message: disjoint `tokens.input` / + /// `tokens.cache.read` / `tokens.cache.write`, plus OpenCode's own `cost`. + #[test] + fn test_assistant_tokens_cache_and_cost_are_forwarded() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("opencode.db"); + create_test_db(&db_path, 0); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute( + "INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + "msg-assistant", + "test-session", + 5000, + 6000, + r#"{"role":"assistant","modelID":"claude-sonnet-4-6","providerID":"anthropic","cost":0.0187,"tokens":{"input":3,"output":245,"reasoning":0,"cache":{"read":13248,"write":2959}}}"#, + ], + ) + .unwrap(); + drop(conn); + + let (events, _) = drain_all(&OpenCodeAgent::new(), &db_path); + let event = crate::daemon::transcript_redaction::redact_json_secrets( + events + .iter() + .find(|e| e["message"]["id"] == "msg-assistant") + .cloned() + .unwrap(), + ); + let data = &event["message"]["data"]; + assert_eq!(data["tokens"]["input"], 3); + assert_eq!(data["tokens"]["cache"]["read"], 13248); + assert_eq!(data["tokens"]["cache"]["write"], 2959); + assert_eq!(data["cost"], 0.0187); + } } diff --git a/src/streams/agents/pi.rs b/src/streams/agents/pi.rs index 2aad9ce..0d3159f 100644 --- a/src/streams/agents/pi.rs +++ b/src/streams/agents/pi.rs @@ -372,4 +372,27 @@ mod tests { assert_eq!(result.events[0]["type"].as_str(), Some("session")); assert_eq!(result.events[1]["type"].as_str(), Some("message")); } + + /// A Pi assistant message (pi-mono `AssistantMessage.usage`): disjoint + /// `input` / `cacheRead` / `cacheWrite`, `cacheWrite1h` a subset of + /// `cacheWrite`, and Pi's own cost breakdown. + #[test] + fn test_assistant_usage_cache_fields_and_cost_are_forwarded() { + use std::io::Write; + use tempfile::NamedTempFile; + + let line = r#"{"type":"message","id":"33333333","parentId":"22222222","timestamp":"2026-11-10T10:00:05.000Z","message":{"role":"assistant","api":"anthropic-messages","provider":"anthropic","model":"claude-sonnet-4-6","content":[{"type":"text","text":"Done."}],"usage":{"input":3,"output":245,"cacheRead":13248,"cacheWrite":2959,"cacheWrite1h":0,"totalTokens":16455,"cost":{"input":0.000009,"output":0.003675,"cacheRead":0.0039744,"cacheWrite":0.01109625,"total":0.01875465}},"stopReason":"stop","timestamp":1794304805000}}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "{}", line).unwrap(); + file.flush().unwrap(); + + let (events, _) = drain_all(&PiAgent::new(), file.path()); + let event = crate::daemon::transcript_redaction::redact_json_secrets(events[0].clone()); + let usage = &event["message"]["usage"]; + assert_eq!(usage["input"], 3); + assert_eq!(usage["cacheRead"], 13248); + assert_eq!(usage["cacheWrite"], 2959); + assert_eq!(usage["cacheWrite1h"], 0); + assert_eq!(usage["cost"]["total"], 0.01875465); + } } diff --git a/src/streams/mod.rs b/src/streams/mod.rs index a442280..649e558 100644 --- a/src/streams/mod.rs +++ b/src/streams/mod.rs @@ -47,6 +47,7 @@ pub mod db; pub mod model_extraction; pub mod sweep; pub mod types; +pub mod usage_snapshot; pub mod watermark; // Re-export main types for convenient access diff --git a/src/streams/usage_snapshot.rs b/src/streams/usage_snapshot.rs new file mode 100644 index 0000000..377beb6 --- /dev/null +++ b/src/streams/usage_snapshot.rs @@ -0,0 +1,252 @@ +//! Session usage snapshots for agents that keep token usage outside the events +//! autter streams. +//! +//! autter forwards agent transcript events verbatim, so for most agents the +//! token usage (including prompt-cache reads and writes) travels inside the raw +//! event and the backend reads it from the agent's own shape. A few agents keep +//! usage somewhere else: +//! +//! - Droid writes a running `tokenUsage` total into `.settings.json`, +//! next to the transcript, and never into the transcript itself. +//! - Continue CLI writes a running `usage` total at the top level of the session +//! file, while autter streams only the `history` items. +//! +//! For those, the reader appends one synthetic event per batch that read new +//! records, built here. The counts follow one convention, documented in +//! `docs/agent-usage.md`: +//! +//! - the counts are disjoint: `input_tokens` excludes `cache_read_tokens` and +//! `cache_write_tokens` (`input_includes_cache` is always `false`); +//! - `output_tokens` includes reasoning; `reasoning_tokens` is a subset of it; +//! - `scope` is `session_total`: every count is the session's running total +//! when the snapshot was taken, so the backend must diff snapshots rather +//! than sum them; +//! - `cost_usd` is the cost the agent itself reported, when it reports one. + +use serde_json::{Value, json}; + +/// `type` of the synthetic event. Never produced by an agent. +pub const USAGE_SNAPSHOT_EVENT_TYPE: &str = "autter_usage_snapshot"; + +/// Version of the snapshot payload. Bump only for breaking changes. +pub const USAGE_SNAPSHOT_VERSION: u64 = 1; + +/// A session's running token totals in autter's disjoint convention. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct SessionUsageSnapshot { + /// Where the totals came from, e.g. `droid_settings`. + pub source: &'static str, + pub model: Option, + /// Input tokens that were neither read from nor written to a prompt cache. + pub input_tokens: u64, + /// Output tokens, reasoning included. + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + /// Reasoning tokens, a subset of `output_tokens`, when reported. + pub reasoning_tokens: Option, + /// Cost the agent itself reported, in USD. + pub cost_usd: Option, +} + +impl SessionUsageSnapshot { + fn is_empty(&self) -> bool { + self.input_tokens == 0 + && self.output_tokens == 0 + && self.cache_read_tokens == 0 + && self.cache_write_tokens == 0 + && self.cost_usd.unwrap_or(0.0) <= 0.0 + } + + /// The synthetic event, or `None` when the snapshot carries no usage. + pub fn to_event(&self, timestamp: Option) -> Option { + if self.is_empty() { + return None; + } + let mut event = json!({ + "type": USAGE_SNAPSHOT_EVENT_TYPE, + "autter_usage_snapshot": { + "version": USAGE_SNAPSHOT_VERSION, + "source": self.source, + "scope": "session_total", + "model": self.model, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "cache_read_tokens": self.cache_read_tokens, + "cache_write_tokens": self.cache_write_tokens, + "reasoning_tokens": self.reasoning_tokens, + "cost_usd": self.cost_usd, + "input_includes_cache": false, + }, + }); + if let Some(ts) = timestamp { + event["timestamp"] = Value::String(ts); + } + Some(event) + } +} + +/// A non-negative integer count, accepting integers, floats and numeric strings. +pub fn count(value: Option<&Value>) -> u64 { + match value { + Some(Value::Number(n)) => n + .as_u64() + .or_else(|| n.as_f64().filter(|f| *f > 0.0).map(|f| f as u64)) + .unwrap_or(0), + Some(Value::String(s)) => s.trim().parse::().unwrap_or(0), + _ => 0, + } +} + +fn optional_count(value: Option<&Value>) -> Option { + value.filter(|v| !v.is_null()).map(|v| count(Some(v))) +} + +/// Droid's `.settings.json`: +/// `{"model": "...", "tokenUsage": {"inputTokens", "outputTokens", +/// "cacheCreationTokens", "cacheReadTokens", "thinkingTokens"}}`. +/// +/// Droid's `inputTokens` is already disjoint from the cache counts (observed: +/// `inputTokens` 6,743 next to `cacheReadTokens` 42,496). `thinkingTokens` is +/// kept as a subset of output, as both Anthropic and OpenAI bill it as output. +pub fn from_droid_settings(settings: &Value) -> Option { + let usage = settings.get("tokenUsage")?.as_object()?; + Some(SessionUsageSnapshot { + source: "droid_settings", + model: settings + .get("model") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(String::from), + input_tokens: count(usage.get("inputTokens")), + output_tokens: count(usage.get("outputTokens")), + cache_read_tokens: count(usage.get("cacheReadTokens")), + cache_write_tokens: count(usage.get("cacheCreationTokens")), + reasoning_tokens: optional_count(usage.get("thinkingTokens")), + cost_usd: None, + }) +} + +/// Continue CLI's session-level `usage`: +/// `{"totalCost", "promptTokens", "completionTokens", +/// "promptTokensDetails": {"cachedTokens", "cacheWriteTokens"}}`. +/// +/// Continue sums each call's `prompt_tokens` and its adapter's +/// `cache_read_tokens` / `cache_write_tokens`. Its Anthropic adapter sets +/// `prompt_tokens` to Anthropic's `input_tokens`, which already excludes cache +/// reads and writes, and only that adapter fills the cache fields, so the +/// totals are taken as disjoint. +pub fn from_continue_session_usage(usage: &Value) -> Option { + let usage = usage.as_object()?; + let details = usage.get("promptTokensDetails"); + Some(SessionUsageSnapshot { + source: "continue_session", + model: None, + input_tokens: count(usage.get("promptTokens")), + output_tokens: count(usage.get("completionTokens")), + cache_read_tokens: count(details.and_then(|d| d.get("cachedTokens"))), + cache_write_tokens: count(details.and_then(|d| d.get("cacheWriteTokens"))), + reasoning_tokens: None, + cost_usd: usage + .get("totalCost") + .and_then(Value::as_f64) + .filter(|c| c.is_finite() && *c > 0.0), + }) +} + +/// File modification time as an RFC 3339 timestamp, for a snapshot's `timestamp`. +pub fn file_mtime_rfc3339(path: &std::path::Path) -> Option { + let modified = std::fs::metadata(path).ok()?.modified().ok()?; + let dt: chrono::DateTime = modified.into(); + Some(dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn droid_settings_fixture_maps_cache_fields() { + let settings: Value = serde_json::from_str( + &std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/droid-session.settings.json" + )) + .unwrap(), + ) + .unwrap(); + let snap = from_droid_settings(&settings).unwrap(); + assert_eq!(snap.source, "droid_settings"); + assert_eq!(snap.model.as_deref(), Some("custom:BYOK-GPT-5-MINI-0")); + assert_eq!(snap.input_tokens, 6743); + assert_eq!(snap.output_tokens, 1450); + assert_eq!(snap.cache_read_tokens, 42496); + assert_eq!(snap.cache_write_tokens, 0); + assert_eq!(snap.reasoning_tokens, Some(1088)); + assert_eq!(snap.cost_usd, None); + + let event = snap + .to_event(Some("2026-01-28T16:57:30.000Z".into())) + .unwrap(); + assert_eq!(event["type"], USAGE_SNAPSHOT_EVENT_TYPE); + assert_eq!(event["timestamp"], "2026-01-28T16:57:30.000Z"); + let s = &event["autter_usage_snapshot"]; + assert_eq!(s["version"], 1); + assert_eq!(s["scope"], "session_total"); + assert_eq!(s["input_includes_cache"], false); + assert_eq!(s["cache_read_tokens"], 42496); + assert_eq!(s["cache_write_tokens"], 0); + assert_eq!(s["reasoning_tokens"], 1088); + assert!(s["cost_usd"].is_null()); + } + + #[test] + fn droid_settings_without_token_usage_is_none() { + let settings = json!({"model": "claude-opus-4-6"}); + assert!(from_droid_settings(&settings).is_none()); + } + + #[test] + fn continue_session_usage_fixture_maps_cache_fields_and_cost() { + let session: Value = serde_json::from_str( + &std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/continue-cli-session-simple.json" + )) + .unwrap(), + ) + .unwrap(); + let snap = from_continue_session_usage(&session["usage"]).unwrap(); + assert_eq!(snap.source, "continue_session"); + assert_eq!(snap.input_tokens, 1813); + assert_eq!(snap.output_tokens, 305); + assert_eq!(snap.cache_read_tokens, 4366); + assert_eq!(snap.cache_write_tokens, 2183); + assert_eq!(snap.cost_usd, Some(0.002423)); + + let event = snap.to_event(None).unwrap(); + assert!(event.get("timestamp").is_none()); + assert_eq!(event["autter_usage_snapshot"]["cost_usd"], 0.002423); + assert!(event["autter_usage_snapshot"]["model"].is_null()); + } + + #[test] + fn empty_usage_produces_no_event() { + let snap = from_continue_session_usage(&json!({ + "totalCost": 0, + "promptTokens": 0, + "completionTokens": 0, + })) + .unwrap(); + assert!(snap.to_event(None).is_none()); + } + + #[test] + fn counts_accept_strings_and_floats_and_reject_negatives() { + assert_eq!(count(Some(&json!("42"))), 42); + assert_eq!(count(Some(&json!(42.0))), 42); + assert_eq!(count(Some(&json!(-3))), 0); + assert_eq!(count(Some(&json!(null))), 0); + assert_eq!(count(None), 0); + } +} diff --git a/tests/integration/continue_cli.rs b/tests/integration/continue_cli.rs index 9b06e34..e993ccf 100644 --- a/tests/integration/continue_cli.rs +++ b/tests/integration/continue_cli.rs @@ -25,7 +25,18 @@ fn test_continue_cli_raw_event_fidelity() { serde_json::from_str(&std::fs::read_to_string(&fixture).unwrap()).unwrap(); let expected: Vec = parsed["history"].as_array().unwrap().clone(); - assert_eq!(result.events, expected); + // History items verbatim, then one snapshot of the session-level usage + // (prompt-cache reads and writes and Continue's own cost). + let (snapshot, history) = result.events.split_last().unwrap(); + assert_eq!(history, expected.as_slice()); + assert_eq!( + snapshot["type"], + autter::streams::usage_snapshot::USAGE_SNAPSHOT_EVENT_TYPE + ); + assert_eq!( + snapshot["autter_usage_snapshot"]["cache_write_tokens"], + 2183 + ); } #[test] diff --git a/tests/integration/droid.rs b/tests/integration/droid.rs index 4a60775..e3ba019 100644 --- a/tests/integration/droid.rs +++ b/tests/integration/droid.rs @@ -32,8 +32,19 @@ fn test_droid_raw_event_fidelity() { .filter(|v| v["type"].as_str() == Some("message")) .collect(); - assert_eq!(result.events.len(), expected.len()); - assert_eq!(result.events, expected); + // The sibling settings file's running token totals (with prompt-cache + // reads and writes) ride along as one trailing usage snapshot event. + let (snapshot, messages) = result.events.split_last().unwrap(); + assert_eq!(messages.len(), expected.len()); + assert_eq!(messages, expected.as_slice()); + assert_eq!( + snapshot["type"], + autter::streams::usage_snapshot::USAGE_SNAPSHOT_EVENT_TYPE + ); + assert_eq!( + snapshot["autter_usage_snapshot"]["cache_read_tokens"], + 42496 + ); } #[test]