Skip to content

feat(langchain): report cached and reasoning token usage in Langchain/Langgraph #6800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joaquinhuigomez wants to merge 1 commit into
base: master
Choose a base branch
from
+164 −10
Open

feat(langchain): report cached and reasoning token usage in Langchain/Langgraph #6800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 58 additions & 10 deletions sentry_sdk/integrations/langchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,9 +730,9 @@ def on_tool_error(

def _extract_tokens(
token_usage: "Any",
) -> "tuple[Optional[int], Optional[int], Optional[int]]":
) -> "tuple[Optional[int], Optional[int], Optional[int], Optional[int], Optional[int]]":
if not token_usage:
return None, None, None
return None, None, None, None, None

input_tokens = _get_value(token_usage, "prompt_tokens") or _get_value(
token_usage, "input_tokens"
Expand All @@ -742,32 +742,64 @@ def _extract_tokens(
)
total_tokens = _get_value(token_usage, "total_tokens")

return input_tokens, output_tokens, total_tokens
# LangChain's usage_metadata nests these under input/output_token_details;
# OpenAI-style dicts use prompt/completion_tokens_details.
input_details = _get_value(token_usage, "input_token_details") or _get_value(
token_usage, "prompt_tokens_details"
)
cached_tokens = None
if input_details is not None:
cached_tokens = _get_value(input_details, "cache_read") or _get_value(
input_details, "cached_tokens"
)
Comment on lines +752 to +754

@sentry sentry Bot Jul 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: The use of the or operator incorrectly handles legitimate 0 values for token counts, causing them to be evaluated as None and dropped from span data.
Severity: MEDIUM

Suggested Fix

Refactor the logic to avoid using the or operator for fallback when 0 is a valid value. Instead, explicitly check if the first value is None before attempting to retrieve the second value. This ensures that a 0 from the primary key is preserved.

Prompt for AI Agent 
Review the code at the location below. A potential bug has been identified by an AI
agent. Verify if this is a real issue. If it is, propose a fix; if not, explain why it's
not valid.

Location: sentry_sdk/integrations/langchain.py#L752-L754

Potential issue: In the `_extract_tokens` function, the use of the `or` operator to
select between two potential keys for token counts (e.g., `cache_read` or
`cached_tokens`) leads to incorrect behavior when a valid token count is `0`. Because
`0` is a falsy value in Python, the expression proceeds to evaluate the right-hand side.
If the alternative key does not exist, the result becomes `None`. This causes legitimate
zero-token counts for cached or reasoning tokens to be silently dropped instead of being
reported in span data. While this has minimal impact on cost calculations, it is a
correctness issue that prevents accurate reporting of API responses.

Also affects:

  • sentry_sdk/integrations/langchain.py:761~763
  • sentry_sdk/integrations/langchain.py:737~738
  • sentry_sdk/integrations/langchain.py:742~743

Did we get this right? 👍 / 👎 to inform future reviews.


output_details = _get_value(token_usage, "output_token_details") or _get_value(
token_usage, "completion_tokens_details"
)
reasoning_tokens = None
if output_details is not None:
reasoning_tokens = _get_value(output_details, "reasoning") or _get_value(
output_details, "reasoning_tokens"
)

return input_tokens, output_tokens, total_tokens, cached_tokens, reasoning_tokens


def _extract_tokens_from_generations(
generations: "Any",
) -> "tuple[Optional[int], Optional[int], Optional[int]]":
) -> "tuple[Optional[int], Optional[int], Optional[int], Optional[int], Optional[int]]":
"""Extract token usage from response.generations structure."""
if not generations:
return None, None, None
return None, None, None, None, None

total_input = 0
total_output = 0
total_total = 0
total_cached = 0
total_reasoning = 0

for gen_list in generations:
for gen in gen_list:
token_usage = _get_token_usage(gen)
input_tokens, output_tokens, total_tokens = _extract_tokens(token_usage)
(
input_tokens,
output_tokens,
total_tokens,
cached_tokens,
reasoning_tokens,
) = _extract_tokens(token_usage)
total_input += input_tokens if input_tokens is not None else 0
total_output += output_tokens if output_tokens is not None else 0
total_total += total_tokens if total_tokens is not None else 0
total_cached += cached_tokens if cached_tokens is not None else 0
total_reasoning += reasoning_tokens if reasoning_tokens is not None else 0

return (
total_input if total_input > 0 else None,
total_output if total_output > 0 else None,
total_total if total_total > 0 else None,
total_cached if total_cached > 0 else None,
total_reasoning if total_reasoning > 0 else None,
)


Expand Down Expand Up @@ -802,11 +834,21 @@ def _get_token_usage(obj: "Any") -> "Optional[Dict[str, Any]]":
def _record_token_usage(span: "Union[Span, StreamedSpan]", response: "Any") -> None:
token_usage = _get_token_usage(response)
if token_usage:
input_tokens, output_tokens, total_tokens = _extract_tokens(token_usage)
(
input_tokens,
output_tokens,
total_tokens,
cached_tokens,
reasoning_tokens,
) = _extract_tokens(token_usage)
else:
input_tokens, output_tokens, total_tokens = _extract_tokens_from_generations(
response.generations
)
(
input_tokens,
output_tokens,
total_tokens,
cached_tokens,
reasoning_tokens,
) = _extract_tokens_from_generations(response.generations)

set_on_span = (
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
Expand All @@ -821,6 +863,12 @@ def _record_token_usage(span: "Union[Span, StreamedSpan]", response: "Any") -> N
if total_tokens is not None:
set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens)

if cached_tokens is not None:
set_on_span(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cached_tokens)

if reasoning_tokens is not None:
set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens)


def _get_request_data(
obj: "Any", args: "Any", kwargs: "Any"
Expand Down
14 changes: 14 additions & 0 deletions sentry_sdk/integrations/langgraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,8 @@ def _set_usage_data(span: "sentry_sdk.tracing.Span", messages: "Any") -> None:
input_tokens = 0
output_tokens = 0
total_tokens = 0
cached_tokens = 0
reasoning_tokens = 0

for message in messages:
response_metadata = message.get("response_metadata")
Expand All @@ -446,6 +448,12 @@ def _set_usage_data(span: "sentry_sdk.tracing.Span", messages: "Any") -> None:
output_tokens += int(token_usage.get("completion_tokens", 0))
total_tokens += int(token_usage.get("total_tokens", 0))

input_details = token_usage.get("prompt_tokens_details") or {}
cached_tokens += int(input_details.get("cached_tokens") or 0)

output_details = token_usage.get("completion_tokens_details") or {}
reasoning_tokens += int(output_details.get("reasoning_tokens") or 0)

@cursor cursor Bot Jul 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Langgraph misses LangChain token shapes

Medium Severity

_set_usage_data only reads cached_tokens and reasoning_tokens from OpenAI-style prompt_tokens_details / completion_tokens_details. It never checks LangChain’s input_token_details / output_token_details or the cache_read / reasoning keys that _extract_tokens handles in the Langchain integration, so cached and reasoning usage can be omitted on Langgraph spans.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit cb26c51. Configure here.


set_on_span = (
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
)
Expand All @@ -462,6 +470,12 @@ def _set_usage_data(span: "sentry_sdk.tracing.Span", messages: "Any") -> None:
total_tokens,
)

if cached_tokens > 0:
set_on_span(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cached_tokens)

if reasoning_tokens > 0:
set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens)


def _set_response_model_name(span: "sentry_sdk.tracing.Span", messages: "Any") -> None:
if len(messages) == 0:
Expand Down
55 changes: 55 additions & 0 deletions tests/integrations/langchain/test_langchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4977,3 +4977,58 @@ def test_transform_list_with_legacy_image_url(self):
"mime_type": "image/jpeg",
"content": "/9j/4AAQ...",
}


def test_extract_tokens_includes_cached_and_reasoning_details():
from sentry_sdk.integrations.langchain import _extract_tokens

# LangChain usage_metadata shape
usage = {
"input_tokens": 100,
"output_tokens": 50,
"total_tokens": 150,
"input_token_details": {"cache_read": 40},
"output_token_details": {"reasoning": 10},
}
assert _extract_tokens(usage) == (100, 50, 150, 40, 10)

# OpenAI-style details shape
usage = {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"prompt_tokens_details": {"cached_tokens": 30},
"completion_tokens_details": {"reasoning_tokens": 5},
}
assert _extract_tokens(usage) == (100, 50, 150, 30, 5)

# No details present
usage = {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}
assert _extract_tokens(usage) == (1, 2, 3, None, None)


def test_record_token_usage_sets_cached_and_reasoning_span_data():
from unittest.mock import MagicMock

from sentry_sdk.consts import SPANDATA
from sentry_sdk.integrations.langchain import _record_token_usage

span = MagicMock(spec=["set_data"])
response = MagicMock()
response.llm_output = None
response.generations = []
response.usage = None
response.token_usage = None
response.message = None
response.usage_metadata = {
"input_tokens": 100,
"output_tokens": 50,
"total_tokens": 150,
"input_token_details": {"cache_read": 40},
"output_token_details": {"reasoning": 10},
}

_record_token_usage(span, response)

span.set_data.assert_any_call(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, 40)
span.set_data.assert_any_call(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, 10)
37 changes: 37 additions & 0 deletions tests/integrations/langgraph/test_langgraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2132,3 +2132,40 @@ def test_graph_bubble_up_ignored(sentry_init, capture_items):
model.invoke([HumanMessage(content="hi")])

assert len(events) == 0


def test_set_usage_data_includes_cached_and_reasoning_tokens():
from unittest.mock import MagicMock

from sentry_sdk.consts import SPANDATA
from sentry_sdk.integrations.langgraph import _set_usage_data

span = MagicMock(spec=["set_data"])
messages = [
{
"response_metadata": {
"token_usage": {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"prompt_tokens_details": {"cached_tokens": 40},
"completion_tokens_details": {"reasoning_tokens": 10},
}
}
},
{
"response_metadata": {
"token_usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
}
}
},
]

_set_usage_data(span, messages)

span.set_data.assert_any_call(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, 110)
span.set_data.assert_any_call(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, 40)
span.set_data.assert_any_call(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, 10)