Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion src/nullrun/__version__.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,31 @@
SDK_MIN_VERSION_FOR_V3 = "0.12.0". Recommended upgrade path:
0.13.1 -> 0.13.2 (typing-only change for end users; visible
delta is the per-file mypy table in pyproject.toml).
v3.16 / 0.13.4 (2026-07-08) -- bug-fix: complete the LangChain
usage-extraction elif-chain.
Pre-fix extract_usage_from_response walked the 4 source branches
if-hasattr-usage_metadata ... elif-hasattr-generations ...
elif-hasattr-usage ... elif-hasattr-response_metadata. A LangChain
AIMessage can carry token info on multiple attributes at once.
When the first branch's hasattr returned True but the value was
empty or 0/0/0 (streaming init state, some provider wrappers),
every subsequent elif was skipped and the SDK shipped tokens=0
to the backend -- making the LLM call invisible on the dashboard.
Switched all 4 source branches to plain if so each one attempts
its read; later branches naturally overwrite the zero default when
the earlier branch value is empty. New regression test
test_extract_usage_metadata_zero_response_metadata_real.
39 tests in test_langgraph_callback.py still pass; no
regression in test_extractors.py or
test_instrumentation_phase41.py. Wire format is unchanged.
Recommended upgrade path: 0.13.3 -> 0.13.4. No SDK_MIN_VERSION
bump; backends on 1.0.0 keep working unchanged.
"""

__version__ = "0.13.3"
__version__ = "0.13.4"
__platform_version__ = "1.0.0"
20 changes: 16 additions & 4 deletions src/nullrun/instrumentation/langgraph.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic
}

# For callback-based LLMResult, check generations[0][0].message.usage_metadata
elif hasattr(response, 'generations') and response.generations:
if hasattr(response, 'generations') and response.generations:
first_gen = response.generations[0][0] if response.generations else None
if first_gen and hasattr(first_gen, 'message'):
msg = first_gen.message
Expand All @@ -233,7 +233,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic
}

# Try response.usage (Anthropic, standard OpenAI format)
elif hasattr(response, 'usage') and response.usage:
if hasattr(response, 'usage') and response.usage:
usage_raw = response.usage
if isinstance(usage_raw, dict):
usage["input_tokens"] = usage_raw.get('input_tokens', 0) or 0
Expand All @@ -251,8 +251,20 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic
'total_tokens': usage["total_tokens"],
}

# All 4 sources above are `if` (not `elif`) because the same
# response can carry token info on multiple attributes (e.g.
# `usage_metadata = {}` plus `response_metadata.token_usage =
# {real tokens}`). `elif` would silently drop the
# `response_metadata` branch whenever the previous branch's
# hasattr() returned True with an empty value. The first
# non-empty source wins; later branches may overwrite (LangChain
# providers in practice never put conflicting numbers on two
# attributes of the same response, so a "last-wins" is safe
# in practice; see the `_extract_usage` docstring for the
# priority order rationale).
#
# Try response_metadata (some providers) - also check llm_output for LLMResult
elif hasattr(response, 'response_metadata'):
if hasattr(response, 'response_metadata'):
resp_meta = response.response_metadata
if isinstance(resp_meta, dict):
# Some providers put token info here
Expand All @@ -269,7 +281,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic
usage["total_tokens"] = token_usage.get('total_tokens', 0) or 0
usage["raw_usage"] = dict(token_usage)
# Check llm_output for LLMResult (callback case)
elif hasattr(response, 'llm_output') and response.llm_output:
if hasattr(response, 'llm_output') and response.llm_output:
token_usage = response.llm_output.get('token_usage', {})
if isinstance(token_usage, dict):
usage["input_tokens"] = (
Expand Down
31 changes: 31 additions & 0 deletions tests/test_langgraph_callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,37 @@ def test_extract_usage_metadata_dict_form():
assert usage["has_usage"] is True


def test_extract_usage_metadata_zero_response_metadata_real() -> None:
"""2026-07-08: regression for the `elif`-chain token-extraction bug.

The pre-fix extractor walked `if hasattr(... usage_metadata): ...
elif hasattr(... response_metadata): ...`. If a LangChain AIMessage
carried an empty `usage_metadata` (0/0/0) but a populated
`response_metadata.token_usage` (the real numbers), the elif skipped
`response_metadata` and the SDK shipped `tokens=0` to the backend —
making the LLM call invisible on the dashboard.

After the fix, all 4 source branches are `if` (not `elif`) so the
populated `response_metadata` overwrites the empty `usage_metadata`.
The test asserts the non-zero numbers come through to the wire shape.
"""
response = SimpleNamespace(
usage_metadata={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
response_metadata={
"token_usage": {
"prompt_tokens": 26,
"completion_tokens": 48,
"total_tokens": 74,
}
},
)
usage = extract_usage_from_response(response, provider="openai", model="gpt-4.1-mini")
assert usage["input_tokens"] == 26, f"expected 26, got {usage['input_tokens']}"
assert usage["output_tokens"] == 48, f"expected 48, got {usage['output_tokens']}"
assert usage["total_tokens"] == 74, f"expected 74, got {usage['total_tokens']}"
assert usage["has_usage"] is True


def test_extract_usage_metadata_object_form():
"""Object with .input_tokens / .output_tokens / .total_tokens attrs."""
response = SimpleNamespace(
Expand Down
Loading