diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index 0068198..9b1f1ff 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -276,7 +276,31 @@ SDK_MIN_VERSION_FOR_V3 = "0.12.0". Recommended upgrade path: 0.13.1 -> 0.13.2 (typing-only change for end users; visible delta is the per-file mypy table in pyproject.toml). + +v3.16 / 0.13.4 (2026-07-08) -- bug-fix: complete the LangChain +usage-extraction elif-chain. + +Pre-fix extract_usage_from_response walked the 4 source branches +if-hasattr-usage_metadata ... elif-hasattr-generations ... +elif-hasattr-usage ... elif-hasattr-response_metadata. A LangChain +AIMessage can carry token info on multiple attributes at once. +When the first branch's hasattr returned True but the value was +empty or 0/0/0 (streaming init state, some provider wrappers), +every subsequent elif was skipped and the SDK shipped tokens=0 +to the backend -- making the LLM call invisible on the dashboard. + +Switched all 4 source branches to plain if so each one attempts +its read; later branches naturally overwrite the zero default when +the earlier branch value is empty. New regression test +test_extract_usage_metadata_zero_response_metadata_real. + +39 tests in test_langgraph_callback.py still pass; no +regression in test_extractors.py or +test_instrumentation_phase41.py. Wire format is unchanged. + +Recommended upgrade path: 0.13.3 -> 0.13.4. No SDK_MIN_VERSION +bump; backends on 1.0.0 keep working unchanged. """ -__version__ = "0.13.3" +__version__ = "0.13.4" __platform_version__ = "1.0.0" diff --git a/src/nullrun/instrumentation/langgraph.py b/src/nullrun/instrumentation/langgraph.py index 81aa74f..a011877 100644 --- a/src/nullrun/instrumentation/langgraph.py +++ b/src/nullrun/instrumentation/langgraph.py @@ -211,7 +211,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic } # For callback-based LLMResult, check generations[0][0].message.usage_metadata - elif hasattr(response, 'generations') and response.generations: + if hasattr(response, 'generations') and response.generations: first_gen = response.generations[0][0] if response.generations else None if first_gen and hasattr(first_gen, 'message'): msg = first_gen.message @@ -233,7 +233,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic } # Try response.usage (Anthropic, standard OpenAI format) - elif hasattr(response, 'usage') and response.usage: + if hasattr(response, 'usage') and response.usage: usage_raw = response.usage if isinstance(usage_raw, dict): usage["input_tokens"] = usage_raw.get('input_tokens', 0) or 0 @@ -251,8 +251,20 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic 'total_tokens': usage["total_tokens"], } + # All 4 sources above are `if` (not `elif`) because the same + # response can carry token info on multiple attributes (e.g. + # `usage_metadata = {}` plus `response_metadata.token_usage = + # {real tokens}`). `elif` would silently drop the + # `response_metadata` branch whenever the previous branch's + # hasattr() returned True with an empty value. The first + # non-empty source wins; later branches may overwrite (LangChain + # providers in practice never put conflicting numbers on two + # attributes of the same response, so a "last-wins" is safe + # in practice; see the `_extract_usage` docstring for the + # priority order rationale). + # # Try response_metadata (some providers) - also check llm_output for LLMResult - elif hasattr(response, 'response_metadata'): + if hasattr(response, 'response_metadata'): resp_meta = response.response_metadata if isinstance(resp_meta, dict): # Some providers put token info here @@ -269,7 +281,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic usage["total_tokens"] = token_usage.get('total_tokens', 0) or 0 usage["raw_usage"] = dict(token_usage) # Check llm_output for LLMResult (callback case) - elif hasattr(response, 'llm_output') and response.llm_output: + if hasattr(response, 'llm_output') and response.llm_output: token_usage = response.llm_output.get('token_usage', {}) if isinstance(token_usage, dict): usage["input_tokens"] = ( diff --git a/tests/test_langgraph_callback.py b/tests/test_langgraph_callback.py index b55fa4b..8075e85 100644 --- a/tests/test_langgraph_callback.py +++ b/tests/test_langgraph_callback.py @@ -46,6 +46,37 @@ def test_extract_usage_metadata_dict_form(): assert usage["has_usage"] is True +def test_extract_usage_metadata_zero_response_metadata_real() -> None: + """2026-07-08: regression for the `elif`-chain token-extraction bug. + + The pre-fix extractor walked `if hasattr(... usage_metadata): ... + elif hasattr(... response_metadata): ...`. If a LangChain AIMessage + carried an empty `usage_metadata` (0/0/0) but a populated + `response_metadata.token_usage` (the real numbers), the elif skipped + `response_metadata` and the SDK shipped `tokens=0` to the backend — + making the LLM call invisible on the dashboard. + + After the fix, all 4 source branches are `if` (not `elif`) so the + populated `response_metadata` overwrites the empty `usage_metadata`. + The test asserts the non-zero numbers come through to the wire shape. + """ + response = SimpleNamespace( + usage_metadata={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + response_metadata={ + "token_usage": { + "prompt_tokens": 26, + "completion_tokens": 48, + "total_tokens": 74, + } + }, + ) + usage = extract_usage_from_response(response, provider="openai", model="gpt-4.1-mini") + assert usage["input_tokens"] == 26, f"expected 26, got {usage['input_tokens']}" + assert usage["output_tokens"] == 48, f"expected 48, got {usage['output_tokens']}" + assert usage["total_tokens"] == 74, f"expected 74, got {usage['total_tokens']}" + assert usage["has_usage"] is True + + def test_extract_usage_metadata_object_form(): """Object with .input_tokens / .output_tokens / .total_tokens attrs.""" response = SimpleNamespace(