From 4840a3318eb8b9bc47306459c0f0a3fe8258cc7a Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 8 Jul 2026 11:29:11 +0400 Subject: [PATCH 1/2] fix(instrumentation): all 4 usage sources are 'if' not 'elif' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-fix `extract_usage_from_response` walked the 4 source branches as an elif chain: ``` if hasattr(response, 'usage_metadata'): # empty dict elif hasattr(response, 'generations'): # LLMResult elif hasattr(response, 'usage'): elif hasattr(response, 'response_metadata'): # real tokens ``` A LangChain AIMessage can carry token info on multiple attributes at once. When the first branch's `hasattr` returns True but the value is an empty / 0/0/0 dict (LangChain-internal init state for streaming, callbacks, some provider wrappers), every subsequent `elif` was skipped — including the one carrying the real token counts. Reproducer (covered by the new test): ``` response.usage_metadata = {input_tokens: 0, output_tokens: 0, total_tokens: 0} response.response_metadata = {token_usage: {prompt_tokens: 26, completion_tokens: 48, total_tokens: 74}} # Pre-fix: ships tokens=0 to /track. LLM call invisible on dashboard. # Post-fix: ships tokens=74, input_tokens=26, output_tokens=48. ``` Switched all 4 source branches to plain `if` so each one attempts its read; later branches naturally overwrite the zero default when the earlier branch's value is empty. In practice LangChain providers never put conflicting token counts on two attributes of the same response, so last-wins is safe; the docstring on the function is updated to spell that out. Added `test_extract_usage_metadata_zero_response_metadata_real` that pins the regression: empty usage_metadata + populated response_metadata must surface the real numbers. All 39 tests in test_langgraph_callback.py still pass; test_extractors.py + test_instrumentation_phase41.py also green (no regression in the wire-shape normalization testers). Co-Authored-By: Hermes --- src/nullrun/instrumentation/langgraph.py | 20 ++++++++++++--- tests/test_langgraph_callback.py | 31 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/nullrun/instrumentation/langgraph.py b/src/nullrun/instrumentation/langgraph.py index 81aa74f..a011877 100644 --- a/src/nullrun/instrumentation/langgraph.py +++ b/src/nullrun/instrumentation/langgraph.py @@ -211,7 +211,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic } # For callback-based LLMResult, check generations[0][0].message.usage_metadata - elif hasattr(response, 'generations') and response.generations: + if hasattr(response, 'generations') and response.generations: first_gen = response.generations[0][0] if response.generations else None if first_gen and hasattr(first_gen, 'message'): msg = first_gen.message @@ -233,7 +233,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic } # Try response.usage (Anthropic, standard OpenAI format) - elif hasattr(response, 'usage') and response.usage: + if hasattr(response, 'usage') and response.usage: usage_raw = response.usage if isinstance(usage_raw, dict): usage["input_tokens"] = usage_raw.get('input_tokens', 0) or 0 @@ -251,8 +251,20 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic 'total_tokens': usage["total_tokens"], } + # All 4 sources above are `if` (not `elif`) because the same + # response can carry token info on multiple attributes (e.g. + # `usage_metadata = {}` plus `response_metadata.token_usage = + # {real tokens}`). `elif` would silently drop the + # `response_metadata` branch whenever the previous branch's + # hasattr() returned True with an empty value. The first + # non-empty source wins; later branches may overwrite (LangChain + # providers in practice never put conflicting numbers on two + # attributes of the same response, so a "last-wins" is safe + # in practice; see the `_extract_usage` docstring for the + # priority order rationale). + # # Try response_metadata (some providers) - also check llm_output for LLMResult - elif hasattr(response, 'response_metadata'): + if hasattr(response, 'response_metadata'): resp_meta = response.response_metadata if isinstance(resp_meta, dict): # Some providers put token info here @@ -269,7 +281,7 @@ def extract_usage_from_response(response: Any, provider: str, model: str) -> dic usage["total_tokens"] = token_usage.get('total_tokens', 0) or 0 usage["raw_usage"] = dict(token_usage) # Check llm_output for LLMResult (callback case) - elif hasattr(response, 'llm_output') and response.llm_output: + if hasattr(response, 'llm_output') and response.llm_output: token_usage = response.llm_output.get('token_usage', {}) if isinstance(token_usage, dict): usage["input_tokens"] = ( diff --git a/tests/test_langgraph_callback.py b/tests/test_langgraph_callback.py index b55fa4b..8075e85 100644 --- a/tests/test_langgraph_callback.py +++ b/tests/test_langgraph_callback.py @@ -46,6 +46,37 @@ def test_extract_usage_metadata_dict_form(): assert usage["has_usage"] is True +def test_extract_usage_metadata_zero_response_metadata_real() -> None: + """2026-07-08: regression for the `elif`-chain token-extraction bug. + + The pre-fix extractor walked `if hasattr(... usage_metadata): ... + elif hasattr(... response_metadata): ...`. If a LangChain AIMessage + carried an empty `usage_metadata` (0/0/0) but a populated + `response_metadata.token_usage` (the real numbers), the elif skipped + `response_metadata` and the SDK shipped `tokens=0` to the backend — + making the LLM call invisible on the dashboard. + + After the fix, all 4 source branches are `if` (not `elif`) so the + populated `response_metadata` overwrites the empty `usage_metadata`. + The test asserts the non-zero numbers come through to the wire shape. + """ + response = SimpleNamespace( + usage_metadata={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + response_metadata={ + "token_usage": { + "prompt_tokens": 26, + "completion_tokens": 48, + "total_tokens": 74, + } + }, + ) + usage = extract_usage_from_response(response, provider="openai", model="gpt-4.1-mini") + assert usage["input_tokens"] == 26, f"expected 26, got {usage['input_tokens']}" + assert usage["output_tokens"] == 48, f"expected 48, got {usage['output_tokens']}" + assert usage["total_tokens"] == 74, f"expected 74, got {usage['total_tokens']}" + assert usage["has_usage"] is True + + def test_extract_usage_metadata_object_form(): """Object with .input_tokens / .output_tokens / .total_tokens attrs.""" response = SimpleNamespace( From c17a582c68bf368a8462f23dd8e78c758a7e34db Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 8 Jul 2026 12:07:24 +0400 Subject: [PATCH 2/2] chore(release): bump version 0.13.3 -> 0.13.4 Pairs with 4840a33 (fix-instrumentation-langchain-elif-chain). Wire format unchanged; pure version bump + changelog entry so the SDK_MIN_VERSION floor is up to date with the bug fix. Co-Authored-By: Hermes --- src/nullrun/__version__.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index 0068198..9b1f1ff 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -276,7 +276,31 @@ SDK_MIN_VERSION_FOR_V3 = "0.12.0". Recommended upgrade path: 0.13.1 -> 0.13.2 (typing-only change for end users; visible delta is the per-file mypy table in pyproject.toml). + +v3.16 / 0.13.4 (2026-07-08) -- bug-fix: complete the LangChain +usage-extraction elif-chain. + +Pre-fix extract_usage_from_response walked the 4 source branches +if-hasattr-usage_metadata ... elif-hasattr-generations ... +elif-hasattr-usage ... elif-hasattr-response_metadata. A LangChain +AIMessage can carry token info on multiple attributes at once. +When the first branch's hasattr returned True but the value was +empty or 0/0/0 (streaming init state, some provider wrappers), +every subsequent elif was skipped and the SDK shipped tokens=0 +to the backend -- making the LLM call invisible on the dashboard. + +Switched all 4 source branches to plain if so each one attempts +its read; later branches naturally overwrite the zero default when +the earlier branch value is empty. New regression test +test_extract_usage_metadata_zero_response_metadata_real. + +39 tests in test_langgraph_callback.py still pass; no +regression in test_extractors.py or +test_instrumentation_phase41.py. Wire format is unchanged. + +Recommended upgrade path: 0.13.3 -> 0.13.4. No SDK_MIN_VERSION +bump; backends on 1.0.0 keep working unchanged. """ -__version__ = "0.13.3" +__version__ = "0.13.4" __platform_version__ = "1.0.0"