From b1d608fd16e5cbc97ba4522520754b79fda43142 Mon Sep 17 00:00:00 2001 From: Dylan Pulver Date: Fri, 21 Aug 2026 14:27:30 +0300 Subject: [PATCH] fix(langchain): Record reasoning tokens as metric --- .../langchain-reasoning-tokens-metric.md | 5 ++ .../langgraph-v1-latest.log-payloads.json | 1 + .../langgraph-v1-latest.span-tree.json | 1 + .../langgraph-v1-latest.span-tree.txt | 1 + .../langgraph-v1.log-payloads.json | 1 + .../__snapshots__/langgraph-v1.span-tree.json | 1 + .../__snapshots__/langgraph-v1.span-tree.txt | 1 + ...ngchain-js-traces-v1-latest.span-tree.json | 6 ++ ...angchain-js-traces-v1-latest.span-tree.txt | 6 ++ ...wrap-langchain-js-traces-v1.span-tree.json | 6 ++ .../wrap-langchain-js-traces-v1.span-tree.txt | 6 ++ .../langchain/callback-handler.test.ts | 62 +++++++++++++++++++ js/src/wrappers/langchain/callback-handler.ts | 4 ++ 13 files changed, 101 insertions(+) create mode 100644 .changeset/langchain-reasoning-tokens-metric.md diff --git a/.changeset/langchain-reasoning-tokens-metric.md b/.changeset/langchain-reasoning-tokens-metric.md new file mode 100644 index 000000000..67b703aed --- /dev/null +++ b/.changeset/langchain-reasoning-tokens-metric.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(langchain): Record reasoning tokens as metric diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json index 159e7c4ae..b64f5bec9 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json @@ -266,6 +266,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": "", "completion_tokens": "", "end": 0, "prompt_cached_tokens": "", diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json index 801834fae..b7429d6fb 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json @@ -113,6 +113,7 @@ "model": "gpt-4o-mini-2024-07-18" }, "metric_keys": [ + "completion_reasoning_tokens", "completion_tokens", "prompt_cached_tokens", "prompt_tokens", diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt index 1c06fb02c..89da2a98e 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt @@ -113,6 +113,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18" │ } │ metric_keys: [ + │ "completion_reasoning_tokens", │ "completion_tokens", │ "prompt_cached_tokens", │ "prompt_tokens", diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json index 2b4b57071..c886ed5aa 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json @@ -265,6 +265,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": "", "completion_tokens": "", "end": 0, "prompt_cached_tokens": "", diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json index 801834fae..b7429d6fb 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json @@ -113,6 +113,7 @@ "model": "gpt-4o-mini-2024-07-18" }, "metric_keys": [ + "completion_reasoning_tokens", "completion_tokens", "prompt_cached_tokens", "prompt_tokens", diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt index 1c06fb02c..89da2a98e 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt @@ -113,6 +113,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18" │ } │ metric_keys: [ + │ "completion_reasoning_tokens", │ "completion_tokens", │ "prompt_cached_tokens", │ "prompt_tokens", diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json index ad63c46a4..23cffa685 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json @@ -156,6 +156,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 12, @@ -422,6 +423,7 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 18, @@ -742,6 +744,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 6, "prompt_cached_tokens": 0, "prompt_tokens": 22, @@ -974,6 +977,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 16, "prompt_cached_tokens": 0, "prompt_tokens": 72, @@ -1233,6 +1237,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 21, "prompt_cached_tokens": 0, "prompt_tokens": 76, @@ -1503,6 +1508,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt index da37b45bb..ba4679a42 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt @@ -155,6 +155,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, @@ -553,6 +554,7 @@ span_tree: │ ] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 18, @@ -716,6 +718,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 6, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 22, @@ -940,6 +943,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 16, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 72, @@ -1191,6 +1195,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 21, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 76, @@ -1457,6 +1462,7 @@ span_tree: "tags": [] } metrics: { + "completion_reasoning_tokens": 0, "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json index 74a535155..9465136eb 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json @@ -156,6 +156,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 12, @@ -422,6 +423,7 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 18, @@ -742,6 +744,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 6, "prompt_cached_tokens": 0, "prompt_tokens": 22, @@ -974,6 +977,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 16, "prompt_cached_tokens": 0, "prompt_tokens": 72, @@ -1233,6 +1237,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 21, "prompt_cached_tokens": 0, "prompt_tokens": 76, @@ -1503,6 +1508,7 @@ "tags": [] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt index cb09946d7..33d3688be 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt @@ -155,6 +155,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, @@ -553,6 +554,7 @@ span_tree: │ ] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 18, @@ -716,6 +718,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 6, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 22, @@ -940,6 +943,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 16, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 72, @@ -1191,6 +1195,7 @@ span_tree: │ "tags": [] │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 21, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 76, @@ -1457,6 +1462,7 @@ span_tree: "tags": [] } metrics: { + "completion_reasoning_tokens": 0, "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, diff --git a/js/src/wrappers/langchain/callback-handler.test.ts b/js/src/wrappers/langchain/callback-handler.test.ts index e441fa1ad..1cf9682ee 100644 --- a/js/src/wrappers/langchain/callback-handler.test.ts +++ b/js/src/wrappers/langchain/callback-handler.test.ts @@ -128,4 +128,66 @@ describe("BraintrustLangChainCallbackHandler metrics", () => { tokens: 12, }); }); + + it("preserves reasoning metrics from message usage metadata", async () => { + const { endLog } = await finishChatModelRun({ + generations: [ + [ + { + message: { + usage_metadata: { + input_tokens: 10, + output_tokens: 20, + total_tokens: 30, + output_token_details: { + reasoning: 16, + }, + }, + }, + }, + ], + ], + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 10, + completion_tokens: 20, + completion_reasoning_tokens: 16, + tokens: 30, + }); + }); + + it("reports zero reasoning tokens rather than dropping the field", async () => { + const { endLog } = await finishChatModelRun({ + generations: [ + [ + { + message: { + usage_metadata: { + input_tokens: 12, + output_tokens: 2, + total_tokens: 14, + input_token_details: { + audio: 0, + cache_read: 0, + }, + output_token_details: { + audio: 0, + reasoning: 0, + }, + }, + }, + }, + ], + ], + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 12, + completion_tokens: 2, + prompt_cached_tokens: 0, + completion_reasoning_tokens: 0, + tokens: 14, + }); + }); }); diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index a0143b338..9bf57bc51 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -523,6 +523,7 @@ function getMetricsFromResponse( } const inputTokenDetails = usageMetadata.input_token_details; + const outputTokenDetails = usageMetadata.output_token_details; return normalizeTokenMetrics({ total_tokens: usageMetadata.total_tokens, prompt_tokens: usageMetadata.input_tokens, @@ -533,6 +534,9 @@ function getMetricsFromResponse( prompt_cached_tokens: isRecord(inputTokenDetails) ? inputTokenDetails.cache_read : undefined, + completion_reasoning_tokens: isRecord(outputTokenDetails) + ? outputTokenDetails.reasoning + : undefined, }); }