From b6731cabad2e0f8332ab1c62d5e8ec52aba1252a Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:26:39 +0200 Subject: [PATCH 1/6] fix(ingest): leave Mastra scorer runs unstamped A Mastra agent with `scorers` exports each scorer run as a root trace of its own with no conversation id. Stamped as `mastra`, every run became a `trace:` agent session: its `scorer_run`/`scorer_step` spans were counted as tool calls (the exporter lowercases unknown span types into `gen_ai.operation.name`, and "code-tool-call-accuracy-scorer" contains "tool"), and an LLM judge's agent showed up as a second agent. In one EU capture, three real conversations produced about 24 of these. A scorer names its run's type (`mastra.span.type = scorer_*`) and stamps the run it grades as `mastra.metadata.targetTraceId`, which Mastra copies onto every span beneath it, the judge's agent and model call included. Those spans no longer get a vendor stamp, so they never reach `ai_trace_index` and no agent session, list or detail, sees them. --- apps/ingest/src/ai_session.rs | 95 +++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/apps/ingest/src/ai_session.rs b/apps/ingest/src/ai_session.rs index a35d0a63e..0bb98f642 100644 --- a/apps/ingest/src/ai_session.rs +++ b/apps/ingest/src/ai_session.rs @@ -33,6 +33,10 @@ //! `maple_ai.usage.*` buckets, whatever convention its emitter reported under //! — see `ai_session/facts.rs` and `ai_session/usage.rs`. //! +//! One vendor's evaluations are left unstamped entirely: a Mastra scorer run +//! grades a finished agent run and is not a conversation (see +//! [`run_predicates`]). +//! //! Detection is ordered first-match over the vendor predicates below; the //! session ID is the first non-empty session-granularity attribute for the //! matched vendor. A vendor with no session-level key of its own (its @@ -426,6 +430,9 @@ struct SpanEvidence<'a> { langsmith: bool, llamaindex: bool, mastra: bool, + /// A Mastra scorer's span: its `scorer_run`/`scorer_step`, or any span it + /// ran (an LLM judge's agent and model calls) - see [`run_predicates`]. + mastra_scorer: bool, agno: bool, agent_framework: bool, executor: bool, @@ -656,6 +663,13 @@ fn absorb_key<'a>(ev: &mut SpanEvidence<'a>, attr: &'a KeyValue, b0: u8) { b'm' => { if key.starts_with("mastra.") { ev.mastra = true; + // A scorer stamps the run it grades as metadata, and Mastra + // copies a span's metadata onto every span beneath it. + if key == "mastra.metadata.targetTraceId" + || (key == "mastra.span.type" && value_str(attr).starts_with("scorer_")) + { + ev.mastra_scorer = true; + } } else if key.starts_with("message.") { ev.message = true; } else if key == "model_request_parameters" { @@ -1039,6 +1053,13 @@ fn run_predicates( .iter() .chain(UNKNOWN_TIER) .find(|vendor| (vendor.detect)(&ctx))?; + // A Mastra scorer grades a finished run in a trace of its own, with no + // conversation id: stamped, every scorer run became a `trace:` session, its + // `scorer_*` spans counted as tool calls and an LLM judge as a second agent. + // It is an evaluation, not a conversation, so none of it is agent work. + if vendor.id == "mastra" && ev.mastra_scorer { + return None; + } let session_id = vendor .session_keys .iter() @@ -2132,6 +2153,80 @@ mod tests { ); } + #[test] + fn mastra_scorer_runs_are_not_agent_work() { + // capture `blind-ts-mastra` (EU, 2026-09-29): a live scorer grades the + // agent's run in a root trace of its own. Every span of it carries the + // graded run as `mastra.metadata.target*`, including the LLM judge's + // agent and model call beneath a `scorer_step`. + const SCOPE: &str = "@mastra/otel-exporter"; + const TARGET: (&str, &str) = ( + "mastra.metadata.targetTraceId", + "7df8c67d7b9310062270dae3ddc6e010", + ); + let spans: &[(&str, &[(&str, &str)])] = &[ + ( + "scorer_run code-tool-call-accuracy-scorer", + &[ + ("gen_ai.operation.name", "scorer_run"), + ("mastra.span.type", "scorer_run"), + TARGET, + ], + ), + ( + "scorer_step translation-quality-scorer", + &[ + ("gen_ai.operation.name", "scorer_step"), + ("mastra.span.type", "scorer_step"), + TARGET, + ], + ), + ( + "invoke_agent judge", + &[ + ("gen_ai.operation.name", "invoke_agent"), + ("gen_ai.agent.name", "judge"), + ("mastra.span.type", "agent_run"), + TARGET, + ], + ), + ( + "chat openai/gpt-5-mini", + &[ + ("gen_ai.operation.name", "chat"), + ("gen_ai.request.model", "openai/gpt-5-mini"), + ("gen_ai.usage.input_tokens", "352"), + ("mastra.span.type", "model_inference"), + TARGET, + ], + ), + // A scorer run with no graded trace still names its own type. + ( + "scorer_run code-tool-call-accuracy-scorer", + &[("mastra.span.type", "scorer_run")], + ), + ]; + for (name, span) in spans { + assert!( + classify(SCOPE, name, span, &[]).is_none(), + "{name} was stamped" + ); + } + // The run it graded is still the agent's. + classified( + SCOPE, + "invoke_agent translator", + &[ + ("gen_ai.operation.name", "invoke_agent"), + ("gen_ai.conversation.id", "maple-demo-conversation-1"), + ("mastra.span.type", "agent_run"), + ], + &[], + "mastra", + Some("maple-demo-conversation-1"), + ); + } + #[test] fn effect_guarded_span_names_require_the_effect_sdk_resource() { classified( From 6132e735cf59a10583c5c35409ceceee4f9a0e98 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:33:24 +0200 Subject: [PATCH 2/6] fix(agent-sessions): read "tool" off a span name only when no operation is named A span whose `gen_ai.operation.name` is outside the convention's set fell back to its name, and a "tool" anywhere in it made it a tool call. A span that names an operation of its own has already said what it is: the Mastra exporter writes its unknown span types as operations (`scorer_step code-tool-call-accuracy-scorer`), and LangSmith's OTel export names LangGraph's `tools` node and `HumanInTheLoopMiddleware.wrap_tool_call` `chain`, double-counting every real tool call beneath them. The name needle now applies only to spans that name no operation; a tool name attribute is still a tool call whatever the operation. Checked against every replayed framework capture in the EU org (September): each real tool call is `execute_tool` or names no operation (OpenAI Agents TS, Claude Code, Vercel `ai.toolCall`), and the only unknown-operation spans the needle matched were the Mastra scorer and LangSmith `chain` wrappers above. The rule is decided at ingest: `facts::is_tool_call` (`maple_ai.tool_call`) and the unknown-dialect model-call fallback in `usage::named_like_a_model_call` (`maple_ai.llm_call`). `classifyAiSpan` keeps the same edge for rows ingested before the stamps, until they age out of the 30-day TTL. --- apps/ingest/src/ai_session/facts.rs | 31 ++++++++++++---- apps/ingest/src/ai_session/usage.rs | 20 ++++++++-- .../agent-sessions/src/session-turns.test.ts | 37 +++++++++++++++++++ packages/agent-sessions/src/session-turns.ts | 9 ++++- 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/apps/ingest/src/ai_session/facts.rs b/apps/ingest/src/ai_session/facts.rs index 2c9910924..5e907fa7c 100644 --- a/apps/ingest/src/ai_session/facts.rs +++ b/apps/ingest/src/ai_session/facts.rs @@ -392,12 +392,15 @@ fn agent_name(facts: &Facts, vendor: &str, span_name: &str) -> Option { } /// A tool call: the convention's tool operation, or, under an operation the -/// convention does not name, a tool name or a span name saying "tool". +/// convention does not name, a tool name. A span name saying "tool" counts +/// only when no operation is named: one that names its own (LangSmith's +/// `chain` over LangGraph's `tools` node, a Mastra `scorer_step`) has said +/// what it is. fn is_tool_call(facts: &Facts, span_name: &str) -> bool { let op = facts.operation(); op == "execute_tool" || (!usage::KNOWN_OPS.contains(&op) - && (facts.has_tool_name() || name_has(span_name, "tool"))) + && (facts.has_tool_name() || (op.is_empty() && name_has(span_name, "tool")))) } /// Mark a stamped tool call as failed after the fact: Claude Code records a @@ -812,14 +815,28 @@ mod tests { assert_eq!(agno[0], pairs(&[("maple_ai.llm_call", "0")])); } - /// Outside the convention's operations a tool name or a span name saying - /// "tool" makes a tool call; a memory operation never does. + /// Outside the convention's operations a tool name makes a tool call, and + /// a span name saying "tool" does only when no operation is named (the + /// Mastra scorer and LangSmith `chain` wrappers name one); a memory + /// operation never does. #[test] fn tool_calls_by_name_under_unknown_operations() { let got = stamps( "support-agent", vec![ - span("run_tools", &[("gen_ai.operation.name", "workflow_step")]), + span("run_tools", &[("traceloop.span.kind", "task")]), + span( + "mcp_tool_call search", + &[ + ("gen_ai.operation.name", "mcp_tool_call"), + ("gen_ai.tool.name", "search"), + ], + ), + span( + "scorer_step code-tool-call-accuracy-scorer", + &[("gen_ai.operation.name", "scorer_step")], + ), + span("tools", &[("gen_ai.operation.name", "chain")]), span( "search_memory notes", &[ @@ -829,8 +846,8 @@ mod tests { ), ], ); - assert!(has(&got[0], TOOL_CALL_ATTR)); - assert!(!has(&got[1], TOOL_CALL_ATTR)); + let tool_calls: Vec = got.iter().map(|span| has(span, TOOL_CALL_ATTR)).collect(); + assert_eq!(tool_calls, [true, true, false, false, false]); } /// A value the warehouse Map holds as a string counts whatever its OTLP diff --git a/apps/ingest/src/ai_session/usage.rs b/apps/ingest/src/ai_session/usage.rs index a3ab7bdf9..41aa4929e 100644 --- a/apps/ingest/src/ai_session/usage.rs +++ b/apps/ingest/src/ai_session/usage.rs @@ -208,7 +208,7 @@ fn named_like_a_model_call(op: &str, span_name: &str, facts: &Facts) -> bool { if KNOWN_OPS.contains(&op) { return false; } - if facts.has_tool_name() || name_has(span_name, "tool") { + if facts.has_tool_name() || (op.is_empty() && name_has(span_name, "tool")) { return false; } if name_has(span_name, "agent") || name_has(span_name, "workflow") { @@ -1368,7 +1368,8 @@ mod tests { /// `captures/jev_agents`: `decide` is jev's own model call, found by the /// unknown-dialect fallback; an unknown workflow op is not a call, nor is a - /// memory operation naming its embedding model. + /// memory operation naming its embedding model. A "tool" in the name of a + /// span that names its own operation does not rule it out. #[test] fn unknown_dialect_calls_by_name_and_model() { let stamped = stamp_spans( @@ -1411,6 +1412,15 @@ mod tests { ("gen_ai.request.model", "text-embedding-3-small"), ], ), + ( + "rank_tools openai/gpt-4o-mini", + &[ + ("gen_ai.operation.name", "rank"), + ("gen_ai.request.model", "openai/gpt-4o-mini"), + ("gen_ai.usage.input_tokens", "52"), + ("gen_ai.usage.output_tokens", "9"), + ], + ), ], ); assert!(stamped.iter().all(|span| span.vendor == "unknown:genai")); @@ -1422,7 +1432,11 @@ mod tests { .iter() .map(|span| span.llm_call.as_deref()) .collect(); - assert_eq!(calls, [Some("1"), Some("1"), Some("0"), Some("0")]); + assert_eq!( + calls, + [Some("1"), Some("1"), Some("0"), Some("0"), Some("1")] + ); + assert_eq!(stamped[4].buckets, Some([52, 0, 0, 9, 0])); } // --- Guards and the write itself ------------------------------------ diff --git a/packages/agent-sessions/src/session-turns.test.ts b/packages/agent-sessions/src/session-turns.test.ts index 94d6e0dfe..1ac18ab87 100644 --- a/packages/agent-sessions/src/session-turns.test.ts +++ b/packages/agent-sessions/src/session-turns.test.ts @@ -567,6 +567,43 @@ describe("classifyAiSpan", () => { expect(named("chat gpt-5")).toBe("inference") }) + it("does not read 'tool' off the name of a span that names an unknown operation", () => { + // `blind-ts-mastra` (EU, 2026-09-29): the Mastra exporter lowercases a span + // type the convention has no name for into `gen_ai.operation.name`; a + // LangSmith OTel `chain` wraps LangGraph's `tools` node. + const unknownOp = (spanName: string, operationName: string) => + makeSpan({ + spanId: "a", + startMs: 0, + durationMs: 1, + spanName, + vendorId: "mastra", + genAi: { operationName }, + }) + + for (const span of [ + unknownOp("scorer_run code-tool-call-accuracy-scorer", "scorer_run"), + unknownOp("scorer_step code-tool-call-accuracy-scorer", "scorer_step"), + unknownOp("tools", "chain"), + unknownOp("HumanInTheLoopMiddleware.wrap_tool_call", "chain"), + ]) { + expect(classifyAiSpan(span)).toBe("agent") + expect(isLlmCall(span)).toBe(false) + } + // A tool name is still a tool call, whatever the operation says. + expect( + classifyAiSpan( + makeSpan({ + spanId: "a", + startMs: 0, + durationMs: 1, + spanName: "mcp_tool_call search", + genAi: { operationName: "mcp_tool_call", toolName: "search" }, + }), + ), + ).toBe("tool") + }) + it("classifies a span with no AI signal as other, whatever it is called", () => { const httpSpan = makeSpan({ spanId: "a", diff --git a/packages/agent-sessions/src/session-turns.ts b/packages/agent-sessions/src/session-turns.ts index 2c58c88e2..be6e7decd 100644 --- a/packages/agent-sessions/src/session-turns.ts +++ b/packages/agent-sessions/src/session-turns.ts @@ -85,9 +85,14 @@ export function classifyAiSpan(span: AiSessionSpan): AiSpanCategory { // `gen_ai.operation.name` is optional and plenty of instrumentations skip it. // The span name is the next best evidence: by convention it leads with the - // operation ("execute_tool read_file", "chat gpt-5"). + // operation ("execute_tool read_file", "chat gpt-5"). Except for "tool": a + // span naming an operation the convention does not know (LangSmith's + // `chain`, a Mastra `scorer_step`) has said what it is, and the LangGraph + // `tools` node or a `code-tool-call-accuracy-scorer` is not a tool call. const name = span.spanName.toLowerCase() - if (span.genAi.toolName !== undefined || name.includes("tool")) return "tool" + if (span.genAi.toolName !== undefined || (operation === undefined && name.includes("tool"))) { + return "tool" + } if (name.includes("agent") || name.includes("workflow")) return "agent" if (spanModel(span) !== undefined || name.includes("chat") || name.includes("completion")) { return "inference" From 00c9656fed8bdf7a4cd2ffc36c7c0e841123081f Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:36:02 +0200 Subject: [PATCH 3/6] fix(agent-sessions): a sessionless trace with no model call and no named agent is not a session Any trace with one vendor-stamped span became an agent session, filed as `trace:` when it carried no session id. Over September in the EU org, every such trace with no model call and no named agent was plumbing: Mastra scorer runs, lone Spring AI advisor spans, and OpenRouter's connection test. The US org had none. The list, its distributions and the facets now keep a trace only when it carries a session id, made a model call or ran a named agent (a HAVING on the per-trace index level every one of them shares). A trace that has a session id still joins its session whatever it holds. The tools pages still count such a trace's tool calls, and a `trace:` link to one still opens. --- ...dex-materialization.clickhouse.e2e.test.ts | 21 ++++++++ .../src/__sql_baseline__/integrations.sql | 48 ++++++++++++------- .../src/ai/ai-sessions.test.ts | 21 ++++++-- .../src/ai/ai-sessions.ts | 18 +++++++ 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts index c0b4d5165..c8e5586f3 100644 --- a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts @@ -1070,6 +1070,27 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { ) }) + it("files a sessionless trace with no model call and no named agent under no session", async () => { + const classification = { ...WINDOW, orgId: CLASSIFICATION_ORG_ID } + const page = compileUnsafe(Integrations.aiSessionPageQuery(), classification) + const rows = Effect.runSync(page.decodeRows(await runJson(page.sql))) + // The scorer run is gone; the LangGraph trace stays, on its named agent. + assert.deepStrictEqual( + rows.map((row) => [row.sessionId, row.toolCalls]), + [[`${MAPLE_AI_TRACE_SESSION_PREFIX}${LANGGRAPH_TRACE}`, 2]], + ) + + const facets = compileUnionUnsafe(Integrations.aiSessionFacetsQuery(), classification) + const vendors = Effect.runSync(facets.decodeRows(await runJson(facets.sql))) + .filter((row) => row.facetType === "vendor") + .map((row) => [row.name, row.count]) + .sort() + assert.deepStrictEqual(vendors, [ + ["langchain", 1], + ["vercel_ai_sdk", 1], + ]) + }) + it("distributes the sessions over each range the way the page measures them", async () => { const compiled = compileUnsafe(Integrations.aiSessionDistributionsQuery(), WINDOW) const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index cd626fa77..4ee6ed16f 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -45,7 +45,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' - GROUP BY traceId) AS agent_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) GROUP BY traceId) AS session_traces INNER JOIN (SELECT @@ -76,7 +77,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' - GROUP BY traceId) AS agent_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) AS index_traces ON session_traces.traceId = index_traces.traceId GROUP BY sessionId ORDER BY startTime DESC @@ -130,7 +132,8 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(DeploymentEnv IN ('production')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 AND countIf(Model IN ('gpt-5.5')) > 0 AND countIf(AgentName IN ('billing-agent')) > 0 AND countIf(ToolName IN ('send_email')) > 0 @@ -166,7 +169,8 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(DeploymentEnv IN ('production')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 AND countIf(Model IN ('gpt-5.5')) > 0 AND countIf(AgentName IN ('billing-agent')) > 0 AND countIf(ToolName IN ('send_email')) > 0 @@ -224,7 +228,8 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(VendorId IN ('eve')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) GROUP BY traceId) AS session_traces @@ -257,7 +262,8 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(VendorId IN ('eve')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) AS index_traces ON session_traces.traceId = index_traces.traceId GROUP BY sessionId @@ -327,7 +333,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS index_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS index_traces GROUP BY sessionId) AS window_sessions) AS netted_sessions) AS session_measures) AS measured_sessions WHERE tupleElement(measured, 2) > 0 GROUP BY measure @@ -346,7 +353,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -363,7 +371,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -380,7 +389,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -397,7 +407,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -414,7 +425,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -431,7 +443,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS facet_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -526,7 +539,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY traceId) AS index_traces + GROUP BY traceId + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS index_traces GROUP BY sessionId ORDER BY agentStart DESC, sessionId ASC LIMIT 50) AS ranked_sessions) AS netted_sessions @@ -623,7 +637,8 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(VendorId IN ('eve')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0 AND countIf(DeploymentEnv IN ('production')) > 0 AND countIf(Model IN ('gpt-5.5')) > 0 @@ -738,7 +753,8 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(VendorId IN ('eve')) > 0 + HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS index_traces GROUP BY sessionId ORDER BY agentStart DESC, sessionId ASC diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts index 621a8fc22..9aabb4042 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts @@ -191,7 +191,7 @@ describe("aiSessionPageQuery", () => { // `trace:` whenever its turn-owning span belongs to the other vendor it // calls through. It also has to match `aiSessionFacetsQuery`'s any-span // counting, or the sidebar's number and the page's length disagree. - expect(sql).toContain("HAVING countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("AND countIf(VendorId IN ('eve')) > 0") expect(sql).toContain("AND countIf(ServiceName IN ('maple-slack-agent')) > 0") expect(sql).not.toContain("WHERE VendorId IN") }) @@ -214,7 +214,7 @@ describe("aiSessionPageQuery", () => { // maple-slack-agent spans are different spans — the ordinary case, since a // trace's spans come from several services. Neither name may appear in the // index read's WHERE at all. - expect(sql.split("countIf(").length - 1).toBe(2) + expect(sql.split("countIf(").length - 1).toBe(3) expect(where).not.toContain("VendorId") expect(where).not.toContain("ServiceName") expect(sql).not.toContain("VendorId IN ('eve') AND ServiceName") @@ -223,7 +223,9 @@ describe("aiSessionPageQuery", () => { it("omits the optional filters when none are given", () => { const { sql } = compileUnsafe(aiSessionPageQuery(), params) - expect(sql).not.toContain("HAVING") + // The one HAVING is the rule for which traces are sessions at all. + expect(sql.split("HAVING ").length - 1).toBe(1) + expect(sql).toContain("HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0") expect(sql).not.toContain("VendorId IN") expect(sql).not.toContain("ServiceName IN") // The one WHERE is the index read's; the usage level filters nothing. @@ -686,7 +688,7 @@ describe("aiSessionDetailsQuery", () => { // They must also be the SAME filters the page ran under, or the two stages // resolve traces differently and the join silently loses rows. const [fanOut, detection] = sql.split("TraceId IN (SELECT") - expect(detection).toContain("HAVING countIf(VendorId IN ('eve')) > 0") + expect(detection).toContain("AND countIf(VendorId IN ('eve')) > 0") expect(detection).toContain("AND countIf(ServiceName IN ('maple-slack-agent')) > 0") expect(fanOut).not.toContain("IN ('eve')") }) @@ -879,6 +881,12 @@ describe("aiSessionFacetsQuery", () => { expect(sql.split(`uniqExact(${SESSION_KEY}) AS count`).length - 1).toBe(6) expect(sql.split("GROUP BY traceId").length - 1).toBe(6) expect(sql).not.toContain("uniqExact(SpanAttributes['maple_ai.session.id'])") + // Only the traces the list shows: a sessionless trace with no model call + // and no named agent is no session, so no facet counts it. + expect( + sql.split("HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0").length - + 1, + ).toBe(6) }) it("repeats the org and window predicates on every union branch", () => { @@ -961,7 +969,10 @@ describe("aiSessionDistributionsQuery", () => { expect(sessions).toContain(`${SESSION_KEY} AS sessionId`) expect(sql).toContain("GROUP BY sessionId") // No range, sort or page: every session in the window is placed. - expect(sql).not.toContain("HAVING") + expect(sql.split("HAVING ").length - 1).toBe(1) + expect(traces).toContain( + "HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0", + ) expect(sql).not.toContain("LIMIT") expect(sql).not.toContain("ORDER BY") expect(traces).toContain(`Timestamp >= '${params.startTime}'`) diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index eb689529e..6d6dec182 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -242,6 +242,21 @@ export const orderTuple = (...parts: ReadonlyArray): CH.Expr = export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr): CH.Expr => CH.if_(rawSessionId.eq(""), CH.concat(MAPLE_AI_TRACE_SESSION_PREFIX, traceId), rawSessionId) +/** + * Whether a trace is a session at all, as a HAVING over its index rows: it + * carries a session id (joining whatever session that names), or it made a + * model call or ran a named agent. A sessionless trace with neither is + * framework plumbing that happened to be stamped — a Mastra scorer run's + * `scorer_*` spans, a lone Spring AI advisor span, OpenRouter's connection + * test — and filing it as `trace:` put an empty session in the list for + * every one of them. + */ +const isSessionTraceCond = ($: { + readonly SessionId: CH.Expr + readonly IsLlmCall: CH.Expr + readonly AgentName: CH.Expr +}): CH.Condition => CH.countIf($.SessionId.neq("").or($.IsLlmCall.eq(1)).or($.AgentName.neq(""))).gt(0) + /** * One trace's failed agent spans — `(SpanId, ParentSpanId, IsToolCall)` per * failed index row — for the tool/turn split one level up, which needs the @@ -568,6 +583,7 @@ const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { ]) .groupBy("traceId") .having(($) => [ + isSessionTraceCond($), CH.when(values(opts.vendorIds), (v) => carries(CH.inList($.VendorId, v))), CH.when(values(opts.serviceNames), (v) => carries(CH.inList($.ServiceName, v))), CH.when(values(opts.deploymentEnvs), (v) => carries(CH.inList($.DeploymentEnv, v))), @@ -1099,6 +1115,8 @@ export function aiSessionFacetsQuery(): CHUnionQuery { $.Timestamp.lte(param.dateTimeString("endTime")), ]) .groupBy("traceId") + // The list's population, so a facet never counts a session it cannot show. + .having(($) => [isSessionTraceCond($)]) return fromQuery(perTrace, "facet_traces") .select(($) => ({ From bcb67c7e844d6c0ab9911079db626ddef9ce6a84 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:54:59 +0200 Subject: [PATCH 4/6] fix(agent-sessions): a sessionless trace of tool calls alone is still a session The session rule kept a sessionless trace only if it made a model call or ran a named agent, so a trace of tool calls alone - a tool server whose caller did not propagate its context - disappeared from the list, its facets and its distributions. A tool call is agent work: `IsToolCall = 1` now keeps the trace too. New scorer runs do not come back through it: the gateway leaves them unstamped, and its `maple_ai.tool_call` no longer reads "tool" off the name of a span that names its own operation (`scorer_*`). The e2e seeds the rule's traces with the gateway's stamps, since the index only projects them. --- ...dex-materialization.clickhouse.e2e.test.ts | 82 +++++++++++++++++-- .../src/__sql_baseline__/integrations.sql | 32 ++++---- .../src/ai/ai-sessions.test.ts | 11 ++- .../src/ai/ai-sessions.ts | 17 ++-- 4 files changed, 108 insertions(+), 34 deletions(-) diff --git a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts index c8e5586f3..e79d21ae4 100644 --- a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts @@ -565,6 +565,68 @@ const NETTING_ORG_SPANS: ReadonlyArray = [ }, ] +// Traces the session rule sorts, under a fifth org, stamped as the gateway +// stamps them: a sessionless plumbing span (a Spring AI advisor) that is no +// session, a LangSmith OTel `chain` over LangGraph's `tools` node beside the +// real tool calls, and a tool server's lone tool call. +const SESSION_RULE_ORG_ID = "org_ai_trace_index_e2e_session_rule" +const PLUMBING_TRACE = "aitraceindexe2e000000000000000030" +const LANGGRAPH_TRACE = "aitraceindexe2e000000000000000031" +/** A tool server's own trace: one tool call, no session id, model or agent. */ +const TOOL_ONLY_TRACE = "aitraceindexe2e000000000000000032" + +const sessionRuleSpan = ( + traceId: string, + spanId: string, + name: string, + offsetMs: number, + vendor: string, + attrs: Readonly>, +): SeedSpan => ({ + traceId, + spanId, + name, + ms: BASE_MS + 600_000 + offsetMs, + service: "session-rule-service", + status: "Ok", + attrs: { [MAPLE_AI_VENDOR_ID_ATTR]: vendor, ...attrs }, +}) + +const SESSION_RULE_ORG_SPANS: ReadonlyArray = [ + sessionRuleSpan( + PLUMBING_TRACE, + "span-advisor", + "spring_ai chat_client advisor", + 0, + "spring_ai", + aiGatewayStamps({}), + ), + sessionRuleSpan(LANGGRAPH_TRACE, "span-lg-agent", "invoke_agent weather", 10, "langchain", { + "gen_ai.operation.name": "invoke_agent", + ...aiGatewayStamps({ agentName: "weather" }), + }), + sessionRuleSpan(LANGGRAPH_TRACE, "span-lg-tools", "tools", 11, "langchain", { + "gen_ai.operation.name": "chain", + ...aiGatewayStamps({}), + }), + sessionRuleSpan(LANGGRAPH_TRACE, "span-lg-tool", "get_weather", 13, "langchain", { + "gen_ai.operation.name": "execute_tool", + ...aiGatewayStamps({ toolCall: true, toolName: "get_weather" }), + }), + sessionRuleSpan( + LANGGRAPH_TRACE, + "span-lg-sdk-tool", + "ai.toolCall", + 14, + "vercel_ai_sdk", + aiGatewayStamps({ toolCall: true }), + ), + sessionRuleSpan(TOOL_ONLY_TRACE, "span-tool-only", "execute_tool search", 20, "unknown:genai", { + "gen_ai.operation.name": "execute_tool", + ...aiGatewayStamps({ toolCall: true, toolName: "search" }), + }), +] + const chMap = (attrs: Readonly>): string => `map(${Object.entries(attrs) .flatMap(([key, value]) => [quote(key), quote(value)]) @@ -576,6 +638,7 @@ const seed = async (): Promise => { [FOREIGN_ORG_ID, FOREIGN_SPAN] as const, ...TOOL_FAILURE_ORG_SPANS.map((span) => [TOOL_FAILURE_ORG_ID, span] as const), ...NETTING_ORG_SPANS.map((span) => [NETTING_ORG_ID, span] as const), + ...SESSION_RULE_ORG_SPANS.map((span) => [SESSION_RULE_ORG_ID, span] as const), ] .map( ([orgId, span]) => @@ -621,7 +684,7 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { VendorVersion, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens, ErrorType, StatusMessage, ToolDescription, FailedToolCallResult, ErrorFingerprint != 0 AS HasErrorFingerprint - FROM ai_trace_index WHERE OrgId != ${quote(NETTING_ORG_ID)} ORDER BY Timestamp ASC`, + FROM ai_trace_index WHERE OrgId NOT IN (${quote(NETTING_ORG_ID)}, ${quote(SESSION_RULE_ORG_ID)}) ORDER BY Timestamp ASC`, ) /** The index row a seed span is expected to produce, by name — the @@ -1070,23 +1133,28 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { ) }) - it("files a sessionless trace with no model call and no named agent under no session", async () => { - const classification = { ...WINDOW, orgId: CLASSIFICATION_ORG_ID } - const page = compileUnsafe(Integrations.aiSessionPageQuery(), classification) + it("files a sessionless trace with no model call, tool call or named agent under no session", async () => { + const sessionRule = { ...WINDOW, orgId: SESSION_RULE_ORG_ID } + const page = compileUnsafe(Integrations.aiSessionPageQuery(), sessionRule) const rows = Effect.runSync(page.decodeRows(await runJson(page.sql))) - // The scorer run is gone; the LangGraph trace stays, on its named agent. + // The advisor's trace is gone; the LangGraph trace stays on its named agent, + // and the tool server's trace on its one tool call. assert.deepStrictEqual( rows.map((row) => [row.sessionId, row.toolCalls]), - [[`${MAPLE_AI_TRACE_SESSION_PREFIX}${LANGGRAPH_TRACE}`, 2]], + [ + [`${MAPLE_AI_TRACE_SESSION_PREFIX}${TOOL_ONLY_TRACE}`, 1], + [`${MAPLE_AI_TRACE_SESSION_PREFIX}${LANGGRAPH_TRACE}`, 2], + ], ) - const facets = compileUnionUnsafe(Integrations.aiSessionFacetsQuery(), classification) + const facets = compileUnionUnsafe(Integrations.aiSessionFacetsQuery(), sessionRule) const vendors = Effect.runSync(facets.decodeRows(await runJson(facets.sql))) .filter((row) => row.facetType === "vendor") .map((row) => [row.name, row.count]) .sort() assert.deepStrictEqual(vendors, [ ["langchain", 1], + ["unknown:genai", 1], ["vercel_ai_sdk", 1], ]) }) diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index 4ee6ed16f..f3eea0621 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -46,7 +46,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS agent_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) GROUP BY traceId) AS session_traces INNER JOIN (SELECT @@ -78,7 +78,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS agent_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) AS index_traces ON session_traces.traceId = index_traces.traceId GROUP BY sessionId ORDER BY startTime DESC @@ -132,7 +132,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(DeploymentEnv IN ('production')) > 0 AND countIf(Model IN ('gpt-5.5')) > 0 AND countIf(AgentName IN ('billing-agent')) > 0 @@ -169,7 +169,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(DeploymentEnv IN ('production')) > 0 AND countIf(Model IN ('gpt-5.5')) > 0 AND countIf(AgentName IN ('billing-agent')) > 0 @@ -228,7 +228,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) @@ -262,7 +262,7 @@ SELECT AND Timestamp >= '2026-01-02 10:30:00' AND Timestamp <= '2026-01-02 12:30:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS agent_traces WHERE if(rawSessionId = '', concat('trace:', traceId), rawSessionId) IN ('wrun_sql_catalog', 'trace:7f3a4b5c6d7e8f901234567890abcdef')) AS index_traces ON session_traces.traceId = index_traces.traceId @@ -334,7 +334,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS index_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS index_traces GROUP BY sessionId) AS window_sessions) AS netted_sessions) AS session_measures) AS measured_sessions WHERE tupleElement(measured, 2) > 0 GROUP BY measure @@ -354,7 +354,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -372,7 +372,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -390,7 +390,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -408,7 +408,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -426,7 +426,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -444,7 +444,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS facet_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS facet_traces GROUP BY name ORDER BY count DESC LIMIT 50 @@ -540,7 +540,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0) AS index_traces + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS index_traces GROUP BY sessionId ORDER BY agentStart DESC, sessionId ASC LIMIT 50) AS ranked_sessions) AS netted_sessions @@ -637,7 +637,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0 AND countIf(DeploymentEnv IN ('production')) > 0 @@ -753,7 +753,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY traceId - HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0 + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0 AND countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0) AS index_traces GROUP BY sessionId diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts index 9aabb4042..6a6af5c3a 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts @@ -225,7 +225,9 @@ describe("aiSessionPageQuery", () => { // The one HAVING is the rule for which traces are sessions at all. expect(sql.split("HAVING ").length - 1).toBe(1) - expect(sql).toContain("HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0") + expect(sql).toContain( + "HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0", + ) expect(sql).not.toContain("VendorId IN") expect(sql).not.toContain("ServiceName IN") // The one WHERE is the index read's; the usage level filters nothing. @@ -884,8 +886,9 @@ describe("aiSessionFacetsQuery", () => { // Only the traces the list shows: a sessionless trace with no model call // and no named agent is no session, so no facet counts it. expect( - sql.split("HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0").length - - 1, + sql.split( + "HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0", + ).length - 1, ).toBe(6) }) @@ -971,7 +974,7 @@ describe("aiSessionDistributionsQuery", () => { // No range, sort or page: every session in the window is placed. expect(sql.split("HAVING ").length - 1).toBe(1) expect(traces).toContain( - "HAVING countIf(((SessionId != '' OR IsLlmCall = 1) OR AgentName != '')) > 0", + "HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0", ) expect(sql).not.toContain("LIMIT") expect(sql).not.toContain("ORDER BY") diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index 6d6dec182..35a729848 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -245,17 +245,20 @@ export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr` put an empty session in the list for - * every one of them. + * model call or a tool call, or ran a named agent. A sessionless trace with + * none of these is framework plumbing that happened to be stamped — a lone + * Spring AI advisor span, OpenRouter's connection test — and filing it as + * `trace:` put an empty session in the list for every one of them. A + * trace of tool calls alone (a tool server whose caller did not propagate its + * context) is still agent work, and stays. */ -const isSessionTraceCond = ($: { +export const isSessionTraceCond = ($: { readonly SessionId: CH.Expr readonly IsLlmCall: CH.Expr + readonly IsToolCall: CH.Expr readonly AgentName: CH.Expr -}): CH.Condition => CH.countIf($.SessionId.neq("").or($.IsLlmCall.eq(1)).or($.AgentName.neq(""))).gt(0) +}): CH.Condition => + CH.countIf($.SessionId.neq("").or($.IsLlmCall.eq(1)).or($.IsToolCall.eq(1)).or($.AgentName.neq(""))).gt(0) /** * One trace's failed agent spans — `(SpanId, ParentSpanId, IsToolCall)` per From 316b483a823ded52522b202e5002083d058cb01a Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:55:57 +0200 Subject: [PATCH 5/6] fix(agent-sessions): the tools page counts the sessions list's population The Tools page's Sessions tile and tab count read `traceFacts`, which still counted every stamped trace, while the list now drops sessionless traces with no model call, tool call or named agent. The tile promises the list's number, so `traceFacts` applies the same `isSessionTraceCond`. A trace holding a tool call always passes it, so the tool-call reads that join it lose nothing. --- .../src/__sql_baseline__/integrations.sql | 51 ++++++++++++------- .../src/ai/ai-tools.test.ts | 11 ++++ .../src/ai/ai-tools.ts | 7 ++- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index f3eea0621..586a92035 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -1082,7 +1082,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1146,7 +1147,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1223,7 +1225,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1295,7 +1298,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1350,7 +1354,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1412,7 +1417,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1459,7 +1465,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1499,7 +1506,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1542,7 +1550,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1584,7 +1593,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1631,7 +1641,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1685,7 +1696,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1736,7 +1748,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1783,7 +1796,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1838,7 +1852,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' @@ -1894,7 +1909,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' AND Timestamp <= '2026-01-01 10:30:00' - GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' @@ -1924,7 +1940,8 @@ SELECT WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' - GROUP BY TraceId) AS window_traces + GROUP BY TraceId + HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0) AS window_traces FORMAT JSON -- builder:billing-usage:dailyProductEventCountQuery:default diff --git a/packages/query-engine-integrations/src/ai/ai-tools.test.ts b/packages/query-engine-integrations/src/ai/ai-tools.test.ts index 53ec155ce..a7a3eb3fd 100644 --- a/packages/query-engine-integrations/src/ai/ai-tools.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-tools.test.ts @@ -114,6 +114,17 @@ describe("tool call population", () => { expect(sql).toContain("uniqExact(sessionKey) AS sessions") }) + it("counts the sessions list's population in the window's Sessions tile", () => { + const { sql } = compileUnionUnsafe(aiToolsTotalsQuery({}, ["window"]), totalsParams) + + // The same per-trace rule the list applies, or the tile counts sessions + // the list does not show. + expect(sql).toContain( + "HAVING countIf((((SessionId != '' OR IsLlmCall = 1) OR IsToolCall = 1) OR AgentName != '')) > 0", + ) + expect(sql).toContain(") AS window_traces") + }) + it("scopes every level that reads the table to the org", () => { // Two reads of `ai_trace_index` per aggregation level without a model // filter: the tool calls and the trace facts. A model filter adds the diff --git a/packages/query-engine-integrations/src/ai/ai-tools.ts b/packages/query-engine-integrations/src/ai/ai-tools.ts index 38b7a61c4..a177789aa 100644 --- a/packages/query-engine-integrations/src/ai/ai-tools.ts +++ b/packages/query-engine-integrations/src/ai/ai-tools.ts @@ -76,7 +76,7 @@ import { AiTraceIndex, TraceDetailSpans } from "@maple/query-engine/ch/tables" import { finiteOrZero, isoBucket, leftUTF8 } from "@maple/query-engine/ch/format" import { CHNumber } from "@maple/query-engine/ch/schema" import { aiFieldSourceKeys } from "./ai-integrations" -import { SESSION_ORDER_SENTINEL, orderTuple, sessionKey } from "./ai-sessions" +import { SESSION_ORDER_SENTINEL, isSessionTraceCond, orderTuple, sessionKey } from "./ai-sessions" /** * The page's selection, as every read here takes it. @@ -184,6 +184,10 @@ const parentModels = (window: AiToolsWindow) => * level — for the failure modal, which links each failure to its session. The * overview's aggregates never select them, and ClickHouse prunes unselected * columns of a derived table. + * + * Only the traces the sessions list shows (`isSessionTraceCond`), so the + * Sessions tile counts that list's population. A trace holding a tool call + * always passes it, so no tool call is lost to the join. */ const traceFacts = (window: AiToolsWindow) => from(AiTraceIndex) @@ -206,6 +210,7 @@ const traceFacts = (window: AiToolsWindow) => $.Timestamp.lte(endParam(window)), ]) .groupBy("TraceId") + .having(($) => [isSessionTraceCond($)]) /** * The model a tool call is attributed to: its parent model call's, else its From 9773e7b8e6f0b49202b20e4a7858228776dafb5d Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 29 Sep 2026 19:56:31 +0200 Subject: [PATCH 6/6] fix(ingest): leave a scorer span unstamped whichever instrumentation recorded it The scorer exclusion ran only after the ordered vendor lookup picked `mastra`. A scorer's descendant recorded by another instrumentation, for example a judge's model call under LangSmith's scope, matched `langchain` first and was stamped anyway. Mastra's target metadata marks the span as an evaluation whatever its scope, so the check now runs before any vendor is chosen. --- apps/ingest/src/ai_session.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/ingest/src/ai_session.rs b/apps/ingest/src/ai_session.rs index 0bb98f642..ee81f3577 100644 --- a/apps/ingest/src/ai_session.rs +++ b/apps/ingest/src/ai_session.rs @@ -1043,6 +1043,15 @@ fn run_predicates( ev: &SpanEvidence, span_attrs: &[KeyValue], ) -> Option { + // A Mastra scorer grades a finished run in a trace of its own, with no + // conversation id: stamped, every scorer run became a `trace:` session, its + // `scorer_*` spans counted as tool calls and an LLM judge as a second agent. + // It is an evaluation, not a conversation, so none of it is agent work - + // whichever instrumentation recorded the span (a judge's model call can sit + // under LangSmith's scope, which the vendor order would name first). + if ev.mastra_scorer { + return None; + } let ctx = Ctx { scope, resource, @@ -1053,13 +1062,6 @@ fn run_predicates( .iter() .chain(UNKNOWN_TIER) .find(|vendor| (vendor.detect)(&ctx))?; - // A Mastra scorer grades a finished run in a trace of its own, with no - // conversation id: stamped, every scorer run became a `trace:` session, its - // `scorer_*` spans counted as tool calls and an LLM judge as a second agent. - // It is an evaluation, not a conversation, so none of it is agent work. - if vendor.id == "mastra" && ev.mastra_scorer { - return None; - } let session_id = vendor .session_keys .iter() @@ -2212,6 +2214,15 @@ mod tests { "{name} was stamped" ); } + // A judge's model call recorded by another instrumentation still + // carries the graded run, and is not stamped under that vendor either. + assert!(classify( + "langsmith", + "chat openai/gpt-5-mini", + &[("gen_ai.operation.name", "chat"), TARGET], + &[], + ) + .is_none()); // The run it graded is still the agent's. classified( SCOPE,