diff --git a/apps/insights/src/agent.ts b/apps/insights/src/agent.ts index 445a74d9d..97988b7e4 100644 --- a/apps/insights/src/agent.ts +++ b/apps/insights/src/agent.ts @@ -617,7 +617,11 @@ function validateDefinitionOutcome( isSuccessfulRead(result.output) ) { const parsed = z - .object({ measurement: insightMeasurementSchema }) + .object({ + measurement: insightMeasurementSchema, + savedDefinition: + insightMeasurementSchema.shape.definition.optional(), + }) .safeParse(result.output); if ( parsed.success && @@ -625,7 +629,11 @@ function validateDefinitionOutcome( parsed.data.measurement.websiteId === (input.appContext.websiteId ?? input.appContext.defaultWebsiteId) ) { - current = { id: entity.id, ...parsed.data.measurement.definition }; + current = { + id: entity.id, + ...(parsed.data.savedDefinition ?? + parsed.data.measurement.definition), + }; } } if (result.toolName !== listTool || !isSuccessfulRead(result.output)) { diff --git a/apps/insights/src/investigation-flow.test.ts b/apps/insights/src/investigation-flow.test.ts index 86de091fc..87a4071ca 100644 --- a/apps/insights/src/investigation-flow.test.ts +++ b/apps/insights/src/investigation-flow.test.ts @@ -584,6 +584,7 @@ describe("intelligence agent", () => { "valid", "native-only", "native-after-list", + "native-cohort", "native-lost-conditions", "uninspected-check", "unanchored", @@ -598,6 +599,12 @@ describe("intelligence agent", () => { ...(native ? { conditions: { plan: "paid" } } : {}), })), }; + let thresholdValue = 20; + if (scenario === "unanchored") { + thresholdValue = 99; + } else if (scenario === "wrong-units") { + thresholdValue = 120; + } const check = { metric: "overall_conversion_rate" as const, startDate: scenario === "past-window" ? "2026-07-01" : "2026-07-13", @@ -606,12 +613,7 @@ describe("intelligence agent", () => { threshold: { anchor: "prior_baseline" as const, comparison: "at_or_above" as const, - value: - scenario === "unanchored" - ? 99 - : scenario === "wrong-units" - ? 120 - : 20, + value: thresholdValue, evidenceRef: { source: "signal" as const }, }, }; @@ -636,6 +638,42 @@ describe("intelligence agent", () => { : {}), }, }; + const inspectionResponses: ReturnType[] = []; + if (scenario === "native-only" || scenario === "native-cohort") { + inspectionResponses.push(toolCallsResponse(["get_funnel_analytics"])); + } else if (scenario === "native-after-list") { + inspectionResponses.push( + toolCallsResponse(["list_funnels"]), + toolCallsResponse(["get_funnel_analytics"]) + ); + } else if (scenario !== "uninspected-check") { + inspectionResponses.push( + toolCallsResponse(["list_funnels", "get_funnel_analytics"]) + ); + } + const savedDefinition = + scenario === "native-cohort" ? { savedDefinition: current } : {}; + const measuredDefinition = + scenario === "native-cohort" + ? { + ...current, + filters: [ + { field: "browser_name", operator: "equals", value: "Safari" }, + ], + } + : current; + const nativeMeasurement = native + ? { + ...savedDefinition, + measurement: { + websiteId: "site-1", + definitionId: "checkout", + startDate: "2026-07-05", + endDate: "2026-07-11", + definition: measuredDefinition, + }, + } + : {}; const run = runInsightAgent( { appContext: appContext(), @@ -648,18 +686,7 @@ describe("intelligence agent", () => { { model: new MockLanguageModelV3({ doGenerate: mockValues( - ...(scenario === "uninspected-check" - ? [] - : scenario === "native-only" - ? [toolCallsResponse(["get_funnel_analytics"])] - : scenario === "native-after-list" - ? [ - toolCallsResponse(["list_funnels"]), - toolCallsResponse(["get_funnel_analytics"]), - ] - : [ - toolCallsResponse(["list_funnels", "get_funnel_analytics"]), - ]), + ...inspectionResponses, outputResponse(proposal), outputResponse(proposal), outputResponse(proposal) @@ -671,17 +698,7 @@ describe("intelligence agent", () => { execute: () => ({ completions: 10, entrants: 100, - ...(native - ? { - measurement: { - websiteId: "site-1", - definitionId: "checkout", - startDate: "2026-07-05", - endDate: "2026-07-11", - definition: current, - }, - } - : {}), + ...nativeMeasurement, }), inputSchema: z.object({}).strict(), }), @@ -695,25 +712,31 @@ describe("intelligence agent", () => { ); if ( - !["legacy", "valid", "native-only", "native-after-list"].includes( - scenario - ) + ![ + "legacy", + "valid", + "native-only", + "native-after-list", + "native-cohort", + ].includes(scenario) ) { - await expect(run).rejects.toThrow( - scenario === "uninspected-check" - ? "Until the exact subject is verified" - : scenario === "native-lost-conditions" - ? "preserve existing step conditions" - : scenario === "unanchored" - ? "99" - : "Verification checks require" - ); + let expectedError = "Verification checks require"; + if (scenario === "uninspected-check") { + expectedError = "Until the exact subject is verified"; + } else if (scenario === "native-lost-conditions") { + expectedError = "preserve existing step conditions"; + } else if (scenario === "unanchored") { + expectedError = "99"; + } + await expect(run).rejects.toThrow(expectedError); return; } const result = await run; const expectedExecution = { ...proposal.next.execution, - changes: native ? { steps: proposal.next.execution.changes.steps } : proposal.next.execution.changes, + changes: native + ? { steps: proposal.next.execution.changes.steps } + : proposal.next.execution.changes, }; expect(result.outcome.next).toEqual({ ...proposal.next, diff --git a/packages/ai/src/ai/tools/cohort-read.test.ts b/packages/ai/src/ai/tools/cohort-read.test.ts new file mode 100644 index 000000000..eefb4d172 --- /dev/null +++ b/packages/ai/src/ai/tools/cohort-read.test.ts @@ -0,0 +1,115 @@ +import { expect, spyOn, test } from "bun:test"; +import { asSchema, type ToolExecutionOptions } from "ai"; +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; +import * as rpc from "./utils/rpc"; +const { createFunnelTools } = await import("./funnels"); +const { createGoalTools } = await import("./goals"); +const options: ToolExecutionOptions = { + toolCallId: "synthetic-cohort", + messages: [], + experimental_context: { + websiteId: "synthetic-site", + websiteDomain: "example.invalid", + }, +}; +const cohort = { + filters: [ + { + field: "browser_name" as const, + operator: "equals" as const, + value: "Safari", + }, + ], +}; +test("native cohort reaches the existing RPC procedure", async () => { + const invoke = spyOn(rpc, "callRPCProcedure").mockResolvedValue({ + synthetic: true, + }); + try { + const dates = { startDate: "2026-08-22", endDate: "2026-08-28", cohort }; + const funnel = createFunnelTools().get_funnel_analytics; + const goal = createGoalTools().get_goal_analytics; + if (!funnel.execute || !goal.execute) throw new Error("Missing executor"); + await funnel.execute({ funnelId: "synthetic-funnel", ...dates }, options); + await goal.execute({ goalId: "synthetic-goal", ...dates }, options); + expect( + invoke.mock.calls.map(([router, method, input]) => ({ + router, + method, + input, + })) + ).toEqual([ + { + router: "funnels", + method: "getAnalytics", + input: { + funnelId: "synthetic-funnel", + websiteId: "synthetic-site", + ...dates, + }, + }, + { + router: "goals", + method: "getAnalytics", + input: { + goalId: "synthetic-goal", + websiteId: "synthetic-site", + ...dates, + }, + }, + ]); + } finally { + invoke.mockRestore(); + } +}); +test("cohort rejects tenant and step selectors", () => { + for (const field of [ + "website_id", + "client_id", + "owner_id", + "path", + "event_name", + "browser_name OR 1=1", + ]) + expect( + analyticsCohortSchema.safeParse({ + filters: [{ field, operator: "equals", value: "x" }], + }).success + ).toBe(false); + expect( + analyticsCohortSchema.safeParse({ + filters: [{ ...cohort.filters[0], target: "event" }], + }).success + ).toBe(false); +}); +test("inaccessible website never reaches RPC", async () => { + const tool = createFunnelTools().get_funnel_analytics; + if (!tool.execute) throw new Error("Missing executor"); + const invoke = spyOn(rpc, "callRPCProcedure").mockResolvedValue({ + synthetic: true, + }); + try { + await expect( + tool.execute( + { + funnelId: "synthetic-funnel", + websiteId: "other-tenant", + startDate: "2026-08-22", + endDate: "2026-08-28", + cohort, + }, + options + ) + ).rejects.toThrow("not in this workspace"); + expect(invoke).not.toHaveBeenCalled(); + } finally { + invoke.mockRestore(); + } +}); +test("model JSON schema exposes the read capability", () => { + const schema = asSchema( + createFunnelTools().get_funnel_analytics.inputSchema + ).jsonSchema; + expect(JSON.stringify(schema)).toContain('"browser_name"'); + expect(JSON.stringify(schema)).toContain('"cohort"'); +}); diff --git a/packages/ai/src/ai/tools/funnels.ts b/packages/ai/src/ai/tools/funnels.ts index 71825d61d..3fd72da38 100644 --- a/packages/ai/src/ai/tools/funnels.ts +++ b/packages/ai/src/ai/tools/funnels.ts @@ -1,3 +1,4 @@ +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; import { tool } from "ai"; import { analyticsDateRangeSchema } from "@databuddy/validation"; import { z } from "zod"; @@ -13,6 +14,7 @@ const logger = createToolLogger("Funnels Tools"); const funnelAnalyticsInputSchema = analyticsDateRangeSchema.safeExtend({ funnelId: z.string(), websiteId: z.string().optional(), + cohort: analyticsCohortSchema.optional(), }); export function createFunnelTools() { @@ -45,10 +47,10 @@ export function createFunnelTools() { const getFunnelAnalyticsTool = tool({ description: - "Funnel definition, measured dates and distinct visitor counts: entrants match the first step; completions reach every ordered step. These are visitors, not projects, occurrences or attempts. Reuse matching verified measurements; remeasure stale or conflicting context.", + "Funnel definition, measured dates and distinct visitor counts: entrants match the first step; completions reach every ordered step. These are visitors, not projects, occurrences or attempts. Optional cohort measures browser, device, country or campaign segments without editing the saved definition. Compare cohorts and periods with parallel calls. Reuse matching verified measurements; remeasure stale or conflicting context.", inputSchema: funnelAnalyticsInputSchema, execute: async ( - { funnelId, websiteId: inputWebsiteId, startDate, endDate }, + { funnelId, websiteId: inputWebsiteId, startDate, endDate, cohort }, options ) => { const context = getAppContext(options); @@ -57,7 +59,7 @@ export function createFunnelTools() { return await callRPCProcedure( "funnels", "getAnalytics", - { funnelId, websiteId, startDate, endDate }, + { funnelId, websiteId, startDate, endDate, cohort }, context ); } catch (error) { @@ -80,7 +82,7 @@ export function createFunnelTools() { "Distinct visitors entering the first funnel step and completing its ordered steps, grouped by referrer/source. Counts are visitors, not projects or attempts. Accepts one date range; compare periods with separate calls.", inputSchema: funnelAnalyticsInputSchema, execute: async ( - { funnelId, websiteId: inputWebsiteId, startDate, endDate }, + { funnelId, websiteId: inputWebsiteId, startDate, endDate, cohort }, options ) => { const context = getAppContext(options); @@ -89,7 +91,7 @@ export function createFunnelTools() { return await callRPCProcedure( "funnels", "getAnalyticsByReferrer", - { funnelId, websiteId, startDate, endDate }, + { funnelId, websiteId, startDate, endDate, cohort }, context ); } catch (error) { diff --git a/packages/ai/src/ai/tools/goals.ts b/packages/ai/src/ai/tools/goals.ts index 1cbcd4fa5..4ef4be645 100644 --- a/packages/ai/src/ai/tools/goals.ts +++ b/packages/ai/src/ai/tools/goals.ts @@ -1,3 +1,4 @@ +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; import { tool } from "ai"; import { analyticsDateRangeSchema } from "@databuddy/validation"; import { z } from "zod"; @@ -19,6 +20,7 @@ const goalFilterSchema = z.object({ const goalAnalyticsInputSchema = analyticsDateRangeSchema.safeExtend({ goalId: z.string(), websiteId: z.string().optional(), + cohort: analyticsCohortSchema.optional(), }); const createGoalInputSchema = z.object({ websiteId: z.string(), @@ -65,10 +67,10 @@ export function createGoalTools() { const getGoalAnalyticsTool = tool({ description: - "Goal definition, measured dates and distinct visitor counts. total_users_entered: website page-view visitors matching filters except event_name. total_users_completed: visitors matching the goal. overall_conversion_rate: completed / entered percent, not login or attempt success. Reuse matching verified measurements; remeasure stale or conflicting context.", + "Goal definition, measured dates and distinct visitor counts. total_users_entered: website page-view visitors matching filters except event_name. total_users_completed: visitors matching the goal. overall_conversion_rate: completed / entered percent, not login or attempt success. Optional cohort measures browser, device, country or campaign segments without editing the saved definition. Compare cohorts and periods with parallel calls. Reuse matching verified measurements; remeasure stale or conflicting context.", inputSchema: goalAnalyticsInputSchema, execute: async ( - { goalId, websiteId: inputWebsiteId, startDate, endDate }, + { goalId, websiteId: inputWebsiteId, startDate, endDate, cohort }, options ) => { const context = getAppContext(options); @@ -77,7 +79,7 @@ export function createGoalTools() { return await callRPCProcedure( "goals", "getAnalytics", - { goalId, websiteId, startDate, endDate }, + { goalId, websiteId, startDate, endDate, cohort }, context ); } catch (error) { diff --git a/packages/rpc/src/lib/analytics-cohort.integration.test.ts b/packages/rpc/src/lib/analytics-cohort.integration.test.ts new file mode 100644 index 000000000..4ccde566b --- /dev/null +++ b/packages/rpc/src/lib/analytics-cohort.integration.test.ts @@ -0,0 +1,317 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { randomUUIDv7 } from "bun"; +import { clickHouse } from "@databuddy/db/clickhouse"; +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; +import { + getTotalWebsiteUsers, + processFunnelAnalytics, + processFunnelAnalyticsByReferrer, + processFunnelConversionCounts, + processGoalAnalytics, + type AnalyticsStep, +} from "./analytics-utils"; + +const enabled = process.env.CLICKHOUSE_COHORT_INTEGRATION_TESTS === "true"; +const suite = enabled ? describe : describe.skip; +const prefix = `cohort-${randomUUIDv7()}`; +const websiteId = `${prefix}-site`; +const otherWebsiteId = `${prefix}-other`; +const boundaryWebsiteId = `${prefix}-boundary`; +const windows = [ + { startDate: "2026-08-22", endDate: "2026-08-28 23:59:59" }, + { startDate: "2026-08-29", endDate: "2026-09-04 23:59:59" }, +] as const; +const steps = [ + { + step_number: 1, + name: "Project created", + type: "EVENT", + target: "project_created", + }, + { + step_number: 2, + name: "Report delivered", + type: "EVENT", + target: "first_report_delivered", + }, +] satisfies [AnalyticsStep, AnalyticsStep]; +const savedFilter = { field: "country", operator: "equals", value: "US" }; +function filters(browser: string) { + return [ + savedFilter, + ...analyticsCohortSchema.parse({ + filters: [{ field: "browser_name", operator: "equals", value: browser }], + }).filters, + ]; +} +function params(window: (typeof windows)[number], site = websiteId) { + return { ...window, websiteId: site }; +} + +suite("native cohort SQL against disposable ClickHouse", () => { + beforeAll(async () => { + // Do not trust .env or a normal integration database for this insertion suite. + if (process.env.CLICKHOUSE_URL !== "http://127.0.0.1:18129") + throw new Error( + "Cohort integration inserts require the disposable loopback endpoint on port 18129" + ); + const events: Record[] = []; + const custom: Record[] = []; + for (const site of [websiteId, otherWebsiteId]) + for (const [period, window] of windows.entries()) + for (const browser of ["Safari", "Chrome"]) { + let completions = 20; + if (site === otherWebsiteId) { + completions = 500; + } else if (browser === "Chrome") { + completions = 80; + } else if (period === 0) { + completions = 100; + } + for (let i = 0; i < 550; i++) { + // Deliberately collide anonymous/session/profile IDs across the two tenants. + const identity = `${prefix}-${period}-${browser}-${i}`; + const anonymous = `${identity}-anon`; + const session = `${identity}-session`; + const profile = i % 3 === 0 ? `${identity}-profile` : ""; + const country = i < 500 ? "US" : "CA"; + events.push({ + id: randomUUIDv7(), + client_id: site, + event_name: "screen_view", + anonymous_id: anonymous, + session_id: session, + time: `${window.startDate} 11:59:00`, + created_at: `${window.startDate} 11:59:00`, + url: "https://example.invalid/start", + path: "/start", + ip: "", + user_agent: "synthetic", + properties: "{}", + country, + browser_name: browser, + profile_id: "", + }); + const base = { + owner_id: site, + website_id: site, + properties: "{}", + anonymous_id: anonymous, + session_id: session, + profile_id: profile, + }; + if (profile) + custom.push({ + ...base, + timestamp: `${window.startDate} 11:59:30`, + event_name: "identify", + }); + const entry = { + ...base, + profile_id: "", + timestamp: `${window.startDate} 12:00:00`, + event_name: "project_created", + }; + custom.push(entry); + if (i === 0) custom.push({ ...entry }); + const valid = i < completions || i >= 500; + const completion = { + ...base, + anonymous_id: i % 3 === 0 || i % 3 === 1 ? null : anonymous, + session_id: i % 3 === 0 ? null : session, + timestamp: `${window.startDate} ${valid ? "12:01:00" : "11:58:00"}`, + event_name: "first_report_delivered", + }; + custom.push(completion); + if (i === 0) custom.push({ ...completion }); + } + } + // Later steps must remain reachable even when their browser/country differs. + for (const [path, browser, country, time] of [ + ["/start", "Safari", "US", "12:00:00"], + ["/finish", "Chrome", "CA", "12:01:00"], + ]) + events.push({ + id: randomUUIDv7(), + client_id: boundaryWebsiteId, + event_name: "screen_view", + anonymous_id: `${prefix}-boundary-anon`, + session_id: `${prefix}-boundary-session`, + time: `2026-08-29 ${time}`, + created_at: `2026-08-29 ${time}`, + url: `https://example.invalid${path}`, + path, + ip: "", + user_agent: "synthetic", + properties: "{}", + country, + browser_name: browser, + }); + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: events, + }); + await clickHouse.insert({ + table: "analytics.custom_events", + format: "JSONEachRow", + values: custom, + }); + }, 30_000); + + for (const [browser, expected] of [ + ["Safari", [100, 20]], + ["Chrome", [80, 80]], + ] as const) + it(`${browser}: ordered completions and stable entrants in both exact windows`, async () => { + for (const period of [0, 1] as const) { + const window = windows[period]; + const result = await processFunnelAnalytics( + steps, + filters(browser), + params(window) + ); + expect(result.total_users_entered).toBe(500); + expect(result.total_users_completed).toBe(expected[period]); + expect(result.overall_conversion_rate).toBe(expected[period] / 5); + expect(result.steps_analytics.map((step) => step.users)).toEqual([ + 500, + expected[period], + ]); + expect( + result.time_series?.map((row) => [ + row.date, + row.users, + row.conversions, + ]) + ).toEqual([[window.startDate, 500, expected[period]]]); + } + }, 30_000); + + it("country filter remains ANDed with the requested browser cohort", async () => { + const cohort = analyticsCohortSchema.parse({ + filters: [{ field: "browser_name", operator: "equals", value: "Safari" }], + }).filters; + const result = await processFunnelAnalytics( + steps, + cohort, + params(windows[1]) + ); + expect(result.total_users_entered).toBe(550); + expect(result.total_users_completed).toBe(70); + const withSavedCountry = await processFunnelAnalytics( + steps, + [savedFilter, ...cohort], + params(windows[1]) + ); + expect(withSavedCountry.total_users_entered).toBe(500); + expect(withSavedCountry.total_users_completed).toBe(20); + }); + it("other tenant's colliding identities and completions cannot inflate this tenant", async () => { + const other = await processFunnelAnalytics( + steps, + filters("Safari"), + params(windows[1], otherWebsiteId) + ); + const original = await processFunnelAnalytics( + steps, + filters("Safari"), + params(windows[1]) + ); + expect([other.total_users_entered, other.total_users_completed]).toEqual([ + 500, 500, + ]); + expect([ + original.total_users_entered, + original.total_users_completed, + ]).toEqual([500, 20]); + }); + it("filters the entry context without filtering a later step's changed context", async () => { + const pages: AnalyticsStep[] = [ + { step_number: 1, name: "Start", target: "/start", type: "PAGE_VIEW" }, + { step_number: 2, name: "Finish", target: "/finish", type: "PAGE_VIEW" }, + ]; + const result = await processFunnelAnalytics( + pages, + filters("Safari"), + params(windows[1], boundaryWebsiteId) + ); + expect([result.total_users_entered, result.total_users_completed]).toEqual([ + 1, 1, + ]); + const chrome = await processFunnelAnalytics( + pages, + filters("Chrome"), + params(windows[1], boundaryWebsiteId) + ); + expect([chrome.total_users_entered, chrome.total_users_completed]).toEqual([ + 0, 0, + ]); + }); + it("supports union and exclusion cohort selectors without changing the denominator", async () => { + const union = await processFunnelAnalytics( + steps, + [ + savedFilter, + { field: "browser_name", operator: "in", value: ["Safari", "Chrome"] }, + ], + params(windows[1]) + ); + expect([union.total_users_entered, union.total_users_completed]).toEqual([ + 1000, 100, + ]); + const excluded = await processFunnelAnalytics( + steps, + [ + savedFilter, + { field: "browser_name", operator: "not_in", value: ["Safari"] }, + ], + params(windows[1]) + ); + expect([ + excluded.total_users_entered, + excluded.total_users_completed, + ]).toEqual([500, 80]); + }); + it("native referrer analytics respects the same entry browser and saved filters", async () => { + const result = await processFunnelAnalyticsByReferrer( + steps, + filters("Safari"), + params(windows[1]) + ); + expect( + result.referrer_analytics.map((row) => [ + row.total_users, + row.completed_users, + ]) + ).toEqual([[500, 20]]); + }); + it("deep and detector counts agree for the exact cohort", async () => { + const result = await processFunnelConversionCounts( + steps, + filters("Safari"), + params(windows[1]) + ); + expect([result.entrants, result.completions, result.rate]).toEqual([ + 500, 20, 4, + ]); + }); + it("goal cohort counts use matching page-view visitors and contextual completions", async () => { + const window = windows[1]; + const entrants = await getTotalWebsiteUsers( + websiteId, + window.startDate, + "2026-09-04", + filters("Safari") + ); + const result = await processGoalAnalytics( + [{ ...steps[1], step_number: 1 }], + filters("Safari"), + params(window), + entrants + ); + expect([result.total_users_entered, result.total_users_completed]).toEqual([ + 500, 20, + ]); + }); +}); diff --git a/packages/rpc/src/routers/analytics-measurement.test.ts b/packages/rpc/src/routers/analytics-measurement.test.ts index 529153fc1..2f164ef6b 100644 --- a/packages/rpc/src/routers/analytics-measurement.test.ts +++ b/packages/rpc/src/routers/analytics-measurement.test.ts @@ -1,4 +1,4 @@ -import { beforeAll, beforeEach, expect, mock, test } from "bun:test"; +import { beforeAll, beforeEach, expect, mock, spyOn, test } from "bun:test"; import { createProcedureClient, os } from "@orpc/server"; import type { Context } from "../orpc"; import type { @@ -46,6 +46,11 @@ const goalQuery = mock( ..._args: Parameters ): Promise>> => metrics ); +const referrerQuery = mock( + async (..._args: Parameters) => ({ + referrer_analytics: [], + }) +); const entrants = mock( async (..._args: Parameters) => 200 ); @@ -78,7 +83,7 @@ beforeAll(async () => { processGoalAnalytics: goalQuery, processFunnelAnalytics: query, getTotalWebsiteUsers: entrants, - processFunnelAnalyticsByReferrer: query, + processFunnelAnalyticsByReferrer: referrerQuery, queryLinkVisitorIds: async () => [], })); mock.module("../middleware/track-mutation", () => ({ @@ -96,9 +101,11 @@ beforeAll(async () => { beforeEach(() => { cache.clear(); - query.mockClear(); + query.mockReset(); + query.mockImplementation(async () => metrics); goalQuery.mockClear(); entrants.mockClear(); + referrerQuery.mockClear(); }); const savedFilter = { field: "country", operator: "equals", value: "US" }; @@ -218,3 +225,165 @@ for (const kind of ["goal", "funnel"] as const) { expect(measuredQuery).toHaveBeenCalledTimes(4); }); } + +for (const kind of ["goal", "funnel"] as const) { + test(`${kind} cohort read preserves saved definition, clips dates and separates cached cohorts`, async () => { + const row = definition(); + const browserFilter = { + field: "browser_name" as const, + operator: "equals" as const, + value: "Safari", + }; + const cohort = { filters: [browserFilter] }; + const measuredQuery = kind === "goal" ? goalQuery : query; + const read = () => + kind === "goal" + ? createProcedureClient(goalsRouter.getAnalytics, { + context: context(row, kind), + })({ ...period, goalId: row.id, cohort }) + : createProcedureClient(funnelsRouter.getAnalytics, { + context: context(row, kind), + })({ ...period, funnelId: row.id, cohort }); + const result = await read(); + expect(result.cohort).toEqual(cohort); + expect(result.savedDefinition.filters).toEqual([savedFilter]); + expect(result.measurement.definition.filters).toContainEqual( + cohort.filters[0] + ); + expect(result.measurement.startDate).toBe("2026-09-01"); + if ("steps" in result.savedDefinition) + expect(result.savedDefinition.steps).toEqual(row.steps); + await read(); + expect(measuredQuery).toHaveBeenCalledTimes(1); + browserFilter.value = "Chrome"; + await read(); + expect(measuredQuery).toHaveBeenCalledTimes(2); + expect(row.filters).toEqual([savedFilter]); + }); +} + +// Synthetic added world: event marginals cannot establish ordered funnel completion. +// This validates the RPC/cache/read contract, not ClickHouse numerical execution. +test("browser cohort comparison exposes Safari loss and retains unchanged Chrome control", async () => { + const row = definition(); + row.ignoreHistoricData = false; + query.mockImplementation(async (_steps, filters, params) => { + const browser = filters.find((f) => f.field === "browser_name")?.value; + const previous = params.startDate === "2026-08-22"; + let completed = previous ? 180 : 100; + if (browser === "Safari") { + completed = previous ? 100 : 20; + } else if (browser === "Chrome") { + completed = 80; + } + return { + ...metrics, + total_users_entered: browser ? 500 : 1000, + total_users_completed: completed, + overall_conversion_rate: (completed / (browser ? 500 : 1000)) * 100, + }; + }); + const read = createProcedureClient(funnelsRouter.getAnalytics, { + context: context(row, "funnel"), + }); + const output = []; + for (const browser of ["Safari", "Chrome"] as const) { + for (const window of [ + { startDate: "2026-08-22", endDate: "2026-08-28" }, + { startDate: "2026-08-29", endDate: "2026-09-04" }, + ]) { + const input = { + ...window, + websiteId: "site-a", + funnelId: row.id, + cohort: { + filters: [ + { + field: "browser_name" as const, + operator: "equals" as const, + value: browser, + }, + ], + }, + }; + const result = await read(input); + output.push({ input, result }); + } + } + expect(output.map((v) => v.result.total_users_completed)).toEqual([ + 100, 20, 80, 80, + ]); + expect(output.every((v) => v.result.total_users_entered === 500)).toBe(true); + expect( + output.every((v) => v.result.savedDefinition.filters.length === 1) + ).toBe(true); +}); + +test("funnel reads restore default measurements after a cohort-specific mock", async () => { + const row = definition(); + const read = createProcedureClient(funnelsRouter.getAnalytics, { + context: context(row, "funnel"), + }); + const result = await read({ ...period, funnelId: row.id }); + expect(result).toMatchObject(metrics); +}); + +test("link analytics rejects a cohort before querying definitions or analytics", async () => { + const row = definition(); + const requestContext = context(row, "funnel"); + const select = spyOn(requestContext.db, "select"); + const read = createProcedureClient(funnelsRouter.getAnalyticsByLink, { + context: requestContext, + }); + try { + await expect( + read({ + ...period, + funnelId: row.id, + linkId: "link-a", + // @ts-expect-error Deliberately exercise the unsupported cohort boundary. + cohort: { + filters: [ + { field: "browser_name", operator: "equals", value: "Safari" }, + ], + }, + }) + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + expect(select).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + } finally { + select.mockRestore(); + } +}); + +test("referrer cohorts return actual dates and saved definition with independently cached measurements", async () => { + const row = definition(); + const browserFilter = { + field: "browser_name" as const, + operator: "equals" as const, + value: "Safari", + }; + const cohort = { filters: [browserFilter] }; + const read = createProcedureClient(funnelsRouter.getAnalyticsByReferrer, { + context: context(row, "funnel"), + }); + const input = { ...period, funnelId: row.id, cohort }; + const result = await read(input); + expect(result.measurement.startDate).toBe("2026-09-01"); + expect(result.measurement.definition.filters).toEqual([ + savedFilter, + ...cohort.filters, + ]); + expect(result.savedDefinition.filters).toEqual([savedFilter]); + expect(result.cohort).toEqual(cohort); + await read(input); + expect(referrerQuery).toHaveBeenCalledTimes(1); + browserFilter.value = "Chrome"; + await read(input); + expect(referrerQuery).toHaveBeenCalledTimes(2); + const signupStep = row.steps[1]; + if (!signupStep) throw new Error("Missing signup step"); + signupStep.target = "/activated"; + await read(input); + expect(referrerQuery).toHaveBeenCalledTimes(3); +}); diff --git a/packages/rpc/src/routers/funnels.ts b/packages/rpc/src/routers/funnels.ts index 55055ca22..1a695d6df 100644 --- a/packages/rpc/src/routers/funnels.ts +++ b/packages/rpc/src/routers/funnels.ts @@ -1,3 +1,4 @@ +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; import { insightMeasurementSchema } from "@databuddy/shared/insights"; import { successOutputSchema } from "../lib/schemas"; import { and, desc, eq, isNull, sql } from "@databuddy/db"; @@ -57,10 +58,12 @@ const filterSchema = z.object({ type Filter = z.infer; const funnelAnalyticsInputSchema = analyticsDateRangeSchema.safeExtend({ + cohort: analyticsCohortSchema.optional(), funnelId: z.string(), websiteId: z.string(), }); const funnelAnalyticsByLinkInputSchema = funnelAnalyticsInputSchema.safeExtend({ + cohort: z.undefined(), linkId: z.string(), }); @@ -108,17 +111,25 @@ async function loadFunnelAnalyticsQuery( funnel.ignoreHistoricData ); + const filters = [ + ...((funnel.filters as Filter[]) || []), + ...(input.cohort?.filters ?? []), + ]; return { + savedDefinition: insightMeasurementSchema.shape.definition.parse({ + ...funnel, + filters: funnel.filters ?? [], + }), measurement: insightMeasurementSchema.parse({ websiteId: input.websiteId, definitionId: funnel.id, startDate: effectiveStartDate, endDate, - definition: { ...funnel, filters: funnel.filters ?? [] }, + definition: { ...funnel, filters }, }), effectiveStartDate, endDate, - filters: (funnel.filters as Filter[]) || [], + filters, queryParams: { endDate: `${endDate} 23:59:59`, startDate: effectiveStartDate, @@ -512,11 +523,13 @@ export const funnelsRouter = { .output( funnelAnalyticsOutputSchema.extend({ measurement: insightMeasurementSchema, + savedDefinition: insightMeasurementSchema.shape.definition, + cohort: analyticsCohortSchema.optional(), }) ) .use(withWebsiteRead) .handler(async ({ context, input }) => { - const { filters, queryParams, steps, measurement } = + const { filters, queryParams, steps, measurement, savedDefinition } = await loadFunnelAnalyticsQuery(context.db, input); const analytics = await funnelCache.withCache({ @@ -526,7 +539,12 @@ export const funnelsRouter = { tag: `funnel:${input.funnelId}`, queryFn: () => processFunnelAnalytics(steps, filters, queryParams), }); - return { ...analytics, measurement }; + return { + ...analytics, + measurement, + savedDefinition, + cohort: input.cohort, + }; }), getAnalyticsByReferrer: publicProcedure @@ -539,20 +557,32 @@ export const funnelsRouter = { tags: ["Funnels"], }) .input(funnelAnalyticsInputSchema) - .output(funnelAnalyticsByReferrerOutputSchema) + .output( + funnelAnalyticsByReferrerOutputSchema.extend({ + measurement: insightMeasurementSchema, + savedDefinition: insightMeasurementSchema.shape.definition, + cohort: analyticsCohortSchema.optional(), + }) + ) .use(withWebsiteRead) .handler(async ({ context, input }) => { - const { effectiveStartDate, endDate, filters, queryParams, steps } = + const { filters, queryParams, steps, measurement, savedDefinition } = await loadFunnelAnalyticsQuery(context.db, input); - return funnelCache.withCache({ - key: `analyticsByReferrer:${input.funnelId}:${effectiveStartDate}:${endDate}`, + const analytics = await funnelCache.withCache({ + key: `analyticsByReferrer:${JSON.stringify(measurement)}`, ttl: ANALYTICS_CACHE_TTL, tables: ["funnelDefinitions"], tag: `funnel:${input.funnelId}`, queryFn: () => processFunnelAnalyticsByReferrer(steps, filters, queryParams), }); + return { + ...analytics, + measurement, + savedDefinition, + cohort: input.cohort, + }; }), getAnalyticsByLink: publicProcedure diff --git a/packages/rpc/src/routers/goals.ts b/packages/rpc/src/routers/goals.ts index 5bc8a9249..f218eec46 100644 --- a/packages/rpc/src/routers/goals.ts +++ b/packages/rpc/src/routers/goals.ts @@ -1,3 +1,4 @@ +import { analyticsCohortSchema } from "@databuddy/shared/analytics-filters"; import { insightMeasurementSchema } from "@databuddy/shared/insights"; import { successOutputSchema } from "../lib/schemas"; import { and, desc, eq, inArray, isNull } from "@databuddy/db"; @@ -50,6 +51,7 @@ const filterSchema = z.object({ type Filter = z.infer; const goalAnalyticsInputSchema = analyticsDateRangeSchema.safeExtend({ + cohort: analyticsCohortSchema.optional(), filters: z.array(filterSchema).optional(), goalId: z.string(), websiteId: z.string(), @@ -388,6 +390,8 @@ export const goalsRouter = { .output( goalAnalyticsOutputSchema.extend({ measurement: insightMeasurementSchema, + savedDefinition: insightMeasurementSchema.shape.definition, + cohort: analyticsCohortSchema.optional(), }) ) .use(withWebsiteRead) @@ -418,6 +422,7 @@ export const goalsRouter = { const combinedFilters = [ ...(input.filters ?? []), + ...(input.cohort?.filters ?? []), ...((goal.filters as Filter[]) || []), ]; const measurement = insightMeasurementSchema.parse({ @@ -460,7 +465,15 @@ export const goalsRouter = { ); }, }); - return { ...analytics, measurement }; + return { + ...analytics, + measurement, + cohort: input.cohort, + savedDefinition: insightMeasurementSchema.shape.definition.parse({ + ...goal, + filters: goal.filters ?? [], + }), + }; }), bulkAnalytics: publicProcedure diff --git a/packages/shared/src/analytics-filters.ts b/packages/shared/src/analytics-filters.ts index d3c826f28..105edcd75 100644 --- a/packages/shared/src/analytics-filters.ts +++ b/packages/shared/src/analytics-filters.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export const goalFunnelFilterFields = [ { value: "event_name", label: "Event Name" }, { value: "path", label: "Page Path" }, @@ -23,3 +25,33 @@ export type GoalFunnelFilterField = export const goalFunnelFilterFieldSet: ReadonlySet = new Set( goalFunnelFilterFields.map((f) => f.value) ); + +/** Read-only segmentation, restricted to context rather than funnel step selectors. */ +export const analyticsCohortSchema = z + .strictObject({ + filters: z + .array( + z.strictObject({ + field: z.enum([ + "browser_name", + "device_type", + "os_name", + "country", + "referrer", + "utm_source", + "utm_medium", + "utm_campaign", + ]), + operator: z.enum(["equals", "not_equals", "in", "not_in"]), + value: z.union([ + z.string().min(1), + z.array(z.string().min(1)).min(1).max(50), + ]), + }) + ) + .min(1) + .max(8), + }) + .describe( + "Read-only cohort. Funnel filters select first-step visitors; subsequent ordered steps may have different context. Goal filters select page-view visitors and matching completions. Saved filters are ANDed. These are visitors, not attempts." + );