diff --git a/SPEC.md b/SPEC.md index 897da1c50..b1fe27e22 100644 --- a/SPEC.md +++ b/SPEC.md @@ -162,7 +162,7 @@ Missing diagnostic access alone is not a coverage finding. Publish a measured mi Customer impact stays explicit about coverage. Anonymous visitor identifiers, sessions, identified profiles, and profiles with prior attributed completed-payment history are different cohorts. Unknown payment status is never reported as non-paying, and payment history is not called an active subscription. Error exposure alone does not prove that a page broke, a task failed, or work was lost. -Saved activation/return comparisons retain their native definition, cohort boundaries, complete eligible-profile counts and activation-event identity coverage in the signal. Code supplies that dated comparison as one evidence entry; the agent interprets its business relevance and may add one distinct sourced control. The complete brief retains the same 60-word budget. Activation is first within each independent cohort, not first-ever, and return is measured within a fixed elapsed-hour horizon. The existing minimum of 50 eligible profiles per complete cohort remains unchanged. Legacy signals without this measurement remain readable. +Saved activation/return comparisons retain their native definition, cohort boundaries, complete eligible-profile counts and activation-event identity coverage in the signal. Code supplies that dated comparison as one evidence entry; the agent interprets its business relevance and may add one distinct sourced control. The complete brief retains the same 60-word budget. Activation is first within each independent cohort, not first-ever, and return is measured within a fixed elapsed-hour horizon. The existing minimum of 50 eligible profiles per complete cohort remains unchanged. Legacy signals without this measurement remain readable. Investigations that query retention as supporting evidence use the same population rules: select two exact native results with `{retention: true}` and let code render their complete overall comparison. Published free-form retention tool claims are rejected. Truncated daily display rows do not invalidate a complete overall aggregate; an unrelated uncited retention read does not suppress an independently supported finding. Unsupported structured comparisons can resolve privately with code-rendered eligible and incomplete profile counts, without asserting a return rate or spending another correction turn. Quantity-only corrections identify the exact authoring fields to change while preserving valid evidence and references. A contradictory read of the exact saved retention population makes the current investigation private, even if the agent omits that read from its citations. Additional retention evidence must match the saved website, events, namespace, horizon, cohort dates and observation cutoff before publication. Retention quantities stay in the generated comparison; additional model prose may describe a qualitative discrepancy or a distinct non-retention fact. Conflicting counts require a fresh consistent investigation; model-selected citations cannot erase a contradictory measurement. diff --git a/apps/insights/src/agent.ts b/apps/insights/src/agent.ts index 439165cc9..a1844d566 100644 --- a/apps/insights/src/agent.ts +++ b/apps/insights/src/agent.ts @@ -4,6 +4,8 @@ import { type BusinessContext, } from "@databuddy/ai/lib/business-context"; import { isDeepStrictEqual } from "node:util"; +import dayjs from "dayjs"; +import { shiftDate } from "@databuddy/ai/query/date-utils"; import { z } from "zod"; import { AI_MODEL_MAX_RETRIES, @@ -22,6 +24,7 @@ import { insightMeasurementSchema, insightVerificationDefinitionSchema, retentionMeasurementSchema, + RETENTION_MINIMUM_PROFILES, type AgentInvestigationOutcome, type InsightDefinitionOperation, type InvestigationOutcome, @@ -71,6 +74,11 @@ const revenueEvidenceSchema = z .describe( "For revenue_overview, select complementary fields: gross revenue, refunds, and attributed revenue when it differs from gross. Select only non-null fields in every cited period; omit redundant counts and subtotals. Refund totals/counts do not establish net revenue or distinct refunded receipts. One entry per population; payment-description comparisons need a second whole-currency control. Cite both complete windows using only get_data references. Code supplies labels, values, periods and deltas." ); +const retentionEvidenceSchema = z + .strictObject({ retention: z.literal(true) }) + .describe( + "For identified_profile_retention without a saved snapshot, select {retention: true} and cite exactly two successful get_data results. Code compares the complete overall populations, each with at least 50 eligible profiles and no incomplete follow-up; never substitute daily rows or events. Keep the headline and summary qualitative. Unsupported comparisons resolve privately; code records their eligibility limits without asserting a retention rate." + ); const finishSchema = z.object({ evidence: z .array( @@ -81,13 +89,14 @@ const finishSchema = z.object({ "One compact comparison: behavior, before → after, dates and denominator, plus any interpretation-changing control. Use about 30 words across all prose claims. Do not repeat event definitions or describe source provenance." ), revenueEvidenceSchema, + retentionEvidenceSchema, ]), }) ) .min(1) .max(2) .describe( - "Select the evidence before deciding whether it merits publication. Keep each claim beside all contributing references. Revenue claims use {currency, fields} with only their contributing get_data references; other claims use concise text." + "Select the evidence before deciding whether it merits publication. Keep each claim beside all contributing references. Revenue claims use {currency, fields}; retention tool comparisons use {retention: true}. Both require their contributing get_data references. Other claims use concise text." ), publish: agentInvestigationOutcomeSchema.shape.publish, ...agentInvestigationOutcomeSchema.omit({ @@ -206,13 +215,14 @@ export function renderRevenueEvidence( }; } -function renderRetentionEvidence(signal: InvestigationSignal): string | null { - if (!signal.retentionMeasurement) { - return null; - } - const measured = retentionMeasurementSchema.parse( - signal.retentionMeasurement - ); +function renderRetentionEvidence( + measured: Pick< + z.infer, + "previous" | "current" | "observationEnd" | "timezone" + >, + period: InvestigationSignal["period"], + horizonDays: number +): string { const percent = (numerator: number, denominator: number) => `${Math.round((numerator / denominator) * 1000) / 10}%`; const windows = [measured.previous, measured.current]; @@ -224,10 +234,175 @@ function renderRetentionEvidence(signal: InvestigationSignal): string | null { (row) => `${row.identifiedEvents}/${row.events} (${percent(row.identifiedEvents, row.events)})` ); - const periods = [signal.period.previous, signal.period.current].map( - (period) => `${period.from}–${period.to}` + const periods = [period.previous, period.current].map( + (window) => `${window.from}–${window.to}` ); - return `Initial snapshot through ${measured.observationEnd} ${measured.timezone}: eligible identified profiles returning within ${measured.definition.horizonDays} days: ${returned.join(" → ")}; cohorts ${periods.join(" → ")}, fully observed. Activation events with identity: ${identity.join(" → ")}; anonymous events excluded.`; + return `Initial snapshot through ${measured.observationEnd} ${measured.timezone}: eligible identified profiles returning within ${horizonDays} days: ${returned.join(" → ")}; cohorts ${periods.join(" → ")}, fully observed. Activation events with identity: ${identity.join(" → ")}; anonymous events excluded.`; +} + +function renderToolRetentionEvidence( + sources: unknown, + input: InsightAgentInput, + completedReads: unknown[], + publish: boolean +) { + const readings = z + .array( + nativeReadingSchema.extend({ + type: z.literal("identified_profile_retention"), + websiteId: z.literal( + z + .string() + .min(1) + .parse( + input.appContext.websiteId ?? input.appContext.defaultWebsiteId + ) + ), + timezone: z.literal(input.appContext.timezone ?? "UTC"), + filters: z + .array( + z.object({ + field: z.enum([ + "activation_event", + "return_event", + "horizon_days", + "observation_end", + "namespace", + ]), + op: z.literal("eq"), + value: z.union([z.string().min(1), z.number()]), + }) + ) + .min(4) + .max(5), + }) + ) + .length(2) + .parse(sources) + .sort((a, b) => a.from.localeCompare(b.from)); + const first = readings[0]; + const scope = (reading: z.infer) => ({ + type: reading.type, + websiteId: reading.websiteId, + from: reading.from, + to: reading.to, + timezone: reading.timezone, + filters: reading.filters + .map((filter) => ({ + ...filter, + value: + filter.field === "horizon_days" ? Number(filter.value) : filter.value, + })) + .sort((a, b) => a.field.localeCompare(b.field)), + }); + const filters = z + .strictObject({ + activation_event: z.string().min(1).max(256), + return_event: z.string().min(1).max(256), + horizon_days: z.coerce + .number() + .pipe(z.union([z.literal(7), z.literal(30)])), + observation_end: z.iso.date(), + namespace: z.string().min(1).max(256).optional(), + }) + .parse( + Object.fromEntries( + first.filters.map((filter) => [filter.field, filter.value]) + ) + ); + const observedBefore = dayjs + .tz(shiftDate(filters.observation_end, 1), first.timezone) + .valueOf(); + const rows = readings.map((reading, index) => { + const overall = reading.data.filter((row) => row.row_type === "overall"); + const row = retentionRowSchema.parse(overall[0]); + if ( + new Set(reading.filters.map((filter) => filter.field)).size !== + reading.filters.length || + !isDeepStrictEqual(scope(reading).filters, scope(first).filters) || + reading.from > reading.to || + Date.parse(reading.to) - Date.parse(reading.from) !== + Date.parse(first.to) - Date.parse(first.from) || + (index > 0 && reading.from <= first.to) || + overall.length !== 1 || + row.cohort_date !== null || + row.cohort_from !== reading.from || + row.cohort_to !== reading.to || + row.timezone !== reading.timezone || + row.horizon_days !== filters.horizon_days || + row.observation_end !== filters.observation_end || + Date.parse(row.cohort_start) !== + dayjs.tz(reading.from, reading.timezone).valueOf() || + Date.parse(row.cohort_end) !== + dayjs.tz(shiftDate(reading.to, 1), reading.timezone).valueOf() || + Date.parse(row.observed_before) !== observedBefore || + observedBefore > Date.parse(input.appContext.currentDateTime) || + Date.parse(row.cohort_end) > observedBefore + ) { + throw new Error( + "Retention comparisons require complete equal-duration non-overlapping cohorts with the same website, events, namespace, horizon, timezone and observation cutoff. Cite their exact overall rows." + ); + } + return row; + }); + const windows = rows.map(retentionWindow); + if ( + !publish && + windows.some( + (window) => + !retentionMeasurementSchema.shape.previous.safeParse(window).success + ) + ) { + return { + text: `Retention comparison withheld. Cohorts ${readings.map((reading) => `${reading.from}–${reading.to}`).join(" → ")} ${first.timezone}, through ${filters.observation_end}: ${rows.map((row) => `${row.eligible_profiles} eligible, ${row.incomplete_profiles} incomplete`).join(" → ")} identified profiles. Publication requires ${RETENTION_MINIMUM_PROFILES} eligible profiles per fully observed cohort.`, + }; + } + const [previous, current] = z + .array(retentionMeasurementSchema.shape.previous) + .length(2) + .parse(windows, { + error: () => + `Retention publication requires at least ${RETENTION_MINIMUM_PROFILES} eligible profiles and no incomplete follow-up in each cohort. Resolve this comparison privately; preserve independently supported findings.`, + }); + if ( + publish && + completedReads.some((value) => { + const reading = nativeReadingSchema.safeParse(value).data; + if (!reading) { + return false; + } + const index = readings.findIndex((selected) => + isDeepStrictEqual(scope(reading), scope(selected)) + ); + if (index < 0) { + return false; + } + const overall = reading.data.filter((row) => row.row_type === "overall"); + return ( + overall.length !== 1 || + !isDeepStrictEqual( + retentionRowSchema.safeParse(overall[0]).data, + rows[index] + ) + ); + }) + ) { + throw new Error( + "A retention read conflicts with the cited comparison. Resolve privately; dropping a citation or reading again cannot erase an unresolved measurement conflict." + ); + } + return { + text: renderRetentionEvidence( + { + previous, + current, + observationEnd: filters.observation_end, + timezone: first.timezone, + }, + { previous: first, current: readings[1] }, + filters.horizon_days + ), + }; } const retentionReadingType = z.object({ @@ -1619,7 +1794,13 @@ export async function runInsightAgent( throw new Error("AI_GATEWAY_API_KEY is required"); } const isDefinition = ["goal", "funnel"].includes(input.signal.entity.type); - const nativeRetention = renderRetentionEvidence(input.signal); + const nativeRetention = input.signal.retentionMeasurement + ? renderRetentionEvidence( + retentionMeasurementSchema.parse(input.signal.retentionMeasurement), + input.signal.period, + input.signal.retentionMeasurement.definition.horizonDays + ) + : null; const outcomeSchema = finishSchema.extend({ evidence: nativeRetention ? z @@ -1854,9 +2035,17 @@ export async function runInsightAgent( ) ) { throw new Error( - "Structured revenue evidence requires exact successful get_data result references." + "Structured evidence requires exact successful get_data result references." ); } + if ("retention" in item.claim) { + return renderToolRetentionEvidence( + citedEvidence[index], + input, + results.flatMap(successfulReadOutputs), + candidate.publish + ).text; + } const native = renderRevenueEvidence( item.claim, citedEvidence[index], @@ -1865,6 +2054,17 @@ export async function runInsightAgent( nativeRevenue.push(native); return native.text; } + if ( + !nativeRetention && + candidate.publish && + citedEvidence[index].some( + (source) => retentionReadingType.safeParse(source).success + ) + ) { + throw new Error( + "For published retention tool evidence, submit {retention: true} with both exact get_data results instead of prose. Code validates eligible profiles, complete follow-up and scope; unsupported comparisons stay private." + ); + } if ( nativeRetention && numericTokens(item.claim).length > 0 && @@ -1954,19 +2154,20 @@ export async function runInsightAgent( steps.flatMap((step) => step.toolCalls.map((call) => call.toolName)) ); if ( + proposed.publish && (nativeRetention || - candidate.evidence.some( - (item) => typeof item.claim !== "string" - )) && - [ - proposed.title.replace(input.signal.entity.label, ""), - verification ? "" : proposed.summary, - proposed.rootCause ?? "", - ].some((text) => numericTokens(text).length > 0) + candidate.evidence.some((item) => typeof item.claim !== "string")) ) { - throw new Error( - "Keep measured quantities in the generated evidence; use a qualitative headline, summary and cause." - ); + const numericFields = Object.entries({ + title: proposed.title.replace(input.signal.entity.label, ""), + summary: verification ? "" : proposed.summary, + rootCause: proposed.rootCause ?? "", + }).filter(([, value]) => numericTokens(value).length > 0); + if (numericFields.length > 0) { + throw new Error( + `Keep measured quantities in the generated evidence; use a qualitative headline, summary and cause. Rewrite only these fields without measured numbers: ${numericFields.map(([field, value]) => `${field}: ${JSON.stringify(value)}`).join("; ")}. Preserve the valid evidence and its references; no new read is needed.` + ); + } } const validated = validateAgentOutcome( proposed, diff --git a/apps/insights/src/retention-publication.test.ts b/apps/insights/src/retention-publication.test.ts new file mode 100644 index 000000000..6396b9c87 --- /dev/null +++ b/apps/insights/src/retention-publication.test.ts @@ -0,0 +1,539 @@ +import "@databuddy/test/env"; +import { describe, expect, it } from "bun:test"; +import type { InvestigationSignal } from "@databuddy/shared/insights"; +import type { StepResult, ToolSet } from "ai"; +import { MockLanguageModelV3, mockValues } from "ai/test"; +import { getDataTool } from "../../../packages/ai/src/ai/tools/get-data"; +import { runInsightAgent } from "./agent"; + +const appContext = { + chatId: "retention-publication-test", + currentDateTime: "2026-09-09T00:00:00.000Z", + defaultWebsiteId: "site-1", + mutationMode: "dry-run" as const, + organizationId: "org-1", + timezone: "UTC", + userId: "system", + websiteDomain: "example.com", + websiteId: "site-1", + websiteName: "Example reports", +}; + +// A real event subject can inspect retention during a reply without a detector +// retention snapshot. Do not manufacture a retentionMeasurement below its floor. +const signal: InvestigationSignal = { + signalKey: "event:report_shared", + entity: { type: "event", id: "report_shared", label: "Shared reports" }, + metric: { + label: "Shared reports", + current: 100, + previous: 200, + format: "number", + }, + changePercent: -50, + severity: "warning", + sentiment: "negative", + period: { + previous: { from: "2026-08-18", to: "2026-08-24" }, + current: { from: "2026-08-25", to: "2026-08-31" }, + }, +}; + +function reading( + period: "previous" | "current", + eligible = 50, + incomplete = 0 +) { + const { from, to } = signal.period[period]; + const retained = Math.floor(eligible * (period === "previous" ? 0.8 : 0.2)); + const row = { + cohort_from: from, + cohort_to: to, + observation_end: "2026-09-08", + cohort_start: `${from}T00:00:00.000Z`, + cohort_end: new Date(Date.parse(to) + 86_400_000).toISOString(), + observed_before: appContext.currentDateTime, + timezone: "UTC", + horizon_days: 7, + identity_basis: "direct_profile_id", + activation_basis: "first_in_cohort_window", + activated_profiles: eligible + incomplete, + eligible_profiles: eligible, + retained_profiles: retained, + not_retained_profiles: eligible - retained, + incomplete_profiles: incomplete, + activation_events: (eligible + incomplete) * 2, + identified_activation_events: eligible + incomplete, + unidentified_activation_events: eligible + incomplete, + }; + return { + type: "identified_profile_retention", + websiteId: "site-1", + from, + to, + timezone: "UTC", + filters: [ + { field: "activation_event", op: "eq" as const, value: "report_shared" }, + { field: "return_event", op: "eq" as const, value: "report_opened" }, + { field: "horizon_days", op: "eq" as const, value: 7 }, + { field: "observation_end", op: "eq" as const, value: "2026-09-08" }, + { field: "namespace", op: "eq" as const, value: "product" }, + ], + data: [ + { ...row, row_type: "overall", cohort_date: null }, + { ...row, row_type: "cohort", cohort_date: from }, + ] as Record[], + rowCount: 2, + returnedRows: 2, + truncated: false, + }; +} + +const previousKey = "identified_profile_retention"; +const currentKey = "identified_profile_retention@site-1"; +const source = (resultKey: string) => ({ + source: "tool" as const, + name: "get_data", + toolCallId: "get_data-1", + resultKey, +}); +const sources = [source(previousKey), source(currentKey)]; + +function finish(claim: unknown = { retention: true }, publish = true) { + return { + title: "Report reuse fell", + summary: "Fewer identified profiles returned after sharing a report.", + rootCause: null, + evidence: [{ sources, claim }], + findingKind: "product_outcome", + publish, + publicationBasis: publish ? "measured_impact" : null, + next: { type: "resolve", reason: "The cause remains unknown." }, + }; +} + +const privateFinish = { + ...finish("The cohort comparison remains unverified.", false), + title: "Report reuse is unverified", + summary: "The returned cohorts do not establish a complete comparison.", +}; + +function response(toolName: string, value: unknown, toolCallId: string) { + return { + content: [ + { + type: "tool-call" as const, + toolName, + toolCallId, + input: JSON.stringify(value), + }, + ], + finishReason: { unified: "tool-calls" as const, raw: undefined }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 }, + }, + warnings: [], + }; +} + +async function investigate( + readings = [reading("previous"), reading("current")], + proposal: unknown = finish(), + correction?: unknown, + options: { + earlierReadings?: ReturnType[]; + limit?: number; + } = {} +) { + const { earlierReadings, limit = 100 } = options; + const queries = readings.map( + ({ type, websiteId, from, to, timezone, filters }) => ({ + type, + websiteId, + from, + to, + timezone, + filters, + limit, + }) + ); + const model = new MockLanguageModelV3({ + doGenerate: mockValues( + ...(earlierReadings + ? [response("get_data", { queries }, "get_data-earlier")] + : []), + response("get_data", { queries }, "get_data-1"), + response("finish_investigation", proposal, "finish-1"), + response("finish_investigation", correction ?? proposal, "finish-2"), + response("finish_investigation", correction ?? proposal, "finish-3") + ), + }); + const steps: StepResult[] = []; + const calls: unknown[] = []; + const result = await runInsightAgent( + { + appContext, + signal, + evidence: [ + "The team defines report sharing as initial value and reopening as reuse.", + ], + history: [], + otherOpenWork: [], + githubRepository: null, + request: { + body: "Check whether profiles return after sharing reports.", + createdAt: appContext.currentDateTime, + }, + }, + { + model, + onStepFinish: (step) => { + steps.push(step); + }, + tools: { + get_data: { + ...getDataTool, + execute: async (input, options) => { + calls.push(input); + const returned = + options.toolCallId === "get_data-earlier" + ? (earlierReadings ?? readings) + : readings; + return { + results: Object.fromEntries( + returned.map((value, index) => [ + index === 0 ? previousKey : currentKey, + value, + ]) + ), + }; + }, + }, + }, + } + ); + // Native argument validation must reach the synthetic executor exactly once. + expect(calls).toEqual( + earlierReadings ? [{ queries }, { queries }] : [{ queries }] + ); + expect(result.toolCallCount).toBe(earlierReadings ? 2 : 1); + return { ...result, steps, model }; +} + +async function expectPrivate( + readings: ReturnType[], + proposal: unknown = finish() +) { + const result = await investigate(readings, proposal, privateFinish); + expect(result.outcome.publish).toBe(false); + expect(result.outcome.rootCause).toBeNull(); + expect(result.outcome.next.type).toBe("resolve"); + expect(result.model.doGenerateCalls).toHaveLength(3); + expect( + result.steps[1].content.some( + (part) => + part.type === "tool-error" && part.toolName === "finish_investigation" + ) + ).toBe(true); + return result; +} + +describe("tool-supplied retention publication without a saved snapshot", () => { + it.each([ + "small", + "incomplete", + ])("records a private structured %s limitation without a repair turn", async (kind) => { + const result = await investigate( + [ + reading("previous", kind === "small" ? 20 : 50), + reading( + "current", + kind === "small" ? 20 : 50, + kind === "incomplete" ? 1 : 0 + ), + ], + { + ...privateFinish, + summary: `${kind === "small" ? 20 : 50} eligible profiles; the comparison remains unconfirmed.`, + evidence: [{ sources, claim: { retention: true } }], + } + ); + expect(result.outcome.publish).toBe(false); + expect(result.model.doGenerateCalls).toHaveLength(2); + expect(result.outcome.evidence[0]).toContain( + "Retention comparison withheld." + ); + expect(result.outcome.evidence[0]).not.toContain("%"); + expect( + result.steps + .flatMap((step) => step.content) + .filter((part) => part.type === "tool-error") + ).toHaveLength(0); + }); + it("identifies the exact numeric summary that needs correction", async () => { + const result = await investigate( + undefined, + { ...finish(), summary: "Identity coverage was 50%." }, + finish() + ); + expect(result.outcome.publish).toBe(true); + const rejection = result.steps[1].content.find( + (part) => part.type === "tool-error" + ); + expect(rejection?.type).toBe("tool-error"); + if (rejection?.type === "tool-error") + expect(String(rejection.error)).toContain( + 'summary: "Identity coverage was 50%."' + ); + }); + + it.each([ + { period: "previous" as const, eligible: 20 }, + { period: "current" as const, eligible: 20 }, + { period: "previous" as const, eligible: 49 }, + { period: "current" as const, eligible: 49 }, + ])("keeps $period cohort with $eligible eligible profiles private", async ({ + period, + eligible, + }) => { + await expectPrivate([ + reading("previous", period === "previous" ? eligible : 200), + reading("current", period === "current" ? eligible : 200), + ]); + }); + + it("publishes exactly 50 eligible profiles per complete cohort using rendered evidence", async () => { + const result = await investigate(); + expect(result.outcome.publish).toBe(true); + expect(result.outcome.rootCause).toBeNull(); + expect(result.model.doGenerateCalls).toHaveLength(2); + const evidence = result.outcome.evidence.join(" "); + expect(evidence).toContain("40/50"); + expect(evidence).toContain("10/50"); + expect(evidence).toContain("2026-08-18"); + expect(evidence).toContain("2026-08-31"); + expect(evidence).toMatch(/7|seven/); + expect(evidence).toMatch(/identified profiles/i); + }); + + it.each([ + "previous", + "current", + ] as const)("keeps incomplete %s follow-up private despite 50 eligible profiles", async (period) => { + await expectPrivate([ + reading("previous", 50, period === "previous" ? 1 : 0), + reading("current", 50, period === "current" ? 1 : 0), + ]); + }); + + it("rejects public prose citing valid native retention but permits a private explanation", async () => { + await expectPrivate( + [reading("previous"), reading("current")], + finish( + "Eligible identified profiles returning within seven days fell from 40/50 to 10/50." + ) + ); + }); + + it("rejects a public native-prose comparison of 16/20 to 4/20 eligible profiles", async () => { + await expectPrivate( + [reading("previous", 20), reading("current", 20)], + finish( + "Eligible identified profiles returning within seven days fell from 16/20 to 4/20." + ) + ); + }); + + it.each([ + "overall-only", + "truncated-daily", + ])("publishes a complete overall aggregate with %s rows", async (mode) => { + const readings = [reading("previous"), reading("current")]; + for (const [index, value] of readings.entries()) { + if (mode === "overall-only") { + // The SQL LIMIT applies after the overall aggregate is computed. + value.data = value.data.slice(0, 1); + value.rowCount = 1; + value.returnedRows = 1; + continue; + } + // Native get_data caps a 28-day table at 20 rows, keeping overall first. + value.from = index === 0 ? "2026-07-07" : "2026-08-04"; + value.to = index === 0 ? "2026-08-03" : "2026-08-31"; + const overall = { + ...value.data[0], + cohort_from: value.from, + cohort_to: value.to, + cohort_start: `${value.from}T00:00:00.000Z`, + cohort_end: new Date(Date.parse(value.to) + 86_400_000).toISOString(), + }; + let remainingRetained = index === 0 ? 40 : 10; + const daily = Array.from({ length: 28 }, (_, day) => { + const eligible = day < 22 ? 2 : 1; + const retained = Math.min(eligible, remainingRetained); + remainingRetained -= retained; + return { + ...overall, + row_type: "cohort", + cohort_date: new Date(Date.parse(value.from) + day * 86_400_000) + .toISOString() + .slice(0, 10), + activated_profiles: eligible, + eligible_profiles: eligible, + retained_profiles: retained, + not_retained_profiles: eligible - retained, + activation_events: eligible * 2, + identified_activation_events: eligible, + unidentified_activation_events: eligible, + }; + }); + value.data = [overall, ...daily].slice(0, 20); + value.returnedRows = 20; + value.rowCount = 29; + value.truncated = true; + } + const result = await investigate(readings, finish(), undefined, { + limit: mode === "overall-only" ? 1 : 100, + }); + expect(result.outcome.publish).toBe(true); + expect(result.outcome.evidence.join(" ")).toContain("40/50"); + }); + + it.each([ + "namespace", + "return-event", + "cohort-dates", + "overlapping-windows", + "missing-overall", + "cutoff", + "identity-basis", + "inconsistent-counts", + ] as const)("rejects mismatched or invalid native metadata: %s", async (mode) => { + const previous = reading("previous"); + const current = reading("current"); + if (mode === "namespace" || mode === "return-event") { + const field = mode === "namespace" ? "namespace" : "return_event"; + current.filters = current.filters.map((filter) => + filter.field === field ? { ...filter, value: "different" } : filter + ); + } else if (mode === "cohort-dates") { + current.data[0].cohort_to = "2026-08-30"; + } else if (mode === "overlapping-windows") { + current.from = previous.from; + current.to = previous.to; + current.data = current.data.map((row) => ({ + ...row, + cohort_from: previous.from, + cohort_to: previous.to, + cohort_start: previous.data[0].cohort_start, + cohort_end: previous.data[0].cohort_end, + })); + } else if (mode === "missing-overall") { + current.data = current.data.slice(1); + current.rowCount = current.returnedRows = 1; + } else if (mode === "cutoff") { + current.data[0].observed_before = "2026-09-08T00:00:00.000Z"; + } else if (mode === "identity-basis") { + current.data[0].identity_basis = "anonymous_visitor_id"; + } else { + current.data[0].not_retained_profiles = 0; + } + await expectPrivate([previous, current]); + }); + + it.each([ + "one-reference", + "duplicate-reference", + "provided-reference", + ])("requires two exact native result references: %s", async (mode) => { + const proposal = finish(); + if (mode === "one-reference") { + proposal.evidence[0].sources = [source(previousKey)]; + } + if (mode === "duplicate-reference") { + proposal.evidence[0].sources = [source(previousKey), source(previousKey)]; + } + if (mode === "provided-reference") { + const malformed = { + ...proposal, + evidence: [ + { + claim: { retention: true }, + sources: [{ source: "provided", index: 0 }], + }, + ], + }; + // The model supplies untrusted JSON; this reference is valid generally, + // but cannot replace a native measured retention result. + const result = await investigate( + [reading("previous"), reading("current")], + malformed, + privateFinish + ); + expect(result.outcome.publish).toBe(false); + expect(result.model.doGenerateCalls).toHaveLength(3); + return; + } + await expectPrivate([reading("previous"), reading("current")], proposal); + }); + + it("preserves an independent measured finding after an uncited undersized retention read", async () => { + const unrelated = reading("previous", 20); + unrelated.filters = unrelated.filters.map((filter) => + filter.field === "activation_event" + ? { ...filter, value: "tutorial_started" } + : filter + ); + const proposal = { + ...finish(), + title: "Report sharing fell", + summary: "Fewer reports were shared; the cause remains unknown.", + evidence: [ + { + sources: [{ source: "signal" }], + claim: "Shared report events fell from 200 to 100.", + }, + ], + }; + const result = await investigate([unrelated], proposal); + expect(result.outcome.publish).toBe(true); + expect(result.outcome.evidence).toEqual([ + "Shared report events fell from 200 to 100.", + ]); + expect(result.model.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + "undersized", + "different-return-count", + ])("keeps an earlier exact-query %s conflict binding when only later matching reads are cited", async (mode) => { + const earlier = [ + reading("previous", mode === "undersized" ? 20 : 50), + reading("current"), + ]; + if (mode === "different-return-count") { + for (const row of earlier[0].data) { + row.retained_profiles = 30; + row.not_retained_profiles = 20; + } + } + const result = await investigate( + [reading("previous"), reading("current")], + finish(), + privateFinish, + { earlierReadings: earlier } + ); + expect(result.outcome.publish).toBe(false); + expect(result.outcome.rootCause).toBeNull(); + expect(result.model.doGenerateCalls).toHaveLength(4); + const rejected = result.steps[2].content.find( + (part) => + part.type === "tool-error" && part.toolName === "finish_investigation" + ); + expect(rejected).toBeDefined(); + if (rejected?.type === "tool-error") { + expect(String(rejected.error)).toMatch(/conflict/i); + } + }); +});