From ad975d978c556739bf60b42fc7019fe075762929 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Wed, 9 Sep 2026 03:16:17 +0300 Subject: [PATCH] fix(insights): preserve business priorities through investigation selection --- SPEC.md | 5 +- .../src/business-aware-selection.test.ts | 67 ++- apps/insights/src/business-aware-selection.ts | 60 ++- apps/insights/src/coverage-planner.test.ts | 11 +- apps/insights/src/coverage-planner.ts | 4 +- apps/insights/src/evals/README.md | 8 + apps/insights/src/evals/context-selection.ts | 407 ++++++++++++++++++ apps/insights/src/evals/quality.ts | 2 +- 8 files changed, 503 insertions(+), 61 deletions(-) create mode 100644 apps/insights/src/evals/context-selection.ts diff --git a/SPEC.md b/SPEC.md index 170e386dd..7ad26e540 100644 --- a/SPEC.md +++ b/SPEC.md @@ -58,7 +58,10 @@ regressions retain priority even when the model selects none. Original measureme constraints and the unverified planning rationale stay in the frozen objective. Scheduled runs investigate at most two; a deliberate manual full scan investigates at most five and covers a distinct eligible specialist family before taking extra work from -one family. The portfolio is diversified across correlated subjects and survives a +one family. This does not reintroduce optional general work excluded by business-aware +selection. Candidate input is bounded by serialized size rather than a count cutoff; +the complete saved brief and newest relevant correction survive source budgeting, or +selection retains the conservative fallback. The portfolio is diversified across correlated subjects and survives a retry unchanged. Each selected signal still gets its own exact agent turn, durable observation, and investigation history; a model does not manufacture a broad report from ungrounded raw data. diff --git a/apps/insights/src/business-aware-selection.test.ts b/apps/insights/src/business-aware-selection.test.ts index e2662e51d..851afac82 100644 --- a/apps/insights/src/business-aware-selection.test.ts +++ b/apps/insights/src/business-aware-selection.test.ts @@ -4,6 +4,7 @@ import type { BusinessContext } from "@databuddy/ai/lib/business-context"; import { MockLanguageModelV3 } from "ai/test"; import { chooseInvestigationSignals } from "./business-aware-selection"; import { planCoveragePortfolio } from "./coverage-planner"; +import { organizationProfileContext } from "./business-context"; import type { DetectedSignal } from "./detection"; import { investigateWebsitePortfolioWithSources, @@ -317,11 +318,6 @@ describe("business-aware investigation selection", () => { [], [outcome], [traffic, { ...outcome, definitionEvidence: "x".repeat(64_001) }], - Array.from({ length: 33 }, (_, index) => ({ - ...outcome, - metric: `goal:${index}`, - subjectKey: `goal:${index}`, - })), ]) { await planInvestigationsWithBusinessContext( input, @@ -335,23 +331,50 @@ describe("business-aware investigation selection", () => { scope ); } - await chooseInvestigationSignals( - { - businessContext: context, - candidates: Array.from({ length: 9 }, (_, index) => ({ - signal: prepareInvestigation( - { - ...outcome, - metric: `goal:${index}`, - subjectKey: `goal:${index}`, - }, - 7 - ).signal, - })), - limit: 2, - }, - model - ); + expect(model.doGenerateCalls).toHaveLength(0); + }); + + it.each([9, 24, 33])("uses business context for %i bounded candidates", async (count) => { + const key = `goal:${count - 1}`; + const model = new MockLanguageModelV3({ doGenerate: async (request) => { + expect(JSON.stringify(request.prompt)).toContain(explanation); + return response({ selections: [{ ...choice, signalKey: key }] }); + } }); + const plan = await planInvestigationsWithBusinessContext(input, + Array.from({ length: count }, (_, index) => ({ ...outcome, metric: `goal:${index}`, subjectKey: `goal:${index}` })), + { loadBusinessProfile: async () => context, selectCandidates: params => chooseInvestigationSignals(params, model) }, + false, scope, { reason: "scheduled" }); + expect(plan.map(candidate => candidate.signal.signalKey)).toEqual([key]); + expect(model.doGenerateCalls).toHaveLength(1); + }); + + it("keeps the complete maximum saved brief and a current correction before optional background", async () => { + const saved = organizationProfileContext({ + content: "Business overview. ".padEnd(11_940, " Background.") + " Final exclusion: generic traffic is already explained.", + origin: "mixed", sources: [{ url: "https://example.com/", title: "Public overview" }], + revision: 4, updatedAt: "2026-07-11T00:00:00.000Z", updatedBy: "example-editor", sourceWebsiteId: null, + teamContext: { priority: "Prioritize delivery. ".padEnd(2000, " Priority."), successDefinition: "Delivery means accepted by the recipient. ".padEnd(2000, " Definition."), exclusions: "Exclude demos. ".padEnd(2000, " Exclusion.") }, + }, input.organizationId, new Date(input.asOf)); + const correction = { id: "current-correction", kind: "team_reply" as const, subjectKey: choice.signalKey, + content: "Current correction: use accepted delivery. ".padEnd(3950, " Team details.") + " Correction ends here.", observedAt: input.asOf }; + saved.sources.push(correction); + const model = new MockLanguageModelV3({ doGenerate: async (request) => { + const sent = JSON.stringify(request.prompt); + expect(sent).toContain("Business overview."); + expect(sent).toContain("Final exclusion: generic traffic is already explained."); + expect(sent).toContain("Current correction: use accepted delivery."); + expect(sent).toContain("Correction ends here."); + expect(sent).toContain("Exclude demos."); + return response({ selections: [choice] }); + } }); + const result = await chooseInvestigationSignals({ businessContext: saved, candidates: [traffic, outcome].map(signal => ({ signal: prepareInvestigation(signal, 7).signal })), limit: 2 }, model); + expect(result?.output.selections).toEqual([choice]); + }); + + it("falls back instead of selecting from an incomplete over-budget saved document", async () => { + const model = new MockLanguageModelV3({ doGenerate: async () => { throw new Error("Selection should be skipped"); } }); + const saved = organizationProfileContext({ content: "\u0000".repeat(12_000), sources: [], origin: "team", revision: 1, updatedAt: input.asOf, updatedBy: "example-editor", sourceWebsiteId: null }, input.organizationId, new Date(input.asOf)); + expect(await chooseInvestigationSignals({ businessContext: saved, candidates: [traffic, outcome].map(signal => ({ signal: prepareInvestigation(signal, 7).signal })), limit: 2 }, model)).toBeNull(); expect(model.doGenerateCalls).toHaveLength(0); }); diff --git a/apps/insights/src/business-aware-selection.ts b/apps/insights/src/business-aware-selection.ts index f222b666f..6901b541c 100644 --- a/apps/insights/src/business-aware-selection.ts +++ b/apps/insights/src/business-aware-selection.ts @@ -45,7 +45,6 @@ export async function chooseInvestigationSignals( if ( !(model || isAiGatewayConfigured) || candidates.length <= 1 || - candidates.length > input.limit * 4 || !businessContext.sources.length || !["ready", "partial"].includes(businessContext.status) || JSON.stringify(candidates).length > 48_000 @@ -69,54 +68,47 @@ export async function chooseInvestigationSignals( ), ...replies, ...businessContext.sources.filter((source) => source.kind === "website"), - ]; - const sources: Pick< - BusinessContext["sources"][number], - | "id" - | "kind" - | "content" - | "observedAt" - | "subjectKey" - | "author" - | "url" - | "references" - | "origin" - >[] = []; - let pageCharacters = 0; - let characters = 0; - for (const { - id, - kind, - content, - observedAt, - subjectKey, - author, - origin, - url, - references, - } of ordered) { - const source = { + ].map( + ({ id, kind, content, observedAt, subjectKey, author, origin, url }) => ({ id, kind, - references, content, observedAt, subjectKey, author, origin, url, - }; + }) + ); + // Keep the complete saved document and the newest relevant correction. + // Bibliography stays on the investigation snapshot; it is not needed to rank work. + const characterLimit = Math.max( + 18_000, + ordered + .filter( + (source) => + source.kind === "organization_profile" || source.id === replies[0]?.id + ) + .reduce((total, source) => total + JSON.stringify(source).length, 0) + ); + if (characterLimit > 32_000) { + return null; + } + const sources: typeof ordered = []; + let pageCharacters = 0; + let characters = 0; + for (const source of ordered) { const size = JSON.stringify(source).length; if ( - sources.some((item) => item.id === id) || - characters + size > 18_000 || - (kind === "website" && pageCharacters + size > 8000) + sources.some((item) => item.id === source.id) || + characters + size > characterLimit || + (source.kind === "website" && pageCharacters + size > 8000) ) { continue; } sources.push(source); characters += size; - if (kind === "website") { + if (source.kind === "website") { pageCharacters += size; } } diff --git a/apps/insights/src/coverage-planner.test.ts b/apps/insights/src/coverage-planner.test.ts index b860a9667..95f693a7a 100644 --- a/apps/insights/src/coverage-planner.test.ts +++ b/apps/insights/src/coverage-planner.test.ts @@ -303,12 +303,19 @@ describe("business preference constraints", () => { expect(plan).toHaveLength(5); expect(plan).toContain(error); expect(plan).toContain(funnel); - expect(plan).toContain(traffic); + expect(plan).not.toContain(traffic); expect(plan.filter((item) => item.metric.startsWith("goal:"))).toHaveLength( - 2 + 3 ); }); + it.each(["manual", "scheduled"] as const)("preserves an explicit traffic exclusion in a %s scan", (reason) => { + const goal = signal({ metric: "goal:activation", subjectKey: "goal:activation" }); + const traffic = signal({ metric: "visitors" }); + expect(planCoveragePortfolio([traffic, goal], { reason, selectedSignalKeys: keys([goal]) })).toEqual([goal]); + expect(planCoveragePortfolio([traffic, goal], { reason, selectedSignalKeys: keys([traffic]) })).toContain(traffic); + }); + it("keeps one correlated subject, the due case first, and the scheduled limit", () => { const due = signal({ metric: "goal:due", subjectKey: "goal:due" }); const visitors = signal({ metric: "visitors" }); diff --git a/apps/insights/src/coverage-planner.ts b/apps/insights/src/coverage-planner.ts index 0a32731d7..de9a084eb 100644 --- a/apps/insights/src/coverage-planner.ts +++ b/apps/insights/src/coverage-planner.ts @@ -199,7 +199,9 @@ export function planCoveragePortfolio( (candidate) => selection.has(candidate.key) || isCriticalReliabilitySignal(candidate.signal) || - (options.reason === "manual" && !usedFamilies.has(candidate.family)) + (options.reason === "manual" && + candidate.family !== "general" && + !usedFamilies.has(candidate.family)) ) : available; const preferred = diff --git a/apps/insights/src/evals/README.md b/apps/insights/src/evals/README.md index a32dc52d0..3f43204a0 100644 --- a/apps/insights/src/evals/README.md +++ b/apps/insights/src/evals/README.md @@ -1,5 +1,13 @@ # Investigation quality evals +`context-selection.ts --out --runs 2` compares native absent/present +context paths before running selected investigations through this evaluator. It uses +synthetic 2-, 9- and 24-signal portfolios, a maximum-sized context correction case and +manual exclusion coverage. `--reverse` reverses candidate order for holdouts; `--cases` +selects scenario IDs. Alternate arms, preserve the copied source and fixtures, and +review the complete outputs as well as final selections. Its zero exit status means +the run completed; `results.json` retains quality failures for manual comparison. + Run from the repository root with `AI_GATEWAY_API_KEY` configured: ```sh diff --git a/apps/insights/src/evals/context-selection.ts b/apps/insights/src/evals/context-selection.ts new file mode 100644 index 000000000..f460d8868 --- /dev/null +++ b/apps/insights/src/evals/context-selection.ts @@ -0,0 +1,407 @@ +import { + appendFileSync, + copyFileSync, + mkdirSync, + writeFileSync, +} from "node:fs"; +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { createModelFromId } from "@databuddy/ai/config/models"; +import type { BusinessContext } from "@databuddy/shared/insights"; +import { wrapLanguageModel } from "ai"; +import { spawnSync } from "bun"; +import { runInsightAgent } from "../agent"; +import { chooseInvestigationSignals } from "../business-aware-selection"; +import { organizationProfileContext } from "../business-context"; +import type { DetectedSignal } from "../detection"; +import { planInvestigationsWithBusinessContext } from "../generation"; +import { evaluate, qualityCases } from "./quality"; + +interface SelectionResult { + id: string; + investigations: Awaited>[]; + planned: { key: string; objective: string | undefined }[]; + requiredFirst: boolean; + requiredSelected: boolean; + selectionCalls: number; + selectionMs: number; + selectionUsage: unknown[]; +} + +// Frozen synthetic measurements and native selection/investigation entry points. +// Only gateway model requests are live. No persistence, billing or delivery runs. +const asOf = "2026-09-05T00:00:00.000Z"; +const modelId = "openai/gpt-5.6-terra"; +const input = { + organizationId: "synthetic-org", + websiteId: "synthetic-site", + domain: "synthetic.example.invalid", + timezone: "UTC", + asOf, +}; +const base: DetectedSignal = { + baseline: 1000, + current: 100, + deltaPercent: -90, + detectedAt: "2026-09-04", + direction: "down", + label: "Visitors", + method: "wow", + metric: "visitors", + severity: "critical", +}; +const activation: DetectedSignal = { + ...base, + baseline: 18, + current: 10, + deltaPercent: -44.4, + severity: "warning", + metric: "funnel:first-report", + subjectKey: "funnel:first-report", + entityId: "first-report", + entityLabel: "First report delivered", + label: "First report delivery rate", + definitionEvidence: + "Funnel first-report: EVENT project_created then EVENT first_report_delivered; no filters; ordered unique visitors, not projects or event occurrences.", + investigationObjective: + "Compare the same ordered visitor population in both full seven-day windows. A first-report event name alone does not establish delivery, payment or causality.", +}; +const profile = { + content: + "Example builds report delivery software. The public documentation moved to a separate domain; that explains its visitor decline. Public demo activity is deliberately separate from customer outcomes.", + origin: "team" as const, + sources: [], + revision: 4, + updatedAt: "2026-09-04T12:00:00.000Z", + updatedBy: "synthetic-owner", + sourceWebsiteId: null, + teamContext: { + priority: + "Improve first successful report delivery after project creation. Investigate its unexplained decline before public docs or demo activity.", + successDefinition: + "The team defines first_report_delivered as an event emitted after the first successful report delivery. Analytics counts distinct visitors through these steps, not distinct projects or customers.", + exclusions: + "Public demo events named demo_action_* only count marketing demo button clicks. They do not represent product outcomes and their changes are outside this team's current investigation scope. The documentation migration is already understood.", + }, +}; +const context = organizationProfileContext( + profile, + input.organizationId, + new Date(asOf) +); +const absent: BusinessContext = { + capturedAt: asOf, + status: "disabled", + sources: [], + issues: [], +}; +const demo = (index: number): DetectedSignal => ({ + ...base, + metric: "custom_event_count", + subjectKey: `event:demo_action_${index}`, + entityId: `demo_action_${index}`, + entityLabel: `Public demo action ${index}`, + label: `Public demo action ${index} occurrences`, + definitionEvidence: `CUSTOM_EVENT demo_action_${index}; total event occurrences; no completion or revenue definition is available in analytics.`, +}); +const production: DetectedSignal = { + ...activation, + metric: "goal:production-delivery", + subjectKey: "goal:production-delivery", + entityId: "production-delivery", + entityLabel: "Production report accepted", + label: "Production report acceptance rate", + definitionEvidence: + 'Goal production-delivery: EVENT customer_delivery_accepted; visitors with environment="production"; counts matching visitors, not payments.', +}; +const long = organizationProfileContext( + { + ...profile, + content: + "Example provides report preparation and delivery, with collaboration, scheduling and public demonstrations. " + .repeat(100) + .slice(0, 10_000), + teamContext: { + priority: profile.teamContext.priority.padEnd( + 1950, + " Details recorded in the business brief." + ), + successDefinition: profile.teamContext.successDefinition.padEnd( + 1950, + " Definitions are team assertions." + ), + exclusions: profile.teamContext.exclusions.padEnd( + 1950, + " Public demos are excluded." + ), + }, + }, + input.organizationId, + new Date(asOf) +); +long.sources.push({ + id: "latest-team-correction", + kind: "team_reply", + subjectKey: "funnel:first-report", + author: "Current teammate", + observedAt: "2026-09-04T23:00:00.000Z", + content: + "Correction to the saved brief: first_report_delivered now belongs only to an intentionally retained public demo. The known demo decline needs no further investigation. The production outcome is customer_delivery_accepted and is measured by goal:production-delivery. Prioritize its unexplained decline. This supersedes the old first-report emitter definition and priority. ".repeat( + 8 + ), +}); +const scenarios = [ + { + id: "manual-exclusions", + signals: [base, activation], + context, + required: "funnel:first-report", + downstream: false, + }, + { + id: "small", + signals: [base, activation], + context, + required: "funnel:first-report", + downstream: true, + }, + { + id: "busy-nine", + signals: [ + base, + ...Array.from({ length: 7 }, (_, index) => demo(index)), + activation, + ], + context, + required: "funnel:first-report", + downstream: true, + }, + { + id: "busy-twenty-four", + signals: [ + base, + ...Array.from({ length: 22 }, (_, index) => demo(index)), + activation, + ], + context, + required: "funnel:first-report", + downstream: false, + }, + { + id: "latest-correction", + signals: [activation, production], + context: long, + required: "goal:production-delivery", + downstream: false, + }, +]; + +if (import.meta.main) { + const { values } = parseArgs({ + options: { + out: { type: "string" }, + runs: { type: "string", default: "2" }, + cases: { type: "string" }, + reverse: { type: "boolean", default: false }, + }, + }); + if (!values.out) { + throw new Error( + "--out is required; use a fresh directory for every experiment" + ); + } + const runs = Number(values.runs); + if (!Number.isInteger(runs) || runs < 1 || runs > 3) { + throw new Error("--runs must be 1–3"); + } + const directory = resolve(values.out); + mkdirSync(directory, { recursive: false, mode: 0o700 }); + for (const name of [ + "agent.ts", + "business-aware-selection.ts", + "business-context.ts", + "generation.ts", + "coverage-planner.ts", + "investigation.ts", + ]) { + copyFileSync( + resolve(import.meta.dir, "..", name), + resolve(directory, name) + ); + } + copyFileSync(import.meta.path, resolve(directory, "context-selection.ts")); + copyFileSync( + resolve(import.meta.dir, "quality.ts"), + resolve(directory, "quality.ts") + ); + writeFileSync( + resolve(directory, "fixtures.json"), + JSON.stringify(scenarios, null, 2) + ); + writeFileSync( + resolve(directory, "metadata.json"), + JSON.stringify( + { + modelId, + asOf, + runs, + reverse: values.reverse, + sourceRevision: spawnSync(["git", "rev-parse", "HEAD"]) + .stdout.toString() + .trim(), + synthetic: true, + }, + null, + 2 + ) + ); + const selected = values.cases + ? scenarios.filter((item) => values.cases?.split(",").includes(item.id)) + : scenarios; + if (!selected.length) { + throw new Error("No matching cases"); + } + const results: SelectionResult[] = []; + for (let iteration = 1; iteration <= runs; iteration++) { + for (const scenario of selected) { + for (const arm of iteration % 2 + ? ["absent", "present"] + : ["present", "absent"]) { + const id = `${scenario.id}-${arm}-${iteration}`; + const trace = resolve(directory, `${id}.selection.jsonl`); + const emit = (kind: string, value: unknown) => + appendFileSync( + trace, + `${JSON.stringify({ kind, value }, (_key, item) => (item && typeof item === "object" && item.type === "reasoning" ? { type: "reasoning", text: "[omitted]" } : item))}\n`, + { mode: 0o600 } + ); + const calls: unknown[] = []; + const usage: unknown[] = []; + const model = wrapLanguageModel({ + model: createModelFromId(modelId), + middleware: { + specificationVersion: "v3", + wrapGenerate: async ({ doGenerate, params }) => { + calls.push(params.prompt); + emit("model.request", params); + try { + const response = await doGenerate(); + usage.push(response.usage); + emit("model.response", { + content: response.content.filter( + (item) => item.type !== "reasoning" + ), + usage: response.usage, + finishReason: response.finishReason, + }); + return response; + } catch (error) { + emit( + "model.error", + error instanceof Error ? error.message : String(error) + ); + throw error; + } + }, + }, + }); + const started = performance.now(); + const supplied = arm === "present" ? scenario.context : absent; + const signals = values.reverse + ? [...scenario.signals].reverse() + : scenario.signals; + emit("case.input", { input, signals, businessContext: supplied }); + const candidates = await planInvestigationsWithBusinessContext( + input, + signals, + { + loadBusinessProfile: async () => supplied, + selectCandidates: (selection) => + chooseInvestigationSignals(selection, model), + }, + false, + undefined, + { + reason: + scenario.id === "manual-exclusions" ? "manual" : "scheduled", + } + ); + const planned = candidates.map((candidate) => ({ + key: candidate.signal.signalKey, + objective: candidate.investigationObjective, + })); + const selectionMs = performance.now() - started; + emit("selection.result", { + planned, + selectionMs, + calls: calls.length, + usage, + }); + const investigations: Awaited>[] = []; + if (scenario.downstream) { + for (const candidate of candidates) { + const source = qualityCases.find( + (fixture) => + fixture.id === + (candidate.signal.signalKey === activation.subjectKey + ? "activation-source-comparison" + : "empty-evidence-signal") + ); + if (!source) { + throw new Error("Missing native investigation fixture"); + } + const fixture = { + ...source, + id: `${id}-${candidate.signal.signalKey.replaceAll(":", "-")}`, + input: { + ...source.input, + signal: candidate.signal, + investigationObjective: candidate.investigationObjective, + businessContext: candidate.businessContext, + evidence: + candidate.signal.signalKey === activation.subjectKey + ? [ + "The unchanged funnel counted 1000 visitors reaching its first step in each window. Completions fell from 180 to 100. No source or implementation cause is established.", + ] + : [], + }, + }; + investigations.push( + await evaluate(runInsightAgent, fixture, directory, 1, modelId) + ); + } + } + const result = { + id, + planned, + selectionMs, + selectionCalls: calls.length, + selectionUsage: usage, + requiredSelected: planned.some( + (candidate) => candidate.key === scenario.required + ), + requiredFirst: planned[0]?.key === scenario.required, + investigations, + }; + results.push(result); + writeFileSync( + resolve(directory, "results.json"), + JSON.stringify(results, null, 2) + ); + console.log( + JSON.stringify({ + id, + chosen: planned.map((candidate) => candidate.key), + calls: calls.length, + investigations: investigations.map((item) => ({ + id: item.id, + completed: item.completed, + failures: item.failures, + })), + }) + ); + } + } + } + process.exit(0); +} diff --git a/apps/insights/src/evals/quality.ts b/apps/insights/src/evals/quality.ts index f2ee420db..fef567d4f 100644 --- a/apps/insights/src/evals/quality.ts +++ b/apps/insights/src/evals/quality.ts @@ -2003,7 +2003,7 @@ for (const available of [true, false]) { }); } -async function evaluate( +export async function evaluate( agent: typeof runInsightAgent, fixture: QualityCase, directory: string,