From 94d8e8f3a529fadafff2212778900006b6a62758 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 00:43:08 +0530 Subject: [PATCH 01/37] feat: structured lineage and column reach for dbt_lineage and impact_analysis Builds on the dbt integration's structured dbt_lineage result. - dbt_lineage takes a project-relative SQL path or a snapshot as well as a model, and with include_graph returns the model's graph from the manifest (two levels up, everything downstream with exposures and tests, capped at 500 nodes). The manifest fallback reports upstream ids, column mappings and edges as data. - impact_analysis with a column traces each downstream model's and snapshot's compiled SQL in dependency order and reports column_reach: certain readers with the columns they carry, possible readers (filters and joins, select * over undocumented columns, shared table names, ephemeral parents), untraced models and the exposures on each. It matches models by unique_id and honours the dialect override. - The MCP server declares an output schema for both tools. Structured content is the tool's own result when it builds one (the dbt integration's model lineage), else its lineage fields, always with the text report and capped at 100 KB, errors included. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/mcp-server/server.ts | 77 +++- .../src/altimate/native/dbt/column-reach.ts | 336 ++++++++++++++ .../opencode/src/altimate/native/dbt/graph.ts | 117 +++++ .../src/altimate/native/dbt/lineage.ts | 41 +- .../opencode/src/altimate/native/types.ts | 6 + .../src/altimate/tools/dbt-lineage.ts | 36 +- .../src/altimate/tools/impact-analysis.ts | 159 ++++--- .../test/altimate/column-reach.test.ts | 413 ++++++++++++++++++ .../test/altimate/dbt-lineage-helpers.test.ts | 18 + .../altimate/dbt-lineage-structured.test.ts | 3 +- .../test/altimate/mcp-server/server.test.ts | 54 +++ .../altimate/tools/impact-analysis.test.ts | 96 +++- 12 files changed, 1286 insertions(+), 70 deletions(-) create mode 100644 packages/opencode/src/altimate/native/dbt/column-reach.ts create mode 100644 packages/opencode/src/altimate/native/dbt/graph.ts create mode 100644 packages/opencode/test/altimate/column-reach.test.ts diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 0374078302..521b0820d2 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -43,6 +43,72 @@ export async function serverInstructions(): Promise { } } +// altimate_change start — lineage tools answer with structured content as well as text +// Lineage tools also return their metadata as structured content, so a client that draws lineage +// (the Claude Code plugin's lineage pane) reads edges and downstream lists without parsing text. +// Claude Code shows structured content to the model in place of the text, so `report` carries the +// (already truncated) text (dbt_lineage: its header, its mappings travel as data), only the lineage +// fields are passed, and the whole is capped. +const STRUCTURED_FIELDS: Record = { + dbt_lineage: ["model_name", "model_unique_id", "confidence", "confidence_factors", "upstream", "columns", "edges", "error"], + impact_analysis: [ + "model", + "column", + "change_type", + "severity", + "direct_count", + "transitive_count", + "test_count", + "downstream", + "exposures", + "column_reach", + "column_reach_error", + "exposures_error", + "error", + ], +} +export const STRUCTURED_TOOLS: ReadonlySet = new Set(Object.keys(STRUCTURED_FIELDS)) +const STRUCTURED_OUTPUT_SCHEMA = { + type: "object" as const, + properties: { + report: { type: "string", description: "The tool's text report" }, + truncated: { type: "boolean", description: "True when the lineage fields were left out for size; the report stands" }, + }, + required: ["report"], + additionalProperties: true, +} +// The structured result reaches the model in Claude Code, so it stays within the same order of +// size as a text result: past this, only the report is sent. +const STRUCTURED_MAX_BYTES = 100_000 + +export function structuredResult(tool: string, metadata: Record, report: string): Record { + // A tool that builds its own structured result (dbt_lineage from the dbt integration) hands it + // over whole; otherwise the lineage fields of its metadata are passed. + const own = metadata.structuredContent + if (own && typeof own === "object" && !Array.isArray(own)) { + const full = { ...(own as Record), report } + return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : { report, truncated: true } + } + const fields = Object.fromEntries((STRUCTURED_FIELDS[tool] ?? []).filter((key) => key in metadata).map((key) => [key, metadata[key]])) + // dbt_lineage's text prints the same mappings and edges as `columns` and `edges`: keep its header. + const head = tool === "dbt_lineage" && "columns" in fields ? report.split("\nColumn Mappings:")[0]!.trim() : report + const full = { ...fields, report: head } + return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : { report, truncated: true } +} +/** An error report held to the same cap as a result, so a huge exception message cannot exceed it. */ +export function structuredError(message: string): Record { + // Measured as sent: JSON escaping can grow the text (a newline is two bytes, a control + // character six), so the cap applies to the serialized result, shortening until it fits. + const size = (value: Record) => Buffer.byteLength(JSON.stringify(value)) + if (size({ report: message }) <= STRUCTURED_MAX_BYTES) return { report: message } + let report = message + while (report.length > 0 && size({ report, truncated: true }) > STRUCTURED_MAX_BYTES) { + report = report.slice(0, Math.floor(report.length * 0.9)) + } + return { report, truncated: true } +} +// altimate_change end + /** Spilled results: a resource_link next to the text, which keeps the preview, row count and path. */ function resultLinks(metadata: Record) { // A batched sql_execute call reports one saved file per spilled query. @@ -97,6 +163,7 @@ export async function createServer(options: ServeOptions = {}) { name, description: descriptions[name] ?? tool.description, inputSchema: tool.jsonSchema as McpTool["inputSchema"], + ...(STRUCTURED_TOOLS.has(name) && { outputSchema: STRUCTURED_OUTPUT_SCHEMA }), })), ...(workspace?.tools ?? [])], })) // Ask the user once per session when the model weighed two readings and the client can ask; any @@ -150,15 +217,21 @@ export async function createServer(options: ServeOptions = {}) { const structured = result.metadata?.structuredContent return { content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], - ...(structured && typeof structured === "object" && !isError + // Tools outside the declared set may still hand over a structured result of their own. + ...(structured && typeof structured === "object" && !isError && !STRUCTURED_TOOLS.has(request.params.name) ? { structuredContent: structured as Record } : {}), isError, + ...(STRUCTURED_TOOLS.has(request.params.name) && { + structuredContent: structuredResult(request.params.name, result.metadata, result.output), + }), } } catch (error) { + const message = error instanceof Error ? error.message : String(error) return { - content: [{ type: "text" as const, text: error instanceof Error ? error.message : String(error) }], + content: [{ type: "text" as const, text: message }], isError: true, + ...(STRUCTURED_TOOLS.has(request.params.name) && { structuredContent: structuredError(message) }), } } } diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts new file mode 100644 index 0000000000..3a39a9f446 --- /dev/null +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -0,0 +1,336 @@ +// altimate_change - new file +// +// Which downstream models read a column, and which exposures (dashboards) sit on them. Each +// downstream model's (or snapshot's) compiled SQL is traced with dbtLineage, in dependency order, so +// a column that is renamed or carried through an intermediate model is still followed. What the +// lineage engine cannot settle is reported as such: models it could not trace (no compiled SQL, or +// SQL it could not parse), models that name the column only outside their select list (where the +// engine reports no edge), and models that read it through a table name several parents share. +import { loadRawManifest, findModel } from "./helpers" +import { dbtLineage } from "./lineage" + +export interface ColumnReachModel { + name: string + unique_id: string + /** Its output columns that come from the traced column. */ + columns: string[] +} + +export interface ColumnReachUnsure { + name: string + unique_id: string + /** Why: no compiled SQL or a parse error, a mention outside the select list, an ambiguous table name, or an unsettled parent. */ + reason: string +} + +export interface ColumnReachExposure { + /** Identifies the exposure across packages that reuse a name. */ + unique_id: string + name: string + /** The exposure's label when it has one, else its name. */ + label: string + type: string +} + +export interface ColumnReach { + /** Models whose output columns certainly come from the column. */ + models: ColumnReachModel[] + /** + * Models that may read it: they name it outside their select list (or a column of that name + * another parent shares), read it through a table name more than one parent shares, or sit + * downstream of a model that could not be traced or read it ambiguously. + */ + uncertain: ColumnReachUnsure[] + /** Downstream models that could not be traced (no compiled SQL, or SQL the engine could not parse). */ + untraced: ColumnReachUnsure[] + /** Exposures on the model or on any model that certainly reads the column. */ + exposures: ColumnReachExposure[] + /** Exposures on models that may read it or could not be traced (and on none of the certain readers). */ + uncertain_exposures: ColumnReachExposure[] + /** True when the walk stopped at the model limit before covering everything downstream. */ + truncated: boolean +} + +const MAX_MODELS = 300 +const TRACED = new Set(["model", "snapshot"]) + +/** "db"."schema"."orders"."amount" → { parts: ["db","schema","orders"], column: "amount" }, lower-cased. */ +function splitSource(source: unknown): { parts: string[]; column: string; exact: string } | null { + const text = typeof source === "string" ? source : typeof (source as any)?.source === "string" ? (source as any).source : null + if (!text) return null + // Split on dots outside quotes: "a"."net.amount" is table a, column net.amount. + const raw = [...text.matchAll(/"([^"]*)"|`([^`]*)`|\[([^\]]*)\]|([^."`\[\]]+)/g)].map((m) => m[1] ?? m[2] ?? m[3] ?? m[4] ?? "") + if (raw.length < 2) return null + // `exact` keeps the column's case, for parents that document two columns differing only by case. + return { parts: raw.slice(0, -1).map((part) => part.toLowerCase()), column: raw.at(-1)!.toLowerCase(), exact: raw.at(-1)! } +} + +/** The ways a parent's relation can be written in a child's compiled SQL, most qualified first. */ +function relationKeys(node: any): string[] { + const table = [node?.alias, node?.identifier, node?.name].filter((v): v is string => typeof v === "string" && v !== "") + const schema = typeof node?.schema === "string" ? node.schema : null + const database = typeof node?.database === "string" ? node.database : null + const keys = new Set() + for (const name of table) { + if (database && schema) keys.add(`${database}.${schema}.${name}`.toLowerCase()) + if (schema) keys.add(`${schema}.${name}`.toLowerCase()) + keys.add(name.toLowerCase()) + } + return [...keys] +} + +/** + * The parents an edge's source table names, matched at the qualification the edge carries: a + * qualified name that matches no parent names none (it is some other relation), and a bare name + * more than one parent answers to names them all. + */ +export function parentsNamed(parts: string[], parents: string[], lookup: (id: string) => any): string[] { + const key = parts.slice(-3).join(".") + return parents.filter((parent) => relationKeys(lookup(parent)).includes(key)) +} + +/** Whether a node documents more than one column with this name once case is ignored ("revenue" and "Revenue"). */ +function caseCollides(node: any, lower: string): boolean { + const names = new Set(Object.entries(node?.columns ?? {}).map(([key, col]) => String(col?.name ?? key))) + return [...names].filter((name) => name.toLowerCase() === lower).length > 1 +} + +/** Column names the manifest documents for a node, lower-cased. */ +function documentedColumns(node: any): Set { + return new Set(Object.entries(node?.columns ?? {}).map(([key, col]) => String(col?.name ?? key).toLowerCase())) +} + +const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** + * Whether compiled SQL names the column anywhere but as an output alias, comments and string + * literals aside. Used only to mark a model as one that may read the column. + */ +/** SQL with comments blanked and string literals emptied, in one pass so neither can swallow the other. */ +function stripSql(sql: string): string { + return sql.replace(/'(?:[^']|'')*'|--[^\r\n]*|\/\*[\s\S]*?\*\//g, (match) => (match.startsWith("'") ? "''" : " ")) +} + +/** + * Whether the SQL selects `*` or `t.*` in any form (DISTINCT ON, TOP n, AS STRUCT, multi-part or + * quoted qualifiers, EXCLUDE/EXCEPT/REPLACE/RENAME modifiers). Every `*` counts except count(*) and + * multiplication, which has an operand on both sides that is not a keyword ending the projection. + */ +export function selectsStar(sql: string): boolean { + const code = stripSql(sql) + const ends = /^(from|exclude|except|replace|rename|ilike|columns)\b/i + for (let at = code.indexOf("*"); at !== -1; at = code.indexOf("*", at + 1)) { + const before = code.slice(0, at).trimEnd() + const after = code.slice(at + 1).trimStart() + const prev = before.at(-1) ?? "" + const next = after[0] ?? "" + // count(*) reads no column; hash(*), object_construct(*), struct(*) read them all. + if (prev === "(" && next === ")" && /\bcount\s*\($/i.test(before)) continue + if (prev === "." || next === "" || next === "," || next === ")" || ends.test(after)) return true + const operand = /[\w"'`\])]/ + if (operand.test(prev) && (operand.test(next) || next === "(" || /\d/.test(next))) { + // An operand before the `*` that is itself a keyword (SELECT, DISTINCT, ALL, STRUCT) is no operand. + if (/\b(select|distinct|all|struct)$/i.test(before)) return true + continue + } + return true + } + return false +} + +function namesColumn(sql: string, column: string): boolean { + const code = stripSql(sql) + const name = escapeRegExp(column) + const ident = new RegExp(`(^|[^A-Za-z0-9_$])("${name}"|${name})(?![A-Za-z0-9_$])`, "gi") + for (const match of code.matchAll(ident)) { + const before = code.slice(0, match.index! + match[1]!.length) + if (/\bas\s*$/i.test(before)) continue + return true + } + return false +} + +function childMapOf(manifest: any): Record { + if (manifest.child_map) return manifest.child_map + const children: Record = {} + for (const group of [manifest.nodes, manifest.exposures]) { + for (const [id, node] of Object.entries(group ?? {})) { + for (const parent of node?.depends_on?.nodes ?? []) (children[parent] ??= []).push(id) + } + } + return children +} + +export function columnReach(manifestPath: string, model: string, column: string, dialect?: string): ColumnReach { + const manifest = loadRawManifest(manifestPath) + if (!manifest) throw new Error(`Manifest not found at ${manifestPath}`) + const nodes: Record = manifest.nodes ?? {} + const lookup = (id: string) => nodes[id] ?? manifest.sources?.[id] + const target = findModel(nodes, model) + if (!target) throw new Error(`Model '${model}' not found in manifest`) + const targetId: string = target.unique_id ?? Object.keys(nodes).find((id) => nodes[id] === target) + const children = childMapOf(manifest) + + // Everything downstream, then a topological order over it, so each node is traced after all its + // traced parents. + const downstream = new Set() + const queue = [targetId] + while (queue.length) { + for (const child of children[queue.shift()!] ?? []) { + if (downstream.has(child) || !TRACED.has(nodes[child]?.resource_type)) continue + downstream.add(child) + queue.push(child) + } + } + const pending = new Map() + for (const id of downstream) { + const parents: string[] = nodes[id]?.depends_on?.nodes ?? [] + pending.set(id, parents.filter((p) => downstream.has(p)).length) + } + const ready = [...downstream].filter((id) => pending.get(id) === 0) + const order: string[] = [] + while (ready.length) { + const id = ready.shift()! + order.push(id) + for (const child of children[id] ?? []) { + if (!pending.has(child)) continue + const left = pending.get(child)! - 1 + pending.set(child, left) + if (left === 0) ready.push(child) + } + } + + const column0 = column.toLowerCase() + // Columns certainly carried from the traced column, per node. + const tracked = new Map>([[targetId, new Set([column0])]]) + // The same columns with their case kept: consulted only where a parent's names differ by case alone. + const trackedExact = new Map>([[targetId, new Set([column])]]) + const carries = (parent: string, source: { column: string; exact: string }) => + !!tracked.get(parent)?.has(source.column) && + (!caseCollides(lookup(parent), source.column) || !!trackedExact.get(parent)?.has(source.exact)) + // Nodes whose outputs are unknown (untraced, or read through an ambiguous name): their children + // may read the column too. A node that only filters on it was traced and carries nothing on. + const unsure = new Set() + const reached: ColumnReachModel[] = [] + const uncertain: ColumnReachUnsure[] = [] + const untraced: ColumnReachUnsure[] = [] + let truncated = false + let traced = 0 + const entry = (id: string, reason: string) => ({ name: nodes[id]?.name ?? id, unique_id: id, reason }) + + for (const id of order) { + const allParents: string[] = nodes[id]?.depends_on?.nodes ?? [] + const parents = allParents.filter((p) => tracked.has(p)) + const fromUnsure = allParents.some((p) => unsure.has(p)) + if (parents.length === 0 && !fromUnsure) continue + if (traced++ >= MAX_MODELS) { + truncated = true + break + } + + const result = dbtLineage({ manifest_path: manifestPath, model: id, dialect }) + const lineage = result.raw_lineage as any + // dbtLineage rates a model low when the engine threw or answered SQL it cannot parse (`errors`, no edges). + if (result.confidence === "low") { + untraced.push(entry(id, result.confidence_factors[0] ?? "the lineage engine could not trace it")) + unsure.add(id) + continue + } + const edges: { source: unknown; target: string; lineage_type?: string }[] = Array.isArray(lineage?.column_lineage) + ? lineage.column_lineage + : Object.entries(lineage?.column_dict ?? {}).flatMap(([out, sources]) => + (Array.isArray(sources) ? sources : [sources]).map((source) => ({ source, target: out, lineage_type: "direct" })), + ) + + // Two parents with the same table name share one entry in the engine's schema, so its edges + // cannot say which one is read; a `select *` from a parent whose schema does not list the + // column cannot be expanded to it. Both are possible reads with unknown outputs. + const sqlText: string = nodes[id]?.compiled_code || nodes[id]?.compiled_sql || "" + const bareName = (p: string) => { + const node = lookup(p) + return String(node?.alias || node?.identifier || node?.name || p).toLowerCase() + } + const collides = parents.some((p) => allParents.some((other) => other !== p && bareName(other) === bareName(p))) + const wildcard = + selectsStar(sqlText) && parents.some((p) => [...tracked.get(p)!].some((col) => !documentedColumns(lookup(p)).has(col))) + if (collides || wildcard) { + unsure.add(id) + uncertain.push( + entry(id, collides ? "reads a parent whose table name another parent shares" : "selects * from a parent whose schema.yml does not list the column"), + ) + continue + } + + const carried = new Set() + const carriedExact = new Set() + let reads = false + let ambiguous = false + for (const edge of edges) { + const source = splitSource(edge.source) + if (!source) continue + const candidates = parentsNamed(source.parts, allParents, lookup) + if (!candidates.some((p) => carries(p, source))) continue + // A bare table name that more than one parent answers to cannot say which one is read. + if (candidates.length > 1) { + ambiguous = true + continue + } + reads = true + if (edge.lineage_type !== "indirect" && typeof edge.target === "string") { + carried.add(edge.target.toLowerCase()) + carriedExact.add(edge.target) + } + } + // The engine has no schema for the tables inside an inlined ephemeral parent, so it reports no + // edge through it: unknown, not unread, even when another parent's edges are traced. + const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") + if (reads) { + tracked.set(id, carried) + trackedExact.set(id, carriedExact) + // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. + if (ambiguous || fromUnsure || viaEphemeral) unsure.add(id) + reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) + continue + } + + if (viaEphemeral) { + unsure.add(id) + uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) + continue + } + const sql: string = nodes[id]?.compiled_code || nodes[id]?.compiled_sql || "" + const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sql, col))) + const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) + if (ambiguous || named.length > 0 || fromUnsure) { + if (ambiguous || fromUnsure) unsure.add(id) + uncertain.push( + entry( + id, + ambiguous + ? "reads it through a table name more than one parent shares" + : named.length > 0 + ? shared + ? "names it outside its select list, or names another parent's column of the same name" + : "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)" + : "is downstream of a model that could not be traced or reads it ambiguously", + ), + ) + } + } + + const reachedIds = new Set([targetId, ...reached.map((m) => m.unique_id)]) + const certain = exposuresOn(manifestPath, reachedIds) + const maybe = exposuresOn(manifestPath, [...uncertain, ...untraced].map((m) => m.unique_id)).filter( + (e) => !certain.some((c) => c.unique_id === e.unique_id), + ) + return { models: reached, uncertain, untraced, exposures: certain, uncertain_exposures: maybe, truncated } +} + +/** Exposures that depend on any of the given unique_ids, read from the raw manifest. */ +export function exposuresOn(manifestPath: string, ids: Iterable): ColumnReachExposure[] { + const manifest = loadRawManifest(manifestPath) + const wanted = new Set(ids) + return Object.entries(manifest?.exposures ?? {}) + .filter(([, node]) => (node?.depends_on?.nodes ?? []).some((dep: string) => wanted.has(dep))) + .map(([id, node]) => ({ unique_id: id, name: node?.name ?? id, label: node?.label || node?.name || id, type: node?.type ?? "exposure" })) +} diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts new file mode 100644 index 0000000000..698bf146ee --- /dev/null +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -0,0 +1,117 @@ +// altimate_change - new file +// +// The part of a dbt manifest's graph a lineage viewer draws for one model: two levels upstream, +// everything downstream (with the tests counted as ids), and the links between them. Clients that +// cannot read a large manifest themselves (the Claude Code plugin reads files up to 4 MiB) get it +// from the CLI (`altimate tool run dbt_lineage --json` with include_graph); MCP carries no graph. + +export interface GraphNode { + id: string + name: string + /** An exposure's label; null for other nodes. */ + label: string | null + type: string + path: string | null + materialized: string | null + /** database.schema.table, lower-cased. */ + relation: string | null + columns: string[] +} + +export interface LineageGraph { + nodes: Record + parents: Record + children: Record + /** True when downstream stopped at the node limit. */ + truncated: boolean +} + +const GROUPS = ["nodes", "sources", "exposures", "metrics", "semantic_models", "saved_queries", "unit_tests"] as const +/** Limit on downstream models only; the target, its two upstream levels and test nodes come on top. */ +const MAX_DOWNSTREAM = 500 + +function lookup(manifest: any, id: string): any { + for (const group of GROUPS) if (manifest[group]?.[id]) return manifest[group][id] + return undefined +} + +function relationOf(node: any): string | null { + // A manifest that carries relation_name with null (ephemeral models, analyses, tests) has no relation. + if (node && "relation_name" in node && !node.relation_name) return null + if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name.replace(/[`"\[\]]/g, "").toLowerCase() + const parts = [node?.database, node?.schema, node?.alias ?? node?.identifier ?? node?.name].filter( + (part): part is string => typeof part === "string" && part !== "", + ) + return parts.length === 3 ? parts.join(".").toLowerCase() : null +} + +function nodeOf(id: string, node: any): GraphNode { + const type: string = node?.resource_type ?? id.split(".")[0] ?? "unknown" + return { + id, + name: node?.name ?? id.split(".").pop() ?? id, + label: type === "exposure" ? node?.label || node?.name || id : null, + type, + path: node?.original_file_path ?? null, + materialized: node?.config?.materialized ?? null, + relation: relationOf(node), + columns: Object.entries(node?.columns ?? {}).map(([key, col]) => String(col?.name ?? key)), + } +} + +function mapsOf(manifest: any): { parents: Record; children: Record } { + if (manifest.parent_map && manifest.child_map) return { parents: manifest.parent_map, children: manifest.child_map } + const parents: Record = {} + const children: Record = {} + for (const group of GROUPS) { + for (const [id, node] of Object.entries(manifest[group] ?? {})) { + parents[id] = node?.depends_on?.nodes ?? [] + for (const parent of parents[id]) (children[parent] ??= []).push(id) + } + } + return { parents, children } +} + +export function lineageGraph(manifest: any, id: string): LineageGraph { + const { parents, children } = mapsOf(manifest) + const isTest = (one: string) => ["test", "unit_test"].includes(lookup(manifest, one)?.resource_type ?? one.split(".")[0]) + const keep = new Set([id]) + for (const parent of parents[id] ?? []) { + keep.add(parent) + for (const grand of parents[parent] ?? []) keep.add(grand) + } + let truncated = false + const queue = [id] + const seen = new Set([id]) + let count = 0 + while (queue.length) { + for (const child of children[queue.shift()!] ?? []) { + if (seen.has(child) || isTest(child)) continue + if (count >= MAX_DOWNSTREAM) { + truncated = true + queue.length = 0 + break + } + seen.add(child) + keep.add(child) + count++ + queue.push(child) + } + } + const graph: LineageGraph = { nodes: {}, parents: {}, children: {}, truncated } + for (const one of keep) { + graph.nodes[one] = nodeOf(one, lookup(manifest, one)) + graph.parents[one] = (parents[one] ?? []).filter((other) => keep.has(other)) + graph.children[one] = (children[one] ?? []).filter((other) => { + if (keep.has(other)) return true + if (!isTest(other)) return false + if (!graph.nodes[other]) { + graph.nodes[other] = { ...nodeOf(other, lookup(manifest, other)), type: "test" } + graph.parents[other] = (parents[other] ?? []).filter((p) => keep.has(p)) + graph.children[other] = [] + } + return true + }) + } + return graph +} diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index 49b4cf9a96..a5918f60b5 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -7,6 +7,13 @@ import * as core from "@altimateai/altimate-core" import type { DbtLineageParams, DbtLineageResult } from "../types" import { loadRawManifest, findModel, getUniqueId, detectDialect, buildSchemaContext } from "./helpers" +import { lineageGraph } from "./graph" + +function unparsed(lineage: Record): boolean { + const errors = Array.isArray(lineage.errors) ? lineage.errors : [] + const edges = Array.isArray(lineage.column_lineage) ? lineage.column_lineage : [] + return errors.length > 0 && edges.length === 0 +} /** * Compute column-level lineage for a dbt model. @@ -35,16 +42,35 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { const nodes = manifest.nodes || {} const sources = manifest.sources || {} - // Find target model by name or unique_id - const modelNode = findModel(nodes, params.model) + // Find target model by name or unique_id; a snapshot by its unique_id (it has compiled SQL too) + // A model by name or unique_id; a snapshot by unique_id; either by its project-relative file path. + const byPath = Object.entries(nodes).find( + ([, node]) => ["model", "snapshot"].includes(node?.resource_type) && node.original_file_path === params.model, + ) + const modelNode = + findModel(nodes, params.model) ?? + (nodes[params.model]?.resource_type === "snapshot" ? nodes[params.model] : null) ?? + byPath?.[1] ?? + Object.values(nodes).find((node) => node?.resource_type === "snapshot" && node.name === params.model) ?? + null if (!modelNode) { return emptyResult([`Model '${params.model}' not found in manifest`]) } + const uniqueId = + getUniqueId(nodes, params.model) ?? + (nodes[params.model] ? params.model : (byPath?.[0] ?? Object.keys(nodes).find((id) => nodes[id] === modelNode))) + const graph = params.include_graph && uniqueId ? { graph: lineageGraph(manifest, uniqueId) } : {} // Extract compiled SQL const sql = modelNode.compiled_code || modelNode.compiled_sql || "" if (!sql) { - return emptyResult(["No compiled SQL found — run `dbt compile` first"]) + return { + ...emptyResult(["No compiled SQL found — run `dbt compile` first"]), + model_name: modelNode.name || params.model, + model_unique_id: uniqueId, + upstream: modelNode.depends_on?.nodes ?? [], + ...graph, + } } // Detect dialect @@ -66,10 +92,13 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { return { model_name: modelNode.name || params.model, - model_unique_id: getUniqueId(nodes, params.model), + model_unique_id: uniqueId, compiled_sql: sql, raw_lineage: rawLineage, - confidence: rawLineage.error ? "low" : "high", - confidence_factors: rawLineage.error ? [String(rawLineage.error)] : [], + // The engine reports SQL it cannot parse in `errors` with no edges, rather than throwing. + confidence: rawLineage.error || unparsed(rawLineage) ? "low" : "high", + confidence_factors: rawLineage.error ? [String(rawLineage.error)] : unparsed(rawLineage) ? [String((rawLineage.errors as unknown[])[0])] : [], + upstream: upstreamIds, + ...graph, } } diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 431d1f41a5..91af63d60a 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -1076,6 +1076,8 @@ export interface DbtLineageParams { manifest_path: string model: string dialect?: string + /** Also return the model's lineage graph (two levels up, everything downstream). */ + include_graph?: boolean } export interface DbtLineageResult { @@ -1085,6 +1087,10 @@ export interface DbtLineageResult { raw_lineage: Record confidence: string confidence_factors: string[] + /** unique_ids the model depends on (`depends_on.nodes`). */ + upstream?: string[] + /** With include_graph: the graph a lineage viewer draws for the model. */ + graph?: import("./dbt/graph").LineageGraph } // --- dbt Profile Discovery --- diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index c97f6daf90..05dc561d65 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -3,6 +3,8 @@ import path from "path" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { DbtLineageResult } from "../native/types" +import { loadRawManifest } from "../native/dbt/helpers" +import { lineageGraph } from "../native/dbt/graph" /** The structured result of dbt-integration's `getModelLineage` (version 1), relayed as is. */ export type ModelLineage = { @@ -36,27 +38,37 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { description: "Column-level lineage for a dbt model: its upstream models, seeds and sources, its downstream models, their columns, and the column edges between them across every hop. Pass `column` to trace one column. Uses the project's dbt integration (compiles on demand, no `dbt compile` needed) and returns the result as structured content; without it, reads the compiled SQL in the manifest.", parameters: z.object({ - model: z.string().describe("Model name or unique_id (e.g. 'my_model' or 'model.project.my_model')"), + model: z + .string() + .describe("Model name, unique_id or project-relative SQL path (e.g. 'my_model', 'model.project.my_model', 'models/my_model.sql'); a snapshot by unique_id or path"), column: z.string().optional().describe("Trace only this column of the model, upstream and downstream"), manifest_path: z .string() .optional() .describe( - "manifest.json for the manifest-only fallback, read only when the dbt integration isn't available (default: the dbt project's target/manifest.json)", + "manifest.json for the manifest-only fallback, read only when the dbt integration isn't available, and for the graph (default: the dbt project's target/manifest.json)", ), dialect: z.string().optional().describe("SQL dialect override for the manifest-only fallback"), + include_graph: z + .boolean() + .optional() + .describe("Also return the model's graph from the manifest (two levels upstream, everything downstream up to 500 models with exposures and tests; graph.truncated is set when cut) in the metadata; for lineage viewers"), }), async execute(args, ctx) { const viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) if (viaAdapter.result) { const result = viaAdapter.result const what = result.focus.column ? `${result.focus.name}.${result.focus.column}` : result.focus.name + // A lineage viewer also wants the manifest's graph: exposures and tests sit there, not here. + const graph = args.include_graph ? manifestGraph(args.manifest_path ?? (await defaultManifestPath()), result.focus.uniqueId) : undefined return { title: `dbt Lineage: ${what} [${result.confidence}]`, metadata: { model_name: result.focus.name, + model_unique_id: result.focus.uniqueId, confidence: result.confidence, structuredContent: result, + ...(graph && { graph }), }, output: formatModelLineage(result), } @@ -69,16 +81,26 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { manifest_path: manifestPath, model: args.model, dialect: args.dialect, + include_graph: args.include_graph, }) if (notUsed) result.confidence_factors.push(notUsed) if (args.column) { result.confidence_factors.push(`tracing ${args.column} needs the dbt integration; this is the whole model's lineage`) } + const lineage = result.raw_lineage as { column_dict?: Record; column_lineage?: unknown[] } return { title: `dbt Lineage: ${result.model_name} [${result.confidence}]`, + // The same lineage as the text, as data: callers (the MCP server's structured result, the + // Claude Code plugin's lineage pane) read it here instead of parsing the text. metadata: { model_name: result.model_name, + model_unique_id: result.model_unique_id ?? null, confidence: result.confidence, + confidence_factors: result.confidence_factors, + upstream: result.upstream ?? [], + columns: lineage?.column_dict ?? {}, + edges: Array.isArray(lineage?.column_lineage) ? lineage.column_lineage : [], + ...(result.graph && { graph: result.graph }), }, output: formatDbtLineage(result), } @@ -93,6 +115,16 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { }, }) +/** The model's graph from the compiled manifest, or undefined when there is no manifest to read. */ +function manifestGraph(manifestPath: string, uniqueId: string) { + try { + const manifest = loadRawManifest(manifestPath) + return manifest ? lineageGraph(manifest, uniqueId) : undefined + } catch { + return undefined + } +} + /** * The lineage from the project's dbt adapter, which the tool starts in read-only mode too: it * compiles and reads column metadata, and runs no user SQL. `reason` says why it wasn't used. diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index 205811f81e..63e205d389 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -6,6 +6,7 @@ import z from "zod" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { Telemetry } from "../telemetry" +import { columnReach, exposuresOn, type ColumnReach, type ColumnReachExposure } from "../native/dbt/column-reach" export const ImpactAnalysisTool = Tool.define("impact_analysis", { description: [ @@ -20,14 +21,14 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { '- impact_analysis({ manifest_path: "target/manifest.json", model: "dim_customers", change_type: "retype" })', ].join("\n"), parameters: z.object({ - model: z.string().describe("dbt model name to analyze impact for (e.g., 'stg_orders', 'dim_customers')"), + model: z.string().describe("dbt model name or unique_id to analyze impact for (e.g., 'stg_orders', 'model.shop.dim_customers')"), column: z .string() .optional() .describe("Specific column to trace impact for. If omitted, analyzes model-level impact."), change_type: z.enum(["remove", "rename", "retype", "add", "modify"]).describe("Type of change being considered"), manifest_path: z.string().optional().default("target/manifest.json").describe("Path to dbt manifest.json file"), - dialect: z.string().optional().default("snowflake").describe("SQL dialect for lineage analysis"), + dialect: z.string().optional().describe("SQL dialect for column lineage (auto-detected from the manifest if omitted)"), }), async execute(args, ctx) { try { @@ -43,9 +44,9 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { } // Step 2: Find the target model and its downstream dependents - const targetModel = manifest.models.find( - (m: { name: string }) => m.name === args.model || m.name.endsWith(`.${args.model}`), - ) + const targetModel = + manifest.models.find((m: { unique_id?: string }) => m.unique_id === args.model) ?? + manifest.models.find((m: { name: string }) => m.name === args.model || m.name.endsWith(`.${args.model}`)) if (!targetModel) { const available = manifest.models @@ -66,52 +67,55 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { } // Find all models that depend on the target (direct + transitive) - const downstream = findDownstream(args.model, manifest.models) + const downstream = findDownstream(targetModel.name, manifest.models, targetModel.unique_id) const direct = downstream.filter((d) => d.depth === 1) const transitive = downstream.filter((d) => d.depth > 1) // Step 4: Count only tests that reference the target model or its downstream models - const affectedModelIds = new Set([ - targetModel.unique_id, - ...downstream.map((d) => modelsByName.get(d.name)?.unique_id).filter(Boolean), - ]) + const affectedModelIds = new Set( + [targetModel.unique_id, ...downstream.map((d) => d.unique_id ?? modelsByName.get(d.name)?.unique_id)].filter( + (id): id is string => Boolean(id), + ), + ) const affectedTests = (manifest.tests ?? []).filter((t) => t.depends_on?.some((dep) => affectedModelIds.has(dep)), ) const affectedTestCount = affectedTests.length - // Step 5: If column specified, attempt column-level lineage - let columnImpact: string[] = [] + // Step 5: Exposures (dashboards) on the model or anything downstream, and, for a column, the + // downstream models whose SQL reads it, traced model by model through the compiled manifest. + let exposures: ColumnReachExposure[] = [] + let reach: ColumnReach | undefined + let reachError: string | undefined + let exposuresError: string | undefined + try { + exposures = exposuresOn(args.manifest_path, affectedModelIds) + } catch (error) { + // A manifest the parser read but the raw loader cannot: say so rather than imply no dashboards. + exposuresError = error instanceof Error ? error.message : String(error) + } if (args.column) { try { - const lineageResult = await Dispatcher.call("lineage.check", { - sql: `SELECT * FROM ${args.model}`, // Use model reference for lineage tracing - dialect: args.dialect, - }) - if (lineageResult.data?.column_dict) { - // Find which downstream columns reference our target column - for (const [outCol, sources] of Object.entries(lineageResult.data.column_dict)) { - const srcArray = Array.isArray(sources) ? sources : [sources] - if (srcArray.some((s: any) => JSON.stringify(s).includes(args.column!))) { - columnImpact.push(outCol) - } - } - } - } catch { - // Column lineage is best-effort — continue without it + reach = columnReach(args.manifest_path, targetModel.unique_id ?? args.model, args.column, args.dialect) + } catch (error) { + // The model-level impact still stands; the reason is reported beside it. + reachError = error instanceof Error ? error.message : String(error) } } // Step 6: Format the impact report const output = formatImpactReport({ - model: args.model, + model: targetModel.name, column: args.column, changeType: args.change_type, direct, transitive, affectedTestCount, - columnImpact, totalModels: manifest.model_count, + exposures, + exposuresError, + columnReach: reach, + columnReachError: reachError, }) const totalAffected = downstream.length @@ -138,8 +142,16 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { direct_count: direct.length, transitive_count: transitive.length, test_count: affectedTestCount, - column_impact: columnImpact.length, + column_reader_count: reach ? reach.models.length : 0, has_schema: false, + model: targetModel.name, + column: args.column ?? null, + change_type: args.change_type, + downstream: downstream.map((d) => ({ name: d.name, depth: d.depth, materialized: d.materialized ?? null, path: d.path })), + exposures, + ...(reach && { column_reach: reach }), + ...(reachError && { column_reach_error: reachError }), + ...(exposuresError && { exposures_error: exposuresError }), ...(findings.length > 0 && { findings }), }, output, @@ -157,6 +169,8 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { export interface DownstreamModel { name: string + /** The node's unique_id when the manifest gives one (versions of a model share a name). */ + unique_id?: string depth: number materialized?: string path: string[] @@ -164,30 +178,41 @@ export interface DownstreamModel { export function findDownstream( targetName: string, - models: Array<{ name: string; depends_on: string[]; materialized?: string }>, + models: Array<{ name: string; unique_id?: string; depends_on: string[]; materialized?: string }>, + targetId?: string, ): DownstreamModel[] { const results: DownstreamModel[] = [] const visited = new Set() - function walk(name: string, depth: number, path: string[]) { - for (const model of models) { - if (visited.has(model.name)) continue - const deps = model.depends_on.map((d) => d.split(".").pop()) - if (deps.includes(name)) { - visited.add(model.name) - const newPath = [...path, model.name] + // Breadth-first so a model's depth is its shortest distance from the target. A parent is matched + // by unique_id when known (a versioned model's id ends in its version, not its name; another + // package may have a model of the same name), else by the last segment of each dependency id. + let level: { name: string; id: string | undefined; path: string[] }[] = [ + { name: targetName, id: targetId, path: [targetName] }, + ] + for (let depth = 1; level.length > 0; depth++) { + const next: typeof level = [] + for (const parent of level) { + for (const model of models) { + const key = model.unique_id ?? model.name + if (visited.has(key)) continue + // An exact id decides alone: another package's model of the same name is not this one. + const reads = parent.id ? model.depends_on.includes(parent.id) : model.depends_on.some((d) => d.split(".").pop() === parent.name) + if (!reads) continue + visited.add(key) + const path = [...parent.path, model.name] results.push({ name: model.name, + ...(model.unique_id && { unique_id: model.unique_id }), depth, materialized: model.materialized, - path: newPath, + path, }) - walk(model.name, depth + 1, newPath) + next.push({ name: model.name, id: model.unique_id, path }) } } + level = next } - - walk(targetName, 1, [targetName]) return results } @@ -198,8 +223,15 @@ export function formatImpactReport(data: { direct: DownstreamModel[] transitive: DownstreamModel[] affectedTestCount: number - columnImpact: string[] totalModels: number + /** Exposures (dashboards) on the model or anything downstream of it. */ + exposures?: ColumnReachExposure[] + /** Why exposures could not be read, when reading them failed. */ + exposuresError?: string + /** Downstream models that read the column, when one was given and traced. */ + columnReach?: ColumnReach + /** Why the column could not be traced, when tracing failed. */ + columnReachError?: string }): string { const lines: string[] = [] @@ -248,13 +280,42 @@ export function formatImpactReport(data: { lines.push("") } - // Column impact - if (data.column && data.columnImpact.length > 0) { - lines.push(`Affected Output Columns (${data.columnImpact.length})`) + // Column reach: the downstream models whose SQL reads the column + if (data.column && data.columnReach) { + const reach = data.columnReach + const dashboards = reach.exposures.length + lines.push( + `Column Reach: ${data.model}.${data.column} is read by ${reach.models.length} model${reach.models.length === 1 ? "" : "s"} and ${dashboards} exposure${dashboards === 1 ? "" : "s"}` + + `${reach.uncertain.length ? `; ${reach.uncertain.length} more may read it` : ""}` + + `${reach.untraced.length ? `; ${reach.untraced.length} could not be traced` : ""}${reach.truncated ? " (stopped early)" : ""}`, + ) lines.push("".padEnd(40, "-")) - for (const col of data.columnImpact) { - lines.push(` ${col}`) - } + for (const m of reach.models) lines.push(` ${m.name}${m.columns.length ? ` → ${m.columns.join(", ")}` : ""}`) + for (const e of reach.exposures) lines.push(` ◆ ${e.label} [${e.type}]`) + for (const m of reach.uncertain) lines.push(` ? ${m.name}: ${m.reason}`) + for (const e of reach.uncertain_exposures) lines.push(` ? ◆ ${e.label} [${e.type}] on a model that may read it`) + for (const m of reach.untraced) lines.push(` ! ${m.name} not traced: ${m.reason}`) + lines.push("") + } + + if (data.column && data.columnReachError) { + lines.push(`Column Reach: not traced (${data.columnReachError}); the model-level impact above stands`) + lines.push("") + } + + // Exposures on anything affected, less those the column reach block already listed + const listed = new Set( + [...(data.columnReach?.exposures ?? []), ...(data.columnReach?.uncertain_exposures ?? [])].map((e) => e.unique_id), + ) + const others = (data.exposures ?? []).filter((e) => !listed.has(e.unique_id)) + if (others.length > 0) { + lines.push(`${data.columnReach ? "Other exposures" : "Exposures"} (${others.length})`) + lines.push("".padEnd(40, "-")) + for (const e of others) lines.push(` ${e.label} [${e.type}]`) + lines.push("") + } + if (data.exposuresError) { + lines.push(`Exposures: not read (${data.exposuresError.replace(/\r?\n/g, " ")})`) lines.push("") } diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts new file mode 100644 index 0000000000..ed3ebd3762 --- /dev/null +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -0,0 +1,413 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { columnReach, exposuresOn, parentsNamed, selectsStar } from "../../src/altimate/native/dbt/column-reach" +import { lineageGraph } from "../../src/altimate/native/dbt/graph" +import { structuredError, structuredResult } from "../../src/altimate/mcp-server/server" +import { dbtLineage } from "../../src/altimate/native/dbt/lineage" + +const dirs: string[] = [] +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }) +}) + +const column = (name: string) => ({ name, data_type: "double" }) +const model = (name: string, deps: string[], sql: string, columns: string[], extra: Record = {}) => ({ + resource_type: "model", + unique_id: `model.shop.${name}`, + name, + alias: name, + schema: "main", + database: "shop", + original_file_path: `models/${name}.sql`, + depends_on: { nodes: deps }, + compiled_code: sql, + columns: Object.fromEntries(columns.map((c) => [c, column(c)])), + ...extra, +}) + +// fct_orders.revenue is carried by fct_daily, renamed to ltv by mart_ltv (and read through it by +// rpt_ltv), filtered on by rpt_big_orders, and never read by rpt_counts. +function manifest() { + const nodes: Record = { + "model.shop.fct_orders": model("fct_orders", [], "select order_id, revenue, gross_revenue from raw.orders", ["order_id", "revenue", "gross_revenue"]), + "model.shop.fct_daily": model("fct_daily", ["model.shop.fct_orders"], "select sum(revenue) as revenue from fct_orders", ["revenue"]), + "model.shop.mart_ltv": model("mart_ltv", ["model.shop.fct_orders"], "select order_id, revenue as ltv from fct_orders", ["order_id", "ltv"]), + "model.shop.rpt_ltv": model("rpt_ltv", ["model.shop.mart_ltv"], "select max(ltv) as top_ltv from mart_ltv", ["top_ltv"]), + "model.shop.rpt_big_orders": model("rpt_big_orders", ["model.shop.fct_orders"], "select order_id from fct_orders where revenue > 100", ["order_id"]), + "model.shop.rpt_counts": model("rpt_counts", ["model.shop.fct_orders"], "select count(order_id) as orders from fct_orders", ["orders"]), + } + return { + metadata: { adapter_type: "duckdb" }, + nodes, + sources: {}, + exposures: { + "exposure.shop.exec_revenue": { name: "exec_revenue", label: "Exec Revenue", type: "dashboard", depends_on: { nodes: ["model.shop.fct_daily"] } }, + "exposure.shop.ops": { name: "ops", type: "application", depends_on: { nodes: ["model.shop.rpt_counts"] } }, + }, + } +} + +function manifestFile(value: Record = manifest()) { + const dir = mkdtempSync(join(tmpdir(), "column-reach-")) + dirs.push(dir) + const file = join(dir, "manifest.json") + writeFileSync(file, JSON.stringify(value)) + return file +} + +describe("columnReach", () => { + test("follows a column through renames to the models and dashboards that read it", () => { + const reach = columnReach(manifestFile(), "fct_orders", "REVENUE") + const byName = Object.fromEntries(reach.models.map((m) => [m.name, m.columns])) + expect(byName).toEqual({ fct_daily: ["revenue"], mart_ltv: ["ltv"], rpt_ltv: ["top_ltv"] }) + // The engine reports no edge for a filter: the model is a possible reader, not a certain one. + expect(reach.uncertain).toEqual([ + { name: "rpt_big_orders", unique_id: "model.shop.rpt_big_orders", reason: "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)" }, + ]) + expect(reach.untraced).toEqual([]) + expect(reach.exposures).toEqual([{ unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }]) + expect(reach.truncated).toBe(false) + }) + + test("an output alias or a comment naming the column is not a read", () => { + const value = manifest() + value.nodes["model.shop.rpt_counts"].compiled_code = "select count(order_id) as revenue /* revenue */ from fct_orders -- revenue" + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"]) + }) + + test("a downstream model without compiled SQL is reported untraced, and what reads it only may read the column", () => { + const value = manifest() + value.nodes["model.shop.mart_ltv"].compiled_code = "" + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name).sort()).toEqual(["fct_daily"]) + expect(reach.untraced).toEqual([ + { name: "mart_ltv", unique_id: "model.shop.mart_ltv", reason: "No compiled SQL found — run `dbt compile` first" }, + ]) + expect(reach.uncertain.map((m) => [m.name, m.reason])).toContainEqual(["rpt_ltv", "is downstream of a model that could not be traced or reads it ambiguously"]) + }) + + test("a model that only filters on the column passes nothing on to its children", () => { + const value = manifest() + value.nodes["model.shop.rpt_after_filter"] = model("rpt_after_filter", ["model.shop.rpt_big_orders"], "select count(order_id) as n from rpt_big_orders", ["n"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"]) + }) + + test("a join key another parent also documents is a possible read, not a dropped one", () => { + const value = manifest() + value.nodes["model.shop.dim_orders"] = model("dim_orders", [], "select order_id, revenue from raw.dim", ["order_id", "revenue"]) + value.nodes["model.shop.rpt_joined"] = model( + "rpt_joined", + ["model.shop.fct_orders", "model.shop.dim_orders"], + "select d.order_id from fct_orders f join dim_orders d on f.order_id = d.order_id where f.revenue > 0", + ["order_id"], + ) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.find((m) => m.name === "rpt_joined")?.reason).toBe( + "names it outside its select list, or names another parent's column of the same name", + ) + }) + + test("a source and a model with the same table name: a qualified source column is not the model's", () => { + const value: any = manifest() + value.sources = { + "source.shop.raw.fct_orders": { resource_type: "source", name: "fct_orders", identifier: "fct_orders", schema: "raw", database: "shop", columns: { revenue: column("revenue") } }, + } + value.nodes["model.shop.rpt_src"] = model( + "rpt_src", + ["model.shop.fct_orders", "source.shop.raw.fct_orders"], + "select s.revenue as src_revenue from raw.fct_orders s join main.fct_orders m on s.revenue = m.order_id", + ["src_revenue"], + ) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_src") + }) + + test("a table name two parents share is ambiguous; a schema-qualified one is not", () => { + const base = manifest() + const other = model("orders_copy", [], "select order_id, revenue from raw.copy", ["order_id", "revenue"], { alias: "fct_orders", schema: "archive" }) + const sql = (from: string) => `select b.revenue as kept from ${from}` + const child = (from: string) => + model("rpt_join", ["model.shop.fct_orders", "model.shop.orders_copy"], sql(from), ["kept"]) + const bare = { ...base, nodes: { ...base.nodes, "model.shop.orders_copy": other, "model.shop.rpt_join": child("fct_orders a join fct_orders b on a.order_id = b.order_id") } } + const ambiguous = columnReach(manifestFile(bare), "fct_orders", "revenue") + expect(ambiguous.models.map((m) => m.name)).not.toContain("rpt_join") + expect(ambiguous.uncertain.find((m) => m.name === "rpt_join")?.reason).toBe("reads a parent whose table name another parent shares") + + // Only archive.fct_orders (not traced) is read: no reader. + const qualified = { ...base, nodes: { ...base.nodes, "model.shop.orders_copy": other, "model.shop.rpt_join": child("main.fct_orders a join archive.fct_orders b on a.order_id = b.order_id") } } + const reach = columnReach(manifestFile(qualified), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_join") + }) + + test("a snapshot is traced and carries the column on", () => { + const value = manifest() + value.nodes["snapshot.shop.snap_orders"] = { ...model("snap_orders", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", ["order_id", "revenue"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap_orders" } + value.nodes["model.shop.rpt_snap"] = model("rpt_snap", ["snapshot.shop.snap_orders"], "select sum(revenue) as snap_revenue from snap_orders", ["snap_revenue"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + const byName = Object.fromEntries(reach.models.map((m) => [m.name, m.columns])) + expect(byName.snap_orders).toEqual(["revenue"]) + expect(byName.rpt_snap).toEqual(["snap_revenue"]) + }) + + test("a column no downstream model reads reaches nothing", () => { + const reach = columnReach(manifestFile(), "fct_orders", "gross_revenue") + expect(reach.models).toEqual([]) + expect(reach.exposures).toEqual([]) + }) + + test("an unknown model or a missing manifest throws", () => { + expect(() => columnReach(manifestFile(), "nope", "revenue")).toThrow(/not found/) + expect(() => columnReach(join(tmpdir(), "no-such-manifest.json"), "fct_orders", "revenue")).toThrow(/Manifest not found/) + }) +}) + +describe("exposuresOn", () => { + test("lists exposures on any of the given nodes, by label when there is one", () => { + const file = manifestFile() + expect(exposuresOn(file, ["model.shop.rpt_counts"])).toEqual([{ unique_id: "exposure.shop.ops", name: "ops", label: "ops", type: "application" }]) + expect(exposuresOn(file, ["model.shop.fct_orders"])).toEqual([]) + }) +}) + +describe("parentsNamed", () => { + test("a qualified name matches the parent at that qualification only; a bare name matches every parent that answers to it", () => { + const nodes: Record = { + a: { alias: "orders", schema: "main", database: "shop" }, + b: { alias: "orders", schema: "archive", database: "shop" }, + } + const lookup = (id: string) => nodes[id] + expect(parentsNamed(["shop", "main", "orders"], ["a", "b"], lookup)).toEqual(["a"]) + expect(parentsNamed(["archive", "orders"], ["a", "b"], lookup)).toEqual(["b"]) + expect(parentsNamed(["orders"], ["a", "b"], lookup)).toEqual(["a", "b"]) + expect(parentsNamed(["raw", "orders"], ["a", "b"], lookup)).toEqual([]) + }) +}) + +describe("columnReach where the engine's schema falls short", () => { + test("select * from a parent whose schema.yml does not list the column is a possible read, and its children may read it", () => { + const value = manifest() + value.nodes["model.shop.fct_orders"].columns = { order_id: column("order_id") } + value.nodes["model.shop.rpt_star"] = model("rpt_star", ["model.shop.fct_orders"], 'select * from "shop"."main"."fct_orders"', []) + value.nodes["model.shop.rpt_after_star"] = model("rpt_after_star", ["model.shop.rpt_star"], "select sum(revenue) as total from rpt_star", ["total"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.find((m) => m.name === "rpt_star")?.reason).toBe("selects * from a parent whose schema.yml does not list the column") + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after_star") + }) + + test("a dashboard downstream of a wildcard reader in any dialect form is reported, not dropped", () => { + for (const sql of ["select distinct on (order_id) * from fct_orders", "select top 10 * from fct_orders", 'select "shop"."main"."fct_orders".* from "shop"."main"."fct_orders"']) { + const value: any = manifest() + value.nodes["model.shop.fct_orders"].columns = { order_id: column("order_id") } + value.nodes["model.shop.rpt_star"] = model("rpt_star", ["model.shop.fct_orders"], sql, []) + value.exposures["exposure.shop.star_board"] = { name: "star_board", label: "Star Board", type: "dashboard", depends_on: { nodes: ["model.shop.rpt_star"] } } + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_star") + expect(reach.uncertain_exposures.map((e) => e.label)).toContain("Star Board") + } + }) + + test("a column name with a dot, quoted, is matched whole", () => { + const value = manifest() + value.nodes["model.shop.fct_orders"].columns = { "net.amount": column("net.amount") } + value.nodes["model.shop.rpt_net"] = model("rpt_net", ["model.shop.fct_orders"], 'select "net.amount" as net from fct_orders', ["net"]) + const reach = columnReach(manifestFile(value), "fct_orders", "net.amount") + expect(reach.models.find((m) => m.name === "rpt_net")?.columns).toEqual(["net"]) + }) +}) + +describe("selectsStar", () => { + test("a CR-terminated line comment does not hide the SQL after it", () => { + expect(selectsStar("select 1 -- note\rfrom t where x = 1 and y = (select *\rfrom u)")).toBe(true) + }) + + test("finds * and t.* in a select list, not multiplication or count(*)", () => { + for (const sql of [ + "select * from a", + "select*from a", + "select a.*, b.x from a", + "select x, t.* from t", + "select distinct * from a", + "select /* c */ * from a", + "select all * from a", + "select * exclude (order_id) from a", + "select * except (order_id) from a", + "select a . * from a", + "select distinct on (customer_id) * from a", + "select analytics.dbt_x.orders.* from analytics.dbt_x.orders", + '"db"."sch"."orders".* ', + "select top 10 * from a", + "select as struct * from a", + 'select "my col".* from t', + "select * replace (1 as x) from a", + ]) + expect(selectsStar(sql)).toBe(true) + for (const sql of [ + "select revenue * 100 as cents from a", + "select count(*) as n from a", + "select a, b*c as d from t", + "select '*' as s from a", + "select (a + b) * c as d from t", + "select a * (b + c) as d from t", + "select x from a -- select *", + ]) + expect(selectsStar(sql)).toBe(false) + // A function over the whole row reads every column; only count(*) reads none. + for (const sql of ["select id, hash(*) as h from a", "select object_construct(*) as o from a", "select to_json(struct(*)) as j from a", "select columns(*) from a"]) + expect(selectsStar(sql)).toBe(true) + }) +}) + +describe("columnReach through an ephemeral parent", () => { + test("a model reading an ephemeral parent is a possible read, never a silent zero", () => { + const value = manifest() + value.nodes["model.shop.eph"] = { ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", ["order_id", "revenue"]), config: { materialized: "ephemeral" } } + value.nodes["model.shop.rpt_eph"] = model( + "rpt_eph", + ["model.shop.eph"], + "with __dbt__cte__eph as (select order_id, revenue from fct_orders) select sum(revenue) as total from __dbt__cte__eph", + ["total"], + ) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).toContain("eph") + expect(reach.uncertain.find((m) => m.name === "rpt_eph")?.reason).toBe( + "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through", + ) + }) +}) + +describe("columnReach with an ephemeral and a regular parent", () => { + test("a traced edge from the regular parent does not hide what flows through the ephemeral one", () => { + const value = manifest() + value.nodes["model.shop.eph"] = { ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue as renamed from fct_orders", ["order_id", "renamed"]), config: { materialized: "ephemeral" } } + value.nodes["model.shop.mix"] = model( + "mix", + ["model.shop.fct_daily", "model.shop.eph"], + "with __dbt__cte__eph as (select order_id, revenue as renamed from fct_orders) select d.revenue as direct, e.renamed as hidden from fct_daily d cross join __dbt__cte__eph e", + ["direct", "hidden"], + ) + value.nodes["model.shop.leaf"] = model("leaf", ["model.shop.mix"], "select max(hidden) as h from mix", ["h"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).toContain("mix") + expect(reach.uncertain.map((m) => m.name)).toContain("leaf") + }) +}) + +describe("columnReach with columns that differ only by case", () => { + test('"Revenue" is not revenue where the parent documents both', () => { + const value: any = manifest() + value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } + value.nodes["model.shop.rpt_case"] = model("rpt_case", ["model.shop.fct_orders"], 'select "Revenue" as other from fct_orders', ["other"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_case") + expect(reach.models.map((m) => m.name)).toContain("fct_daily") + }) +}) + +describe("columnReach on SQL the engine cannot parse", () => { + test("is untraced, and a dashboard on it is reported apart", () => { + const value = manifest() + value.nodes["model.shop.fct_daily"].compiled_code = "select sum(revenue) as revenue frm fct_orders whre" + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.untraced.map((m) => m.name)).toEqual(["fct_daily"]) + expect(reach.untraced[0]!.reason).toContain("Parse error") + expect(reach.exposures).toEqual([]) + expect(reach.uncertain_exposures).toEqual([{ unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }]) + }) +}) + +describe("lineageGraph", () => { + test("two levels up, everything downstream, tests as ids, from dbt's child_map when present", () => { + const value = { + ...manifest(), + nodes: { ...manifest().nodes, "test.shop.t1": { resource_type: "test", name: "t1", depends_on: { nodes: ["model.shop.fct_daily"] } } }, + } + const graph = lineageGraph(value, "model.shop.mart_ltv") + expect(graph.parents["model.shop.mart_ltv"]).toEqual(["model.shop.fct_orders"]) + expect(graph.children["model.shop.mart_ltv"]).toEqual(["model.shop.rpt_ltv"]) + expect(graph.nodes["model.shop.mart_ltv"]).toEqual({ + id: "model.shop.mart_ltv", + name: "mart_ltv", + label: null, + type: "model", + path: "models/mart_ltv.sql", + materialized: null, + relation: "shop.main.mart_ltv", + columns: ["order_id", "ltv"], + }) + expect(graph.truncated).toBe(false) + const fromFct = lineageGraph({ ...value, child_map: { "model.shop.fct_orders": ["model.shop.fct_daily", "test.shop.t1"], "model.shop.fct_daily": [] }, parent_map: { "model.shop.fct_orders": [], "test.shop.t1": ["model.shop.fct_orders"] } }, "model.shop.fct_orders") + expect(Object.keys(fromFct.nodes).sort()).toEqual(["model.shop.fct_daily", "model.shop.fct_orders", "test.shop.t1"]) + expect(fromFct.nodes["test.shop.t1"]!.type).toBe("test") + expect(fromFct.parents["test.shop.t1"]).toEqual(["model.shop.fct_orders"]) + expect(fromFct.children["test.shop.t1"]).toEqual([]) + }) + + test("a node whose manifest relation_name is null has no relation", () => { + const base = manifest() + const value = { + ...base, + nodes: { ...base.nodes, "model.shop.mart_ltv": { ...base.nodes["model.shop.mart_ltv"], relation_name: null } }, + } + expect(lineageGraph(value, "model.shop.mart_ltv").nodes["model.shop.mart_ltv"]!.relation).toBeNull() + }) +}) + +describe("structuredError", () => { + test("passes a short message and caps a huge one", () => { + expect(structuredError("boom")).toEqual({ report: "boom" }) + const out = structuredError("x".repeat(300_000)) + expect(out.truncated).toBe(true) + expect(Buffer.byteLength(out.report as string)).toBeLessThanOrEqual(100_000) + }) + test("exposures_error reaches the structured fields", () => { + expect(structuredResult("impact_analysis", { exposures_error: "bad json" }, "R")).toEqual({ exposures_error: "bad json", report: "R" }) + }) +}) + +describe("structuredError escaping", () => { + test("the cap holds for the serialized result, whatever JSON escaping adds", () => { + for (const text of ["\n".repeat(100_000), "\u0000".repeat(300_000), "é".repeat(80_000)]) { + const out = structuredError(text) + expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(100_000) + expect(out.truncated).toBe(true) + } + }) +}) + + +describe("structuredResult", () => { + test("passes the lineage fields with the report, and only the report when they are too big", () => { + expect(structuredResult("impact_analysis", { severity: "LOW", findings: [{ category: "x" }], downstream: [] }, "Impact")).toEqual({ + severity: "LOW", + downstream: [], + report: "Impact", + }) + const huge = { edges: Array.from({ length: 5000 }, (_, i) => ({ source: `"t"."c${i}"`, target: `c${i}` })) } + expect(structuredResult("dbt_lineage", huge, "Model: x")).toEqual({ report: "Model: x", truncated: true }) + // dbt_lineage's text repeats the columns and edges: only its header goes with them. + expect(structuredResult("dbt_lineage", { columns: {}, edges: [], graph: { nodes: {} } }, "Model: x\nID: y\n\nColumn Mappings:\n a ← [\"t.a\"]")).toEqual({ + columns: {}, + edges: [], + report: "Model: x\nID: y", + }) + }) +}) + +describe("dbtLineage", () => { + test("reports the model's upstream unique_ids, and the graph when asked", () => { + const file = manifestFile() + const result = dbtLineage({ manifest_path: file, model: "mart_ltv" }) + expect(result.upstream).toEqual(["model.shop.fct_orders"]) + expect(result.graph).toBeUndefined() + expect(Object.keys(dbtLineage({ manifest_path: file, model: "mart_ltv", include_graph: true }).graph!.nodes)).toContain("model.shop.rpt_ltv") + // By project-relative path too. + expect(dbtLineage({ manifest_path: file, model: "models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + const withSnapshot: any = manifest() + withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" } + expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap") + }) +}) diff --git a/packages/opencode/test/altimate/dbt-lineage-helpers.test.ts b/packages/opencode/test/altimate/dbt-lineage-helpers.test.ts index 31bfcd5ec0..6fc28fbb6b 100644 --- a/packages/opencode/test/altimate/dbt-lineage-helpers.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-helpers.test.ts @@ -316,6 +316,24 @@ describe("dbtLineage: error handling", () => { expect(result.confidence_factors).toContain("No compiled SQL found — run `dbt compile` first") }) + test("reports the model's name, not the path used to find it, when there is no compiled SQL", () => { + const dir = makeTmpDir() + const manifest = { + nodes: { + "model.proj.uncompiled": { + resource_type: "model", + name: "uncompiled", + original_file_path: "models/uncompiled.sql", + depends_on: { nodes: [] }, + columns: {}, + }, + }, + sources: {}, + } + const result = dbtLineage({ manifest_path: writeManifest(dir, manifest), model: "models/uncompiled.sql" }) + expect(result.model_name).toBe("uncompiled") + }) + test("handles manifest with no nodes key at all", () => { const dir = makeTmpDir() const manifestPath = writeManifest(dir, { metadata: {} }) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index c7e566c0e2..8879d67a37 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -167,7 +167,8 @@ describe("dbt_lineage with the project's dbt adapter", () => { arguments: { model: "customers", column: "customer_lifetime_value" }, }) expect(answer.isError).toBe(false) - expect(answer.structuredContent).toEqual(TRACE) + // The MCP server adds the text report beside the trace (clients may show structured content in place of the text). + expect(answer.structuredContent).toEqual({ ...TRACE, report: expect.any(String) }) expect((answer.content as { text: string }[])[0].text).toContain("Trace of customers.customer_lifetime_value") } finally { await client.close() diff --git a/packages/opencode/test/altimate/mcp-server/server.test.ts b/packages/opencode/test/altimate/mcp-server/server.test.ts index f0060c7045..ed4dde9b83 100644 --- a/packages/opencode/test/altimate/mcp-server/server.test.ts +++ b/packages/opencode/test/altimate/mcp-server/server.test.ts @@ -1657,3 +1657,57 @@ mcpTest("startup skips a local catalog over the table gate with a short note", a expect(executed[0]!.sql).toContain("LIMIT 41") }) // altimate_change end + +// altimate_change start — lineage tools return structured content beside the text +mcpTest("dbt_lineage and impact_analysis declare an output schema and return their metadata as structured content", async () => { + const { client, call } = await fixture() + const tools = (await client.listTools()).tools + for (const name of ["dbt_lineage", "impact_analysis"]) expect(tools.find((tool) => tool.name === name)?.outputSchema).toBeDefined() + expect(tools.find((tool) => tool.name === "sql_execute")?.outputSchema).toBeUndefined() + + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-lineage-test-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const columns = (...names: string[]) => Object.fromEntries(names.map((name) => [name, { name, data_type: "double" }])) + const node = (name: string, deps: string[], sql: string, cols: string[]) => ({ + resource_type: "model", unique_id: `model.shop.${name}`, name, alias: name, original_file_path: `models/${name}.sql`, + depends_on: { nodes: deps }, compiled_code: sql, columns: columns(...cols), + }) + const manifestPath = path.join(dir, "manifest.json") + await Bun.write(manifestPath, JSON.stringify({ + metadata: { adapter_type: "duckdb" }, + sources: {}, + nodes: { + "model.shop.fct_orders": node("fct_orders", [], "select order_id, revenue from raw.orders", ["order_id", "revenue"]), + "model.shop.fct_daily": node("fct_daily", ["model.shop.fct_orders"], "select sum(revenue) as revenue from fct_orders", ["revenue"]), + }, + exposures: { "exposure.shop.exec": { name: "exec", label: "Exec Revenue", type: "dashboard", depends_on: { nodes: ["model.shop.fct_daily"] } } }, + })) + + const lineage = await call("dbt_lineage", { manifest_path: manifestPath, model: "fct_daily" }) + const structured = lineage.structuredContent as Record + // The structured report keeps the text's header; the mappings and edges travel as data. + const text = (lineage.content as { text: string }[])[0]!.text + expect(structured.report).toBe(text.split("\nColumn Mappings:")[0]!.trim()) + expect(text).toContain("Column Mappings:") + expect(structured.upstream).toEqual(["model.shop.fct_orders"]) + expect(structured.columns.revenue[0]).toContain("revenue") + expect(structured.edges[0].target).toBe("revenue") + + const impact = await call("impact_analysis", { manifest_path: manifestPath, model: "fct_orders", column: "revenue", change_type: "modify", dialect: "duckdb" }) + const reach = impact.structuredContent as Record + expect(reach.downstream.map((d: { name: string }) => d.name)).toEqual(["fct_daily"]) + expect(reach.column_reach.models.map((m: { name: string }) => m.name)).toEqual(["fct_daily"]) + expect(reach.exposures).toEqual([{ unique_id: "exposure.shop.exec", name: "exec", label: "Exec Revenue", type: "dashboard" }]) +}) +// altimate_change end + +// altimate_change start — a failing lineage call still answers with structured content its schema accepts +mcpTest("dbt_lineage and impact_analysis errors still return a structured report", async () => { + const { call } = await fixture() + for (const name of ["dbt_lineage", "impact_analysis"]) { + const result = await call(name, {}) + expect(result.isError).toBe(true) + expect(typeof (result.structuredContent as { report?: unknown } | undefined)?.report).toBe("string") + } +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/tools/impact-analysis.test.ts b/packages/opencode/test/altimate/tools/impact-analysis.test.ts index 85c71a29df..7ecf843b00 100644 --- a/packages/opencode/test/altimate/tools/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/tools/impact-analysis.test.ts @@ -2,6 +2,40 @@ import { describe, test, expect } from "bun:test" import { findDownstream, formatImpactReport } from "../../../src/altimate/tools/impact-analysis" import type { DownstreamModel } from "../../../src/altimate/tools/impact-analysis" +describe("findDownstream: depth", () => { + test("a direct dependent also reachable through another dependent stays direct", () => { + const models = [ + { name: "y", depends_on: ["model.p.t"] }, + { name: "x", depends_on: ["model.p.y", "model.p.t"] }, + ] + const result = findDownstream("t", models) + expect(result.find((d) => d.name === "x")!.depth).toBe(1) + expect(result.find((d) => d.name === "x")!.path).toEqual(["t", "x"]) + }) +}) + +describe("formatImpactReport: exposures", () => { + const base = { model: "m", changeType: "remove", direct: [], transitive: [], affectedTestCount: 0, totalModels: 1 } + test("says when exposures could not be read", () => { + expect(formatImpactReport({ ...base, exposuresError: "bad json" })).toContain("Exposures: not read (bad json)") + }) + test("keeps a multi-line read error on one line", () => { + expect(formatImpactReport({ ...base, exposuresError: "a\nb" })).toContain("Exposures: not read (a b)") + }) + test("lists a dashboard once when column reach already printed it", () => { + const dash = { unique_id: "exposure.p.d", name: "d", label: "Dash", type: "dashboard" } + const out = formatImpactReport({ + ...base, + column: "c", + exposures: [dash, { unique_id: "exposure.q.d", name: "d", label: "Other", type: "dashboard" }], + columnReach: { models: [], uncertain: [], untraced: [], exposures: [dash], uncertain_exposures: [], truncated: false }, + }) + expect(out.match(/Dash \[dashboard\]/g)).toHaveLength(1) + expect(out).toContain("Other exposures (1)") + expect(out).toContain("Other [dashboard]") + }) +}) + describe("findDownstream: DAG traversal", () => { test("returns empty for leaf model with no dependents", () => { const models = [ @@ -114,7 +148,6 @@ describe("formatImpactReport", () => { direct: [], transitive: [], affectedTestCount: 0, - columnImpact: [], totalModels: 10, }) expect(report).toContain("REMOVE stg_temp") @@ -130,7 +163,6 @@ describe("formatImpactReport", () => { direct: [{ name: "fct_orders", depth: 1, path: ["stg_orders", "fct_orders"] }], transitive: [], affectedTestCount: 0, - columnImpact: [], totalModels: 20, }) expect(report).toContain("WARNING: This is a BREAKING change") @@ -146,7 +178,6 @@ describe("formatImpactReport", () => { direct: [{ name: "fct_orders", depth: 1, path: ["stg_orders", "fct_orders"] }], transitive: [], affectedTestCount: 5, - columnImpact: [], totalModels: 10, }) expect(report).toContain("WARNING: Rename requires updating all downstream references.") @@ -154,7 +185,7 @@ describe("formatImpactReport", () => { expect(report).toContain("Affected tests: 5") }) - test("column-level impact shows affected columns", () => { + test("column-level impact names the column and the change", () => { const report = formatImpactReport({ model: "stg_orders", column: "order_id", @@ -162,14 +193,40 @@ describe("formatImpactReport", () => { direct: [{ name: "fct_orders", depth: 1, path: ["stg_orders", "fct_orders"] }], transitive: [], affectedTestCount: 0, - columnImpact: ["total_amount", "order_count"], totalModels: 10, }) expect(report).toContain("RETYPE stg_orders.order_id") expect(report).toContain("CAUTION: Type change may cause implicit casts") - expect(report).toContain("Affected Output Columns (2)") - expect(report).toContain("total_amount") - expect(report).toContain("order_count") + }) + + test("column reach lists certain readers, possible readers, untraced models and exposures", () => { + const report = formatImpactReport({ + model: "fct_orders", + column: "revenue", + changeType: "modify", + direct: [{ name: "fct_daily", depth: 1, path: ["fct_orders", "fct_daily"] }], + transitive: [], + affectedTestCount: 0, + totalModels: 5, + exposures: [{ unique_id: "exposure.p.exec", name: "exec", label: "Exec Revenue", type: "dashboard" }], + columnReach: { + models: [{ name: "fct_daily", unique_id: "model.p.fct_daily", columns: ["revenue"] }], + uncertain: [{ name: "rpt_big", unique_id: "model.p.rpt_big", reason: "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)" }], + untraced: [{ name: "mart_ltv", unique_id: "model.p.mart_ltv", reason: "No compiled SQL found — run `dbt compile` first" }], + exposures: [{ unique_id: "exposure.p.exec", name: "exec", label: "Exec Revenue", type: "dashboard" }], + uncertain_exposures: [{ unique_id: "exposure.p.ops", name: "ops", label: "Ops", type: "dashboard" }], + truncated: true, + }, + }) + expect(report).toContain( + "Column Reach: fct_orders.revenue is read by 1 model and 1 exposure; 1 more may read it; 1 could not be traced (stopped early)", + ) + expect(report).toContain(" fct_daily → revenue") + expect(report).toContain(" ◆ Exec Revenue [dashboard]") + expect(report).toContain(" ? rpt_big: names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)") + expect(report).toContain(" ! mart_ltv not traced: No compiled SQL found — run `dbt compile` first") + expect(report).toContain(" ? ◆ Ops [dashboard] on a model that may read it") + expect(report).not.toContain("Exposures (") }) test("percentage calculation with 0 total models does not produce NaN or Infinity", () => { @@ -179,7 +236,6 @@ describe("formatImpactReport", () => { direct: [], transitive: [], affectedTestCount: 0, - columnImpact: [], totalModels: 0, }) expect(report).not.toContain("NaN") @@ -194,7 +250,6 @@ describe("formatImpactReport", () => { direct: [{ name: "fct_orders", depth: 1, path: ["stg_orders", "fct_orders"] }], transitive: [{ name: "report", depth: 2, materialized: "table", path: ["stg_orders", "fct_orders", "report"] }], affectedTestCount: 0, - columnImpact: [], totalModels: 50, }) expect(report).toContain("Direct Dependents (1)") @@ -203,3 +258,24 @@ describe("formatImpactReport", () => { expect(report).toContain("Blast radius: 2/50 models (4.0%)") }) }) + +describe("findDownstream with versioned models", () => { + test("a versioned parent is matched by unique_id, whose last segment is the version", () => { + const models = [ + { name: "orders", unique_id: "model.p.orders.v2", depends_on: [] }, + { name: "rpt", unique_id: "model.p.rpt", depends_on: ["model.p.orders.v2"] }, + ] + expect(findDownstream("orders", models, "model.p.orders.v2").map((d) => [d.name, d.unique_id])).toEqual([["rpt", "model.p.rpt"]]) + expect(findDownstream("orders", models)).toEqual([]) + }) + + test("with an exact id, another package's model of the same name is not matched", () => { + const models = [ + { name: "orders", unique_id: "model.a.orders", depends_on: [] }, + { name: "orders", unique_id: "model.b.orders", depends_on: [] }, + { name: "b_only", unique_id: "model.b.b_only", depends_on: ["model.b.orders"] }, + ] + expect(findDownstream("orders", models, "model.a.orders")).toEqual([]) + expect(findDownstream("orders", models, "model.b.orders").map((d) => d.name)).toEqual(["b_only"]) + }) +}) From e434c5ba75b1bbb210808391537ce508e9d4ae0e Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 05:10:13 +0530 Subject: [PATCH 02/37] fix: dbt_lineage uses the adapter only for the named manifest's own project Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-lineage.ts | 36 ++++++++++++++++++- .../altimate/dbt-lineage-structured.test.ts | 32 ++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 05dc561d65..af1becd6b7 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -1,4 +1,5 @@ import z from "zod" +import fs from "fs" import path from "path" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" @@ -55,7 +56,13 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { .describe("Also return the model's graph from the manifest (two levels upstream, everything downstream up to 500 models with exposures and tests; graph.truncated is set when cut) in the metadata; for lineage viewers"), }), async execute(args, ctx) { - const viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) + let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) + // The adapter answers for the project it found; a manifest named outside that project's folder + // wins, so its graph and its lineage are never mixed with this one's. + if (viaAdapter.result && args.manifest_path) { + const outside = await manifestOutsideAdapterProject(args.manifest_path) + if (outside) viaAdapter = { reason: outside } + } if (viaAdapter.result) { const result = viaAdapter.result const what = result.focus.column ? `${result.focus.name}.${result.focus.column}` : result.focus.name @@ -115,6 +122,33 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { }, }) +/** + * Why a named manifest is not the adapter's project's, or undefined when it is: it must lie in that + * project's folder (links resolved) with no other dbt_project.yml between (a nested project or an + * installed package is another project). A path that cannot be resolved counts as not this one's. + */ +async function manifestOutsideAdapterProject(manifestPath: string): Promise { + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + if (!root) return "the dbt adapter has no project root" + const real = (p: string) => { + try { + return fs.realpathSync(path.resolve(p)) + } catch { + return undefined + } + } + const projectDir = real(root) + const manifest = real(manifestPath) + if (!projectDir || !manifest) return `the manifest ${manifestPath} cannot be read` + const inside = path.relative(projectDir, manifest) + if (!inside || inside.startsWith("..") || path.isAbsolute(inside)) return `the manifest ${manifestPath} is not in the dbt project at ${root}` + for (let dir = path.dirname(manifest); dir !== projectDir && dir.startsWith(projectDir); dir = path.dirname(dir)) { + if (fs.existsSync(path.join(dir, "dbt_project.yml"))) return `the manifest ${manifestPath} belongs to the dbt project at ${dir}` + } + return undefined +} + /** The model's graph from the compiled manifest, or undefined when there is no manifest to read. */ function manifestGraph(manifestPath: string, uniqueId: string) { try { diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index 8879d67a37..080fedf2da 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -5,7 +5,7 @@ // without a dbt.json. The dbt adapter is replaced through `dbtAdapterInternals`, so // these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test" -import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { Effect } from "effect" @@ -155,6 +155,36 @@ describe("dbt_lineage with the project's dbt adapter", () => { expect(at).toEqual([...at].sort((a, b) => a - b)) }) + test("a manifest named outside the adapter's project wins over the adapter", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "dbt-lineage-project-")) + const mine = path.join(dir, "mine") + const other = path.join(dir, "other") + await Promise.all([mine, other].map((d) => mkdir(path.join(d, "target"), { recursive: true }))) + const empty = JSON.stringify({ metadata: { adapter_type: "duckdb" }, nodes: {}, sources: {} }) + await writeFile(path.join(mine, "target", "manifest.json"), empty) + await writeFile(path.join(other, "target", "manifest.json"), empty) + const root = spyOn(register, "getDbtProjectRoot").mockResolvedValue(mine) + try { + const tool = await initTool(DbtLineageTool) + const fallback = await tool.execute({ model: "customers", manifest_path: path.join(other, "target", "manifest.json") }, ctx) + expect(fallback.metadata.structuredContent).toBeUndefined() + expect(fallback.output).toContain(`is not in the dbt project at ${mine}`) + const kept = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "manifest.json") }, ctx) + expect(kept.metadata.structuredContent).toEqual(TRACE) + // A dbt project nested in this one's folder is another project. + const nested = path.join(mine, "subproject") + await mkdir(path.join(nested, "target"), { recursive: true }) + await writeFile(path.join(nested, "dbt_project.yml"), "name: sub\n") + await writeFile(path.join(nested, "target", "manifest.json"), empty) + const inner = await tool.execute({ model: "customers", manifest_path: path.join(nested, "target", "manifest.json") }, ctx) + expect(inner.metadata.structuredContent).toBeUndefined() + expect(inner.output).toContain("belongs to the dbt project at") + } finally { + root.mockRestore() + await rm(dir, { recursive: true, force: true }) + } + }) + test("is handed to an MCP client as structuredContent", async () => { const server = await createServer() const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() From d29034fcee642b6adb8c8ebe165b49453702fe6d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 09:29:49 +0530 Subject: [PATCH 03/37] fix: an oversized structured result keeps its report within the cap and stays marked truncated Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/mcp-server/server.ts | 12 +++++++++--- packages/opencode/test/altimate/column-reach.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 521b0820d2..63b93f6a78 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -87,20 +87,26 @@ export function structuredResult(tool: string, metadata: Record const own = metadata.structuredContent if (own && typeof own === "object" && !Array.isArray(own)) { const full = { ...(own as Record), report } - return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : { report, truncated: true } + return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) } const fields = Object.fromEntries((STRUCTURED_FIELDS[tool] ?? []).filter((key) => key in metadata).map((key) => [key, metadata[key]])) // dbt_lineage's text prints the same mappings and edges as `columns` and `edges`: keep its header. const head = tool === "dbt_lineage" && "columns" in fields ? report.split("\nColumn Mappings:")[0]!.trim() : report const full = { ...fields, report: head } - return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : { report, truncated: true } + return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) } /** An error report held to the same cap as a result, so a huge exception message cannot exceed it. */ export function structuredError(message: string): Record { + return capReport(message, false) +} +// `dropped`: the result's other fields were left out, so it is marked truncated even when the +// report itself fits. +function capReport(message: string, dropped: boolean): Record { // Measured as sent: JSON escaping can grow the text (a newline is two bytes, a control // character six), so the cap applies to the serialized result, shortening until it fits. const size = (value: Record) => Buffer.byteLength(JSON.stringify(value)) - if (size({ report: message }) <= STRUCTURED_MAX_BYTES) return { report: message } + const whole = dropped ? { report: message, truncated: true } : { report: message } + if (size(whole) <= STRUCTURED_MAX_BYTES) return whole let report = message while (report.length > 0 && size({ report, truncated: true }) > STRUCTURED_MAX_BYTES) { report = report.slice(0, Math.floor(report.length * 0.9)) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index ed3ebd3762..3b821857c9 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -368,6 +368,14 @@ describe("structuredError", () => { }) }) +describe("structuredResult over the cap", () => { + test("a report too big on its own is shortened too", () => { + const out = structuredResult("dbt_lineage", { structuredContent: { focus: {} } }, "x".repeat(300_000)) + expect(out.truncated).toBe(true) + expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(100_000) + }) +}) + describe("structuredError escaping", () => { test("the cap holds for the serialized result, whatever JSON escaping adds", () => { for (const text of ["\n".repeat(100_000), "\u0000".repeat(300_000), "é".repeat(80_000)]) { From 391fd7c1119019313a20e6e3779d3e393dc2c7b0 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 09:44:12 +0530 Subject: [PATCH 04/37] docs: the truncated flag says the report may be shortened too Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/mcp-server/server.ts | 3 +-- packages/opencode/test/altimate/column-reach.test.ts | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 63b93f6a78..c5f9ea155b 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -72,7 +72,7 @@ const STRUCTURED_OUTPUT_SCHEMA = { type: "object" as const, properties: { report: { type: "string", description: "The tool's text report" }, - truncated: { type: "boolean", description: "True when the lineage fields were left out for size; the report stands" }, + truncated: { type: "boolean", description: "True when the lineage fields were left out for size; the report is kept, shortened if it is too large on its own" }, }, required: ["report"], additionalProperties: true, @@ -219,7 +219,6 @@ export async function createServer(options: ServeOptions = {}) { const clarified = request.params.name === "answer_commit" ? await clarifyReading(request.params.arguments, result, extra.signal) : undefined if (clarified) return { content: [{ type: "text" as const, text: clarified }], isError: true } const isError = resultFailed(result) - // A tool that builds a structured result (dbt_lineage) hands it over beside its text. const structured = result.metadata?.structuredContent return { content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 3b821857c9..d3385a815c 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -373,6 +373,7 @@ describe("structuredResult over the cap", () => { const out = structuredResult("dbt_lineage", { structuredContent: { focus: {} } }, "x".repeat(300_000)) expect(out.truncated).toBe(true) expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(100_000) + expect("x".repeat(300_000).startsWith(out.report as string)).toBe(true) }) }) From 852124201fcfe2c165a53bdf4a5d76520d97603c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 10:19:07 +0530 Subject: [PATCH 05/37] feat: dbt_lineage include_graph returns the model's whole upstream, capped at 500 nodes and marked when cut Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/native/dbt/graph.ts | 44 +++++++++---------- .../opencode/src/altimate/native/types.ts | 2 +- .../src/altimate/tools/dbt-lineage.ts | 2 +- .../test/altimate/column-reach.test.ts | 17 ++++++- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts index 698bf146ee..c708bf70e4 100644 --- a/packages/opencode/src/altimate/native/dbt/graph.ts +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -1,6 +1,6 @@ // altimate_change - new file // -// The part of a dbt manifest's graph a lineage viewer draws for one model: two levels upstream, +// The part of a dbt manifest's graph a lineage viewer draws for one model: everything upstream, // everything downstream (with the tests counted as ids), and the links between them. Clients that // cannot read a large manifest themselves (the Claude Code plugin reads files up to 4 MiB) get it // from the CLI (`altimate tool run dbt_lineage --json` with include_graph); MCP carries no graph. @@ -24,11 +24,14 @@ export interface LineageGraph { children: Record /** True when downstream stopped at the node limit. */ truncated: boolean + /** True when upstream stopped at the node limit. */ + upTruncated: boolean } const GROUPS = ["nodes", "sources", "exposures", "metrics", "semantic_models", "saved_queries", "unit_tests"] as const -/** Limit on downstream models only; the target, its two upstream levels and test nodes come on top. */ +/** Limits on each side; the target and test nodes come on top. A source-to-mart chain of a few hundred models fits. */ const MAX_DOWNSTREAM = 500 +const MAX_UPSTREAM = 500 function lookup(manifest: any, id: string): any { for (const group of GROUPS) if (manifest[group]?.[id]) return manifest[group][id] @@ -76,29 +79,26 @@ export function lineageGraph(manifest: any, id: string): LineageGraph { const { parents, children } = mapsOf(manifest) const isTest = (one: string) => ["test", "unit_test"].includes(lookup(manifest, one)?.resource_type ?? one.split(".")[0]) const keep = new Set([id]) - for (const parent of parents[id] ?? []) { - keep.add(parent) - for (const grand of parents[parent] ?? []) keep.add(grand) - } - let truncated = false - const queue = [id] - const seen = new Set([id]) - let count = 0 - while (queue.length) { - for (const child of children[queue.shift()!] ?? []) { - if (seen.has(child) || isTest(child)) continue - if (count >= MAX_DOWNSTREAM) { - truncated = true - queue.length = 0 - break + // Breadth first, so a cut keeps the nearest nodes. + const walk = (links: Record, limit: number) => { + const queue = [id] + const seen = new Set([id]) + let count = 0 + while (queue.length) { + for (const next of links[queue.shift()!] ?? []) { + if (seen.has(next) || isTest(next)) continue + if (count >= limit) return true + seen.add(next) + keep.add(next) + count++ + queue.push(next) } - seen.add(child) - keep.add(child) - count++ - queue.push(child) } + return false } - const graph: LineageGraph = { nodes: {}, parents: {}, children: {}, truncated } + const upTruncated = walk(parents, MAX_UPSTREAM) + const truncated = walk(children, MAX_DOWNSTREAM) + const graph: LineageGraph = { nodes: {}, parents: {}, children: {}, truncated, upTruncated } for (const one of keep) { graph.nodes[one] = nodeOf(one, lookup(manifest, one)) graph.parents[one] = (parents[one] ?? []).filter((other) => keep.has(other)) diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 91af63d60a..5de10c591f 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -1076,7 +1076,7 @@ export interface DbtLineageParams { manifest_path: string model: string dialect?: string - /** Also return the model's lineage graph (two levels up, everything downstream). */ + /** Also return the model's lineage graph (everything upstream and downstream, up to 500 nodes each side). */ include_graph?: boolean } diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index af1becd6b7..5bc9cc83b5 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -53,7 +53,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { include_graph: z .boolean() .optional() - .describe("Also return the model's graph from the manifest (two levels upstream, everything downstream up to 500 models with exposures and tests; graph.truncated is set when cut) in the metadata; for lineage viewers"), + .describe("Also return the model's graph from the manifest (everything upstream and downstream, up to 500 nodes each side, with exposures and tests; graph.upTruncated / graph.truncated are set when a side is cut) in the metadata; for lineage viewers"), }), async execute(args, ctx) { let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index d3385a815c..c9bed34903 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -320,7 +320,22 @@ describe("columnReach on SQL the engine cannot parse", () => { }) describe("lineageGraph", () => { - test("two levels up, everything downstream, tests as ids, from dbt's child_map when present", () => { + test("a twelve-hop chain comes back whole to its source; upstream past 500 nodes is cut and marked", () => { + const chain: Record = {} + for (let i = 1; i <= 12; i++) chain[`model.shop.m${i}`] = model(`m${i}`, [i === 1 ? "source.shop.raw.orders" : `model.shop.m${i - 1}`], "select 1", []) + const graph = lineageGraph({ nodes: chain, sources: { "source.shop.raw.orders": { resource_type: "source", name: "orders", source_name: "raw" } } }, "model.shop.m12") + expect(graph.nodes["source.shop.raw.orders"]).toBeDefined() + expect(Object.keys(graph.nodes)).toHaveLength(13) + expect(graph.upTruncated).toBe(false) + + const wide: Record = { "model.shop.top": model("top", Array.from({ length: 600 }, (_, i) => `model.shop.p${i}`), "select 1", []) } + for (let i = 0; i < 600; i++) wide[`model.shop.p${i}`] = model(`p${i}`, [], "select 1", []) + const cut = lineageGraph({ nodes: wide }, "model.shop.top") + expect(cut.upTruncated).toBe(true) + expect(Object.keys(cut.nodes)).toHaveLength(501) + }) + + test("everything up and downstream, tests as ids, from dbt's child_map when present", () => { const value = { ...manifest(), nodes: { ...manifest().nodes, "test.shop.t1": { resource_type: "test", name: "t1", depends_on: { nodes: ["model.shop.fct_daily"] } } }, From 9f92fa24e8e64cc4915dbf225cab2283b8f633f8 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 10:32:06 +0530 Subject: [PATCH 06/37] fix: include_graph leaves out the tests of upstream models Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/graph.ts | 9 +++++---- packages/opencode/test/altimate/column-reach.test.ts | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts index c708bf70e4..89ef133344 100644 --- a/packages/opencode/src/altimate/native/dbt/graph.ts +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -80,9 +80,8 @@ export function lineageGraph(manifest: any, id: string): LineageGraph { const isTest = (one: string) => ["test", "unit_test"].includes(lookup(manifest, one)?.resource_type ?? one.split(".")[0]) const keep = new Set([id]) // Breadth first, so a cut keeps the nearest nodes. - const walk = (links: Record, limit: number) => { + const walk = (links: Record, limit: number, seen = new Set([id])) => { const queue = [id] - const seen = new Set([id]) let count = 0 while (queue.length) { for (const next of links[queue.shift()!] ?? []) { @@ -97,14 +96,16 @@ export function lineageGraph(manifest: any, id: string): LineageGraph { return false } const upTruncated = walk(parents, MAX_UPSTREAM) - const truncated = walk(children, MAX_DOWNSTREAM) + // Tests are counted for the target and what is downstream of it only; upstream ones would only add weight. + const counted = new Set([id]) + const truncated = walk(children, MAX_DOWNSTREAM, counted) const graph: LineageGraph = { nodes: {}, parents: {}, children: {}, truncated, upTruncated } for (const one of keep) { graph.nodes[one] = nodeOf(one, lookup(manifest, one)) graph.parents[one] = (parents[one] ?? []).filter((other) => keep.has(other)) graph.children[one] = (children[one] ?? []).filter((other) => { if (keep.has(other)) return true - if (!isTest(other)) return false + if (!isTest(other) || !counted.has(one)) return false if (!graph.nodes[other]) { graph.nodes[other] = { ...nodeOf(other, lookup(manifest, other)), type: "test" } graph.parents[other] = (parents[other] ?? []).filter((p) => keep.has(p)) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index c9bed34903..4a01610ee2 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -323,7 +323,10 @@ describe("lineageGraph", () => { test("a twelve-hop chain comes back whole to its source; upstream past 500 nodes is cut and marked", () => { const chain: Record = {} for (let i = 1; i <= 12; i++) chain[`model.shop.m${i}`] = model(`m${i}`, [i === 1 ? "source.shop.raw.orders" : `model.shop.m${i - 1}`], "select 1", []) + chain["test.shop.m3_unique"] = { resource_type: "test", name: "m3_unique", depends_on: { nodes: ["model.shop.m3"] } } const graph = lineageGraph({ nodes: chain, sources: { "source.shop.raw.orders": { resource_type: "source", name: "orders", source_name: "raw" } } }, "model.shop.m12") + // An upstream model's tests are left out: only the target's and downstream tests are counted. + expect(graph.nodes["test.shop.m3_unique"]).toBeUndefined() expect(graph.nodes["source.shop.raw.orders"]).toBeDefined() expect(Object.keys(graph.nodes)).toHaveLength(13) expect(graph.upTruncated).toBe(false) From 36736970bd35779f023ac92e6e501c786b864a7b Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 10:50:42 +0530 Subject: [PATCH 07/37] fix: address review comments on lineage structured results Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/mcp-server/server.ts | 2 +- .../src/altimate/native/dbt/column-reach.ts | 28 +++++++--- .../src/altimate/native/dbt/lineage.ts | 18 +++--- .../src/altimate/tools/dbt-lineage.ts | 15 ++--- .../src/altimate/tools/impact-analysis.ts | 9 +-- .../test/altimate/column-reach.test.ts | 56 ++++++++++++++++++- .../altimate/dbt-lineage-structured.test.ts | 18 ++++-- .../test/altimate/impact-analysis.test.ts | 12 ++++ 8 files changed, 121 insertions(+), 37 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index c5f9ea155b..6bd4cafeff 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -91,7 +91,7 @@ export function structuredResult(tool: string, metadata: Record } const fields = Object.fromEntries((STRUCTURED_FIELDS[tool] ?? []).filter((key) => key in metadata).map((key) => [key, metadata[key]])) // dbt_lineage's text prints the same mappings and edges as `columns` and `edges`: keep its header. - const head = tool === "dbt_lineage" && "columns" in fields ? report.split("\nColumn Mappings:")[0]!.trim() : report + const head = tool === "dbt_lineage" && "columns" in fields ? report.split(/\n(?:Column Mappings|Lineage Edges):/)[0]!.trim() : report const full = { ...fields, report: head } return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) } diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 3a39a9f446..4f58eee8cf 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -103,12 +103,13 @@ function documentedColumns(node: any): Set { const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") /** - * Whether compiled SQL names the column anywhere but as an output alias, comments and string - * literals aside. Used only to mark a model as one that may read the column. + * SQL with comments blanked and string literals (quoted or dollar-quoted) emptied, in one pass so + * none can swallow another. */ -/** SQL with comments blanked and string literals emptied, in one pass so neither can swallow the other. */ function stripSql(sql: string): string { - return sql.replace(/'(?:[^']|'')*'|--[^\r\n]*|\/\*[\s\S]*?\*\//g, (match) => (match.startsWith("'") ? "''" : " ")) + return sql.replace(/'(?:[^']|'')*'|\$([A-Za-z_]\w*)?\$[\s\S]*?\$\1\$|--[^\r\n]*|\/\*[\s\S]*?\*\//g, (match) => + match.startsWith("'") || match.startsWith("$") ? "''" : " ", + ) } /** @@ -128,7 +129,9 @@ export function selectsStar(sql: string): boolean { if (prev === "(" && next === ")" && /\bcount\s*\($/i.test(before)) continue if (prev === "." || next === "" || next === "," || next === ")" || ends.test(after)) return true const operand = /[\w"'`\])]/ - if (operand.test(prev) && (operand.test(next) || next === "(" || /\d/.test(next))) { + // A sign on the right operand (`revenue * -1`) belongs to it. + const right = after.replace(/^[+-]\s*/, "")[0] ?? "" + if (operand.test(prev) && (operand.test(right) || right === "(" || /\d/.test(right))) { // An operand before the `*` that is itself a keyword (SELECT, DISTINCT, ALL, STRUCT) is no operand. if (/\b(select|distinct|all|struct)$/i.test(before)) return true continue @@ -138,6 +141,10 @@ export function selectsStar(sql: string): boolean { return false } +/** + * Whether compiled SQL names the column anywhere but as an output alias, comments and string + * literals aside. Used only to mark a model as one that may read the column. + */ function namesColumn(sql: string, column: string): boolean { const code = stripSql(sql) const name = escapeRegExp(column) @@ -204,7 +211,11 @@ export function columnReach(manifestPath: string, model: string, column: string, // Columns certainly carried from the traced column, per node. const tracked = new Map>([[targetId, new Set([column0])]]) // The same columns with their case kept: consulted only where a parent's names differ by case alone. - const trackedExact = new Map>([[targetId, new Set([column])]]) + // The target's own spelling comes from the manifest: the caller's when documented as typed, else + // every spelling matching ignoring case, else the caller's. + const spellings = Object.entries(target.columns ?? {}).map(([key, col]) => String(col?.name ?? key)) + const spelled = spellings.includes(column) ? [column] : spellings.filter((name) => name.toLowerCase() === column0) + const trackedExact = new Map>([[targetId, new Set(spelled.length > 0 ? spelled : [column])]]) const carries = (parent: string, source: { column: string; exact: string }) => !!tracked.get(parent)?.has(source.column) && (!caseCollides(lookup(parent), source.column) || !!trackedExact.get(parent)?.has(source.exact)) @@ -236,7 +247,7 @@ export function columnReach(manifestPath: string, model: string, column: string, unsure.add(id) continue } - const edges: { source: unknown; target: string; lineage_type?: string }[] = Array.isArray(lineage?.column_lineage) + const edges: { source: unknown; target: string; lineage_type?: string }[] = lineage?.column_lineage?.length ? lineage.column_lineage : Object.entries(lineage?.column_dict ?? {}).flatMap(([out, sources]) => (Array.isArray(sources) ? sources : [sources]).map((source) => ({ source, target: out, lineage_type: "direct" })), @@ -298,8 +309,7 @@ export function columnReach(manifestPath: string, model: string, column: string, uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) continue } - const sql: string = nodes[id]?.compiled_code || nodes[id]?.compiled_sql || "" - const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sql, col))) + const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sqlText, col))) const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) if (ambiguous || named.length > 0 || fromUnsure) { if (ambiguous || fromUnsure) unsure.add(id) diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index a5918f60b5..5522cb9972 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -6,7 +6,7 @@ import * as core from "@altimateai/altimate-core" import type { DbtLineageParams, DbtLineageResult } from "../types" -import { loadRawManifest, findModel, getUniqueId, detectDialect, buildSchemaContext } from "./helpers" +import { loadRawManifest, findModel, detectDialect, buildSchemaContext } from "./helpers" import { lineageGraph } from "./graph" function unparsed(lineage: Record): boolean { @@ -42,23 +42,25 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { const nodes = manifest.nodes || {} const sources = manifest.sources || {} - // Find target model by name or unique_id; a snapshot by its unique_id (it has compiled SQL too) // A model by name or unique_id; a snapshot by unique_id; either by its project-relative file path. - const byPath = Object.entries(nodes).find( - ([, node]) => ["model", "snapshot"].includes(node?.resource_type) && node.original_file_path === params.model, - ) + const byPath = () => + Object.values(nodes).find( + (node) => ["model", "snapshot"].includes(node?.resource_type) && node.original_file_path === params.model, + ) const modelNode = findModel(nodes, params.model) ?? (nodes[params.model]?.resource_type === "snapshot" ? nodes[params.model] : null) ?? - byPath?.[1] ?? + byPath() ?? Object.values(nodes).find((node) => node?.resource_type === "snapshot" && node.name === params.model) ?? null if (!modelNode) { return emptyResult([`Model '${params.model}' not found in manifest`]) } + // The id of the node that was selected, not of whatever else in the manifest the request string spells. const uniqueId = - getUniqueId(nodes, params.model) ?? - (nodes[params.model] ? params.model : (byPath?.[0] ?? Object.keys(nodes).find((id) => nodes[id] === modelNode))) + (typeof modelNode.unique_id === "string" && nodes[modelNode.unique_id] === modelNode ? modelNode.unique_id : undefined) ?? + Object.keys(nodes).find((id) => nodes[id] === modelNode) ?? + params.model const graph = params.include_graph && uniqueId ? { graph: lineageGraph(manifest, uniqueId) } : {} // Extract compiled SQL diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 5bc9cc83b5..81d99dca6f 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -53,7 +53,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { include_graph: z .boolean() .optional() - .describe("Also return the model's graph from the manifest (everything upstream and downstream, up to 500 nodes each side, with exposures and tests; graph.upTruncated / graph.truncated are set when a side is cut) in the metadata; for lineage viewers"), + .describe("Also return the model's graph from the manifest (everything upstream and downstream, up to 500 nodes each side; the exposures and tests are those of the model and its downstream, upstream tests are not included; graph.upTruncated / graph.truncated are set when a side is cut) in the metadata; for lineage viewers"), }), async execute(args, ctx) { let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) @@ -75,7 +75,8 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { model_unique_id: result.focus.uniqueId, confidence: result.confidence, structuredContent: result, - ...(graph && { graph }), + ...(graph?.graph && { graph: graph.graph }), + ...(graph?.error && { graph_error: graph.error }), }, output: formatModelLineage(result), } @@ -149,13 +150,13 @@ async function manifestOutsideAdapterProject(manifestPath: string): Promise; error?: string } { try { const manifest = loadRawManifest(manifestPath) - return manifest ? lineageGraph(manifest, uniqueId) : undefined - } catch { - return undefined + return manifest ? { graph: lineageGraph(manifest, uniqueId) } : { error: `no manifest at ${manifestPath}` } + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) } } } diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index 63e205d389..507ee87918 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -61,11 +61,6 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { } // Step 3: Build the dependency graph and find all downstream models - const modelsByName = new Map() - for (const m of manifest.models) { - modelsByName.set(m.name, m) - } - // Find all models that depend on the target (direct + transitive) const downstream = findDownstream(targetModel.name, manifest.models, targetModel.unique_id) const direct = downstream.filter((d) => d.depth === 1) @@ -73,7 +68,7 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { // Step 4: Count only tests that reference the target model or its downstream models const affectedModelIds = new Set( - [targetModel.unique_id, ...downstream.map((d) => d.unique_id ?? modelsByName.get(d.name)?.unique_id)].filter( + [targetModel.unique_id, ...downstream.map((d) => d.unique_id)].filter( (id): id is string => Boolean(id), ), ) @@ -147,7 +142,7 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { model: targetModel.name, column: args.column ?? null, change_type: args.change_type, - downstream: downstream.map((d) => ({ name: d.name, depth: d.depth, materialized: d.materialized ?? null, path: d.path })), + downstream: downstream.map((d) => ({ name: d.name, depth: d.depth, materialized: d.materialized ?? null })), exposures, ...(reach && { column_reach: reach }), ...(reachError && { column_reach_error: reachError }), diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 4a01610ee2..fb476e2268 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, test } from "bun:test" +import { afterEach, describe, expect, spyOn, test } from "bun:test" +import * as core from "@altimateai/altimate-core" import { mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" @@ -253,6 +254,12 @@ describe("selectsStar", () => { "select (a + b) * c as d from t", "select a * (b + c) as d from t", "select x from a -- select *", + "select revenue * -1 as x from t", + "select a*-1 as x from t", + "select a * + b as x from t", + "select a * - (b + c) as x from t", + "select $$ * $$ as s from a", + "select $q$ select * $q$ as s from a", ]) expect(selectsStar(sql)).toBe(false) // A function over the whole row reads every column; only count(*) reads none. @@ -307,6 +314,30 @@ describe("columnReach with columns that differ only by case", () => { }) }) +describe("columnReach case handling", () => { + test("a caller's spelling of a column the parent documents twice by case still reaches its readers", () => { + const value: any = manifest() + value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } + const reach = columnReach(manifestFile(value), "fct_orders", "REVENUE") + expect(reach.models.map((m) => m.name)).toContain("fct_daily") + }) +}) + +describe("columnReach with an empty column_lineage", () => { + test("falls back to column_dict edges", () => { + const spy = spyOn(core, "columnLineage").mockImplementation((() => ({ + column_lineage: [], + column_dict: { revenue: ['"shop"."main"."fct_orders"."revenue"'] }, + })) as any) + try { + const reach = columnReach(manifestFile(), "fct_orders", "revenue") + expect(reach.models.find((m) => m.name === "fct_daily")?.columns).toEqual(["revenue"]) + } finally { + spy.mockRestore() + } + }) +}) + describe("columnReach on SQL the engine cannot parse", () => { test("is untraced, and a dashboard on it is reported apart", () => { const value = manifest() @@ -338,6 +369,19 @@ describe("lineageGraph", () => { expect(Object.keys(cut.nodes)).toHaveLength(501) }) + test("downstream past 500 nodes is cut and marked; downstream tests are still counted", () => { + const wide: Record = { "model.shop.top": model("top", [], "select 1", []) } + for (let i = 0; i < 600; i++) wide[`model.shop.c${i}`] = model(`c${i}`, ["model.shop.top"], "select 1", []) + wide["test.shop.top_unique"] = { resource_type: "test", name: "top_unique", depends_on: { nodes: ["model.shop.top"] } } + wide["test.shop.c0_unique"] = { resource_type: "test", name: "c0_unique", depends_on: { nodes: ["model.shop.c0"] } } + const cut = lineageGraph({ nodes: wide }, "model.shop.top") + expect(cut.truncated).toBe(true) + expect(cut.upTruncated).toBe(false) + const kept = Object.values(cut.nodes) + expect(kept.filter((n) => n.type !== "test")).toHaveLength(501) + expect(kept.filter((n) => n.type === "test").map((n) => n.id).sort()).toEqual(["test.shop.c0_unique", "test.shop.top_unique"]) + }) + test("everything up and downstream, tests as ids, from dbt's child_map when present", () => { const value = { ...manifest(), @@ -421,6 +465,12 @@ describe("structuredResult", () => { edges: [], report: "Model: x\nID: y", }) + // With no column_dict the text has no Column Mappings section: the edges are cut at their own header. + expect(structuredResult("dbt_lineage", { columns: {}, edges: [] }, "Model: x\nID: y\n\nLineage Edges:\n {}")).toEqual({ + columns: {}, + edges: [], + report: "Model: x\nID: y", + }) }) }) @@ -436,5 +486,9 @@ describe("dbtLineage", () => { const withSnapshot: any = manifest() withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" } expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap") + // A snapshot named like another node's unique_id keeps its own id. + withSnapshot.nodes["test.shop.t1"] = { resource_type: "test", name: "t1", depends_on: { nodes: [] } } + withSnapshot.nodes["snapshot.shop.odd"] = { ...withSnapshot.nodes["snapshot.shop.snap"], unique_id: "snapshot.shop.odd", name: "test.shop.t1" } + expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "test.shop.t1" }).model_unique_id).toBe("snapshot.shop.odd") }) }) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index 080fedf2da..3ad24321a7 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -96,9 +96,9 @@ function fakeAdapter(initialized: boolean, lineage?: (params: { model: string; c } } -function useConfig(autoDiscovered: boolean, dbtIntegration = "corecommand") { +function useConfig(autoDiscovered: boolean, dbtIntegration = "corecommand", projectRoot = project) { dbtAdapterInternals.readConfig = async () => ({ - projectRoot: project, + projectRoot, pythonPath: "python3", dbtIntegration, queryLimit: 500, @@ -161,9 +161,12 @@ describe("dbt_lineage with the project's dbt adapter", () => { const other = path.join(dir, "other") await Promise.all([mine, other].map((d) => mkdir(path.join(d, "target"), { recursive: true }))) const empty = JSON.stringify({ metadata: { adapter_type: "duckdb" }, nodes: {}, sources: {} }) + await writeFile(path.join(mine, "dbt_project.yml"), "name: mine\n") await writeFile(path.join(mine, "target", "manifest.json"), empty) await writeFile(path.join(other, "target", "manifest.json"), empty) - const root = spyOn(register, "getDbtProjectRoot").mockResolvedValue(mine) + // The adapter itself is configured with `mine`, so the tool reads the root the adapter cached. + resetDbtAdapter() + useConfig(false, "corecommand", mine) try { const tool = await initTool(DbtLineageTool) const fallback = await tool.execute({ model: "customers", manifest_path: path.join(other, "target", "manifest.json") }, ctx) @@ -180,11 +183,18 @@ describe("dbt_lineage with the project's dbt adapter", () => { expect(inner.metadata.structuredContent).toBeUndefined() expect(inner.output).toContain("belongs to the dbt project at") } finally { - root.mockRestore() await rm(dir, { recursive: true, force: true }) } }) + test("include_graph with no manifest to read says so in graph_error", async () => { + const tool = await initTool(DbtLineageTool) + const result = await tool.execute({ model: "customers", include_graph: true }, ctx) + expect(result.metadata.structuredContent).toEqual(TRACE) + expect(result.metadata.graph).toBeUndefined() + expect(result.metadata.graph_error).toBe(`no manifest at ${path.join(project, "target", "manifest.json")}`) + }) + test("is handed to an MCP client as structuredContent", async () => { const server = await createServer() const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() diff --git a/packages/opencode/test/altimate/impact-analysis.test.ts b/packages/opencode/test/altimate/impact-analysis.test.ts index 2ce470e672..e93083aa84 100644 --- a/packages/opencode/test/altimate/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/impact-analysis.test.ts @@ -107,6 +107,18 @@ describe("impact_analysis: DAG traversal", () => { expect(result.output).toContain("BREAKING") }) + test("the structured downstream entries carry no path", async () => { + mockDispatcher({ "dbt.manifest": linearDAG }) + const tool = await initTool(ImpactAnalysisTool) + const result = await tool.execute( + { model: "stg_orders", change_type: "remove", manifest_path: "target/manifest.json", dialect: "snowflake" }, + ctx, + ) + const downstream = result.metadata.downstream as Record[] + expect(downstream.map((d) => d.name)).toEqual(["int_orders", "fct_orders", "rpt_daily"]) + expect(downstream.every((d) => !("path" in d))).toBe(true) + }) + test("SAFE severity when no downstream models exist", async () => { mockDispatcher({ "dbt.manifest": linearDAG }) const tool = await initTool(ImpactAnalysisTool) From 4ac010e88110cc5099c7560d4d46e41a0ccc3aee Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 10:57:52 +0530 Subject: [PATCH 08/37] fix: a column spelling that matches two columns differing only by case leaves their readers uncertain Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/native/dbt/column-reach.ts | 11 +++++++---- packages/opencode/test/altimate/column-reach.test.ts | 12 ++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 4f58eee8cf..33f38243e9 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -212,16 +212,19 @@ export function columnReach(manifestPath: string, model: string, column: string, const tracked = new Map>([[targetId, new Set([column0])]]) // The same columns with their case kept: consulted only where a parent's names differ by case alone. // The target's own spelling comes from the manifest: the caller's when documented as typed, else - // every spelling matching ignoring case, else the caller's. + // the one spelling matching ignoring case, else the caller's. Two documented spellings that only + // case tells apart, neither typed as given, leave the column unsettled: no reader of either is certain. const spellings = Object.entries(target.columns ?? {}).map(([key, col]) => String(col?.name ?? key)) - const spelled = spellings.includes(column) ? [column] : spellings.filter((name) => name.toLowerCase() === column0) - const trackedExact = new Map>([[targetId, new Set(spelled.length > 0 ? spelled : [column])]]) + const matches = spellings.filter((name) => name.toLowerCase() === column0) + const unsettled = !spellings.includes(column) && matches.length > 1 + const spelled = spellings.includes(column) ? [column] : unsettled ? [] : matches.length > 0 ? matches : [column] + const trackedExact = new Map>([[targetId, new Set(spelled)]]) const carries = (parent: string, source: { column: string; exact: string }) => !!tracked.get(parent)?.has(source.column) && (!caseCollides(lookup(parent), source.column) || !!trackedExact.get(parent)?.has(source.exact)) // Nodes whose outputs are unknown (untraced, or read through an ambiguous name): their children // may read the column too. A node that only filters on it was traced and carries nothing on. - const unsure = new Set() + const unsure = new Set(unsettled ? [targetId] : []) const reached: ColumnReachModel[] = [] const uncertain: ColumnReachUnsure[] = [] const untraced: ColumnReachUnsure[] = [] diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index fb476e2268..0f58f17fc1 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -315,11 +315,19 @@ describe("columnReach with columns that differ only by case", () => { }) describe("columnReach case handling", () => { - test("a caller's spelling of a column the parent documents twice by case still reaches its readers", () => { + test("a spelling matching one documented column ignoring case reaches its readers", () => { + const reach = columnReach(manifestFile(), "fct_orders", "REVENUE") + expect(reach.models.map((m) => m.name)).toContain("fct_daily") + }) + + test("a spelling matching two columns that differ only by case leaves both columns' readers uncertain, never certain", () => { const value: any = manifest() value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } + value.nodes["model.shop.rpt_case"] = model("rpt_case", ["model.shop.fct_orders"], 'select "Revenue" as other from fct_orders', ["other"]) const reach = columnReach(manifestFile(value), "fct_orders", "REVENUE") - expect(reach.models.map((m) => m.name)).toContain("fct_daily") + expect(reach.models.map((m) => m.name)).not.toContain("fct_daily") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_case") + expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["fct_daily", "rpt_case"])) }) }) From f270dbbc0385d50a970b9793d1e75af4370e2d67 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 15:52:13 +0530 Subject: [PATCH 09/37] fix: an unsettled column spelling leaves the model's own dashboards uncertain, and project containment compares whole path segments Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 13 +++++++++---- packages/opencode/src/altimate/tools/dbt-lineage.ts | 6 ++++-- .../opencode/test/altimate/column-reach.test.ts | 9 +++++++++ .../test/altimate/dbt-lineage-structured.test.ts | 12 ++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 33f38243e9..f3e721f8b2 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -216,8 +216,11 @@ export function columnReach(manifestPath: string, model: string, column: string, // case tells apart, neither typed as given, leave the column unsettled: no reader of either is certain. const spellings = Object.entries(target.columns ?? {}).map(([key, col]) => String(col?.name ?? key)) const matches = spellings.filter((name) => name.toLowerCase() === column0) - const unsettled = !spellings.includes(column) && matches.length > 1 - const spelled = spellings.includes(column) ? [column] : unsettled ? [] : matches.length > 0 ? matches : [column] + const exact = spellings.includes(column) + const unsettled = !exact && matches.length > 1 + let spelled: string[] = [column] + if (unsettled) spelled = [] + else if (!exact && matches.length > 0) spelled = matches const trackedExact = new Map>([[targetId, new Set(spelled)]]) const carries = (parent: string, source: { column: string; exact: string }) => !!tracked.get(parent)?.has(source.column) && @@ -331,9 +334,11 @@ export function columnReach(manifestPath: string, model: string, column: string, } } - const reachedIds = new Set([targetId, ...reached.map((m) => m.unique_id)]) + // An unsettled spelling leaves the target's own dashboards uncertain too: which column they read is unknown. + const reachedIds = new Set([...(unsettled ? [] : [targetId]), ...reached.map((m) => m.unique_id)]) const certain = exposuresOn(manifestPath, reachedIds) - const maybe = exposuresOn(manifestPath, [...uncertain, ...untraced].map((m) => m.unique_id)).filter( + const unsureIds = [...uncertain, ...untraced].map((m) => m.unique_id) + const maybe = exposuresOn(manifestPath, unsettled ? [targetId, ...unsureIds] : unsureIds).filter( (e) => !certain.some((c) => c.unique_id === e.unique_id), ) return { models: reached, uncertain, untraced, exposures: certain, uncertain_exposures: maybe, truncated } diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 984ff9548d..d78fb89865 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -143,8 +143,10 @@ async function manifestOutsideAdapterProject(manifestPath: string): Promise { expect(reach.models.map((m) => m.name)).not.toContain("rpt_case") expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["fct_daily", "rpt_case"])) }) + + test("with the spelling unsettled, a dashboard on the model itself is uncertain too", () => { + const value: any = manifest() + value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } + value.exposures["exposure.shop.orders_board"] = { name: "orders_board", type: "dashboard", depends_on: { nodes: ["model.shop.fct_orders"] } } + const reach = columnReach(manifestFile(value), "fct_orders", "REVENUE") + expect(reach.exposures.map((e) => e.unique_id)).not.toContain("exposure.shop.orders_board") + expect(reach.uncertain_exposures.map((e) => e.unique_id)).toContain("exposure.shop.orders_board") + }) }) describe("columnReach with an empty column_lineage", () => { diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index 8825c55a61..d1c716570e 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -179,6 +179,18 @@ describe("dbt_lineage with the project's dbt adapter", () => { const inner = await tool.execute({ model: "customers", manifest_path: path.join(nested, "target", "manifest.json") }, ctx) expect(inner.metadata.structuredContent).toBeUndefined() expect(inner.output).toContain("belongs to the dbt project at") + // A folder whose name starts with `..` is still inside the project. + const dotted = path.join(mine, "..data") + await mkdir(path.join(dotted, "target"), { recursive: true }) + await writeFile(path.join(dotted, "target", "manifest.json"), empty) + const own = await tool.execute({ model: "customers", manifest_path: path.join(dotted, "target", "manifest.json") }, ctx) + expect(own.metadata.structuredContent).toEqual(TRACE) + // A sibling whose name only starts with this project's is outside it. + const sibling = `${mine}-2` + await mkdir(path.join(sibling, "target"), { recursive: true }) + await writeFile(path.join(sibling, "target", "manifest.json"), empty) + const next = await tool.execute({ model: "customers", manifest_path: path.join(sibling, "target", "manifest.json") }, ctx) + expect(next.metadata.structuredContent).toBeUndefined() } finally { await rm(dir, { recursive: true, force: true }) } From 7c89b71e7a7583ae5fc8f065dfda0b533d9bc0a7 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:07:09 +0530 Subject: [PATCH 10/37] fix: uncertainty for models the engine cannot fully trace, a token-safe structured cap, header-only adapter report, dashboards through metrics Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/mcp-server/server.ts | 9 ++- .../src/altimate/native/dbt/column-reach.ts | 26 +++++- .../test/altimate/column-reach.test.ts | 81 ++++++++++++++++++- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index bbd9545858..30f9603a7a 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -78,15 +78,18 @@ const STRUCTURED_OUTPUT_SCHEMA = { additionalProperties: true, } // The structured result reaches the model in Claude Code, so it stays within the same order of -// size as a text result: past this, only the report is sent. -const STRUCTURED_MAX_BYTES = 100_000 +// size as a text result: past this, only the report is sent. Claude Code caps an MCP result at +// 25k tokens by default and JSON of ids and edges runs ~3-4 characters a token, so this stays under that. +const STRUCTURED_MAX_BYTES = 60_000 export function structuredResult(tool: string, metadata: Record, report: string): Record { // A tool that builds its own structured result (dbt_lineage from the dbt integration) hands it // over whole; otherwise the lineage fields of its metadata are passed. const own = metadata.structuredContent if (own && typeof own === "object" && !Array.isArray(own)) { - const full = { ...(own as Record), report } + // The text repeats every edge that `columnLineage` already carries: only its header goes with them. + const head = report.split(/\n(?:Column edges \(|Trace of )/)[0]!.trim() + const full = { ...(own as Record), report: head } return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) } const fields = Object.fromEntries((STRUCTURED_FIELDS[tool] ?? []).filter((key) => key in metadata).map((key) => [key, metadata[key]])) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index f3e721f8b2..48072485bc 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -258,6 +258,8 @@ export function columnReach(manifestPath: string, model: string, column: string, : Object.entries(lineage?.column_dict ?? {}).flatMap(([out, sources]) => (Array.isArray(sources) ? sources : [sources]).map((source) => ({ source, target: out, lineage_type: "direct" })), ) + // Errors beside some edges: the engine traced only part of the SQL. + const partial = Array.isArray(lineage?.errors) && lineage.errors.length > 0 // Two parents with the same table name share one entry in the engine's schema, so its edges // cannot say which one is read; a `select *` from a parent whose schema does not list the @@ -305,16 +307,28 @@ export function columnReach(manifestPath: string, model: string, column: string, tracked.set(id, carried) trackedExact.set(id, carriedExact) // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. - if (ambiguous || fromUnsure || viaEphemeral) unsure.add(id) + if (ambiguous || fromUnsure || viaEphemeral || partial) unsure.add(id) reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) continue } + if (partial) { + unsure.add(id) + uncertain.push(entry(id, "the lineage engine traced only part of its SQL")) + continue + } if (viaEphemeral) { unsure.add(id) uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) continue } + // With a known parent, no edge at all (a PIVOT, say) is the engine not seeing through the SQL, + // not a model that reads nothing. Below an unsure parent it is already reported as uncertain. + if (edges.length === 0 && !fromUnsure) { + untraced.push(entry(id, "the lineage engine found no column edges in its SQL")) + unsure.add(id) + continue + } const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sqlText, col))) const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) if (ambiguous || named.length > 0 || fromUnsure) { @@ -348,6 +362,16 @@ export function columnReach(manifestPath: string, model: string, column: string, export function exposuresOn(manifestPath: string, ids: Iterable): ColumnReachExposure[] { const manifest = loadRawManifest(manifestPath) const wanted = new Set(ids) + // A metric reads its semantic model, which reads a model: whatever depends on one depends on the others. + const layers = [...Object.entries(manifest?.semantic_models ?? {}), ...Object.entries(manifest?.metrics ?? {})] + for (let grew = true; grew; ) { + grew = false + for (const [id, node] of layers) { + if (wanted.has(id) || !(node?.depends_on?.nodes ?? []).some((dep: string) => wanted.has(dep))) continue + wanted.add(id) + grew = true + } + } return Object.entries(manifest?.exposures ?? {}) .filter(([, node]) => (node?.depends_on?.nodes ?? []).some((dep: string) => wanted.has(dep))) .map(([id, node]) => ({ unique_id: id, name: node?.name ?? id, label: node?.label || node?.name || id, type: node?.type ?? "exposure" })) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index de973073d7..e252f59959 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -166,6 +166,19 @@ describe("columnReach", () => { }) }) +describe("exposuresOn through metrics", () => { + test("an exposure on a metric reaches the model under its semantic model", () => { + const value: any = manifest() + value.semantic_models = { "semantic_model.shop.orders": { depends_on: { nodes: ["model.shop.rpt_counts"] } } } + value.metrics = { + "metric.shop.order_count": { depends_on: { nodes: ["semantic_model.shop.orders"] } }, + "metric.shop.double_count": { depends_on: { nodes: ["metric.shop.order_count"] } }, + } + value.exposures["exposure.shop.kpi"] = { name: "kpi", type: "dashboard", depends_on: { nodes: ["metric.shop.double_count"] } } + expect(exposuresOn(manifestFile(value), ["model.shop.rpt_counts"]).map((e) => e.unique_id).sort()).toEqual(["exposure.shop.kpi", "exposure.shop.ops"]) + }) +}) + describe("exposuresOn", () => { test("lists exposures on any of the given nodes, by label when there is one", () => { const file = manifestFile() @@ -355,6 +368,60 @@ describe("columnReach with an empty column_lineage", () => { }) }) +describe("columnReach on a model the engine cannot fully trace", () => { + const answer = (byModel: (sql: string) => unknown) => + spyOn(core, "columnLineage").mockImplementation(((sql: string) => byModel(sql)) as any) + const direct = (sql: string) => ({ column_lineage: [{ source: '"shop"."main"."fct_orders"."revenue"', target: "revenue", lineage_type: "direct" }], column_dict: {} }) + + test("no edge at all is untraced, and its child is uncertain", () => { + const value: any = manifest() + value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.mart_ltv"], "select 1", ["top_ltv"]) + const spy = answer((sql) => (sql === "select 1" ? { column_lineage: [], column_dict: {} } : direct(sql))) + try { + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.untraced.map((m) => m.name)).toEqual(["rpt_ltv"]) + expect(reach.untraced[0]!.reason).toContain("no column edges") + } finally { + spy.mockRestore() + } + const chain: any = manifest() + chain.nodes["model.shop.fct_daily"].compiled_code = "select 2" + chain.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(revenue) as top from fct_daily", ["top"]) + const again = answer((sql) => (sql === "select 2" ? { column_lineage: [], column_dict: {} } : direct(sql))) + try { + const reach = columnReach(manifestFile(chain), "fct_orders", "revenue") + expect(reach.untraced.map((m) => m.name)).toEqual(["fct_daily"]) + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_ltv") + } finally { + again.mockRestore() + } + }) + + test("errors beside edges: a reader stays reached but unsure, a non-reader is uncertain", () => { + const value: any = manifest() + value.nodes["model.shop.fct_daily"].compiled_code = "select 3" + value.nodes["model.shop.rpt_counts"].compiled_code = "select 4" + value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(revenue) as top from fct_daily", ["top"]) + const spy = answer((sql) => + sql === "select 3" + ? { ...direct(sql), errors: ["partial"] } + : sql === "select 4" + ? { column_lineage: [{ source: '"shop"."main"."fct_orders"."order_id"', target: "orders", lineage_type: "direct" }], column_dict: {}, errors: ["partial"] } + : direct(sql), + ) + try { + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).toContain("fct_daily") + // Its own child is checked though fct_daily carried the column on. + expect(reach.uncertain.find((m) => m.name === "rpt_counts")?.reason).toBe("the lineage engine traced only part of its SQL") + expect(reach.uncertain_exposures.map((e) => e.unique_id)).toContain("exposure.shop.ops") + expect(reach.exposures.map((e) => e.unique_id)).not.toContain("exposure.shop.ops") + } finally { + spy.mockRestore() + } + }) +}) + describe("columnReach on SQL the engine cannot parse", () => { test("is untraced, and a dashboard on it is reported apart", () => { const value = manifest() @@ -440,7 +507,15 @@ describe("structuredError", () => { expect(structuredError("boom")).toEqual({ report: "boom" }) const out = structuredError("x".repeat(300_000)) expect(out.truncated).toBe(true) - expect(Buffer.byteLength(out.report as string)).toBeLessThanOrEqual(100_000) + expect(Buffer.byteLength(out.report as string)).toBeLessThanOrEqual(60_000) + }) + test("a tool's own structured result sends the report's header, not its edge lists", () => { + const text = "Model: x (model.a.x) · table · confidence high\n\nUpstream (0):\n none\n\nColumn edges (1):\n x.a ← y.a\n\nNotes: n" + expect(structuredResult("dbt_lineage", { structuredContent: { columnLineage: [1] } }, text)).toEqual({ + columnLineage: [1], + report: "Model: x (model.a.x) · table · confidence high\n\nUpstream (0):\n none", + }) + expect((structuredResult("dbt_lineage", { structuredContent: {} }, "Model: x\n\nTrace of x.a (1 edges):\n e") as any).report).toBe("Model: x") }) test("exposures_error reaches the structured fields", () => { expect(structuredResult("impact_analysis", { exposures_error: "bad json" }, "R")).toEqual({ exposures_error: "bad json", report: "R" }) @@ -451,7 +526,7 @@ describe("structuredResult over the cap", () => { test("a report too big on its own is shortened too", () => { const out = structuredResult("dbt_lineage", { structuredContent: { focus: {} } }, "x".repeat(300_000)) expect(out.truncated).toBe(true) - expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(100_000) + expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(60_000) expect("x".repeat(300_000).startsWith(out.report as string)).toBe(true) }) }) @@ -460,7 +535,7 @@ describe("structuredError escaping", () => { test("the cap holds for the serialized result, whatever JSON escaping adds", () => { for (const text of ["\n".repeat(100_000), "\u0000".repeat(300_000), "é".repeat(80_000)]) { const out = structuredError(text) - expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(100_000) + expect(Buffer.byteLength(JSON.stringify(out))).toBeLessThanOrEqual(60_000) expect(out.truncated).toBe(true) } }) From a0a47c0e5794265b6e897886638d47d73386689c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:22:19 +0530 Subject: [PATCH 11/37] fix: a model with no column edges is uncertain only when its SQL names the column Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 9 +++--- .../test/altimate/column-reach.test.ts | 28 +++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 48072485bc..5310be65ba 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -322,10 +322,11 @@ export function columnReach(manifestPath: string, model: string, column: string, uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) continue } - // With a known parent, no edge at all (a PIVOT, say) is the engine not seeing through the SQL, - // not a model that reads nothing. Below an unsure parent it is already reported as uncertain. - if (edges.length === 0 && !fromUnsure) { - untraced.push(entry(id, "the lineage engine found no column edges in its SQL")) + // No edge at all while the SQL names the column (a PIVOT, say) is the engine not seeing through + // it, so its outputs are unknown. SQL that never names it (`select 1`, `count(*)`) reads nothing and + // falls through. Below an unsure parent it is already reported as uncertain. + if (edges.length === 0 && !fromUnsure && parents.some((p) => [...tracked.get(p)!].some((col) => namesColumn(sqlText, col)))) { + uncertain.push(entry(id, "names it, but the lineage engine found no column edges in its SQL")) unsure.add(id) continue } diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index e252f59959..021e867b30 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -373,25 +373,25 @@ describe("columnReach on a model the engine cannot fully trace", () => { spyOn(core, "columnLineage").mockImplementation(((sql: string) => byModel(sql)) as any) const direct = (sql: string) => ({ column_lineage: [{ source: '"shop"."main"."fct_orders"."revenue"', target: "revenue", lineage_type: "direct" }], column_dict: {} }) - test("no edge at all is untraced, and its child is uncertain", () => { - const value: any = manifest() - value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.mart_ltv"], "select 1", ["top_ltv"]) - const spy = answer((sql) => (sql === "select 1" ? { column_lineage: [], column_dict: {} } : direct(sql))) + test("no edge at all: SQL that names the column (a PIVOT) is uncertain and so is its child; SQL that never names it reads nothing", () => { + const pivot = "select * from fct_orders pivot(sum(revenue) for status in ('paid' as paid_total))" + const chain: any = manifest() + chain.nodes["model.shop.fct_daily"].compiled_code = pivot + chain.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(paid_total) as top from fct_daily", ["top"]) + const spy = answer((sql) => (sql === pivot ? { column_lineage: [], column_dict: {} } : direct(sql))) try { - const reach = columnReach(manifestFile(value), "fct_orders", "revenue") - expect(reach.untraced.map((m) => m.name)).toEqual(["rpt_ltv"]) - expect(reach.untraced[0]!.reason).toContain("no column edges") + const reach = columnReach(manifestFile(chain), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["fct_daily", "rpt_ltv"])) + expect(reach.uncertain.find((m) => m.name === "fct_daily")!.reason).toContain("no column edges") } finally { spy.mockRestore() } - const chain: any = manifest() - chain.nodes["model.shop.fct_daily"].compiled_code = "select 2" - chain.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(revenue) as top from fct_daily", ["top"]) - const again = answer((sql) => (sql === "select 2" ? { column_lineage: [], column_dict: {} } : direct(sql))) + const constant: any = manifest() + constant.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.mart_ltv"], "select 1 as one", ["one"]) + const again = answer((sql) => (sql === "select 1 as one" ? { column_lineage: [], column_dict: {} } : direct(sql))) try { - const reach = columnReach(manifestFile(chain), "fct_orders", "revenue") - expect(reach.untraced.map((m) => m.name)).toEqual(["fct_daily"]) - expect(reach.uncertain.map((m) => m.name)).toContain("rpt_ltv") + const reach = columnReach(manifestFile(constant), "fct_orders", "revenue") + expect([...reach.untraced, ...reach.uncertain].map((m) => m.name)).not.toContain("rpt_ltv") } finally { again.mockRestore() } From eac31e5f321b2aac64a988554a4c3e83158020f2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:23:29 +0530 Subject: [PATCH 12/37] test: a model below a partly traced reader is a possible reader Co-Authored-By: Claude Opus 5.5 --- packages/opencode/test/altimate/column-reach.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 021e867b30..c9f76fa1dd 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -401,7 +401,8 @@ describe("columnReach on a model the engine cannot fully trace", () => { const value: any = manifest() value.nodes["model.shop.fct_daily"].compiled_code = "select 3" value.nodes["model.shop.rpt_counts"].compiled_code = "select 4" - value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(revenue) as top from fct_daily", ["top"]) + // Its SQL never names the column: only its partly traced parent makes it a possible reader. + value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(total) as top from fct_daily", ["top"]) const spy = answer((sql) => sql === "select 3" ? { ...direct(sql), errors: ["partial"] } @@ -412,6 +413,9 @@ describe("columnReach on a model the engine cannot fully trace", () => { try { const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.models.map((m) => m.name)).toContain("fct_daily") + // fct_daily is reached but only partly traced: its child may read the column, so it is uncertain, never a confident non-reader. + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_ltv") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_ltv") // Its own child is checked though fct_daily carried the column on. expect(reach.uncertain.find((m) => m.name === "rpt_counts")?.reason).toBe("the lineage engine traced only part of its SQL") expect(reach.uncertain_exposures.map((e) => e.unique_id)).toContain("exposure.shop.ops") From fe673cf81f65babb6fbd2700068c348322f7a6af Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:27:56 +0530 Subject: [PATCH 13/37] fix: a structured result passed through for a tool outside the declared set is held to the same cap Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/mcp-server/server.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 30f9603a7a..2f940e56e6 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -225,8 +225,13 @@ export async function createServer(options: ServeOptions = {}) { const structured = result.metadata?.structuredContent return { content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], - // Tools outside the declared set may still hand over a structured result of their own. - ...(structured && typeof structured === "object" && !isError && !STRUCTURED_TOOLS.has(request.params.name) + // Tools outside the declared set may still hand over a structured result of their own, held to + // the same cap: past it only the text goes. + ...(structured && + typeof structured === "object" && + !isError && + !STRUCTURED_TOOLS.has(request.params.name) && + Buffer.byteLength(JSON.stringify(structured)) <= STRUCTURED_MAX_BYTES ? { structuredContent: structured as Record } : {}), isError, From e80cf4cf67a2621887766be2dfc8ab0b74adfe8e Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:34:54 +0530 Subject: [PATCH 14/37] refactor: the column-name scan runs once per model; test rules out a certain reader for a constant model Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/column-reach.ts | 4 ++-- packages/opencode/test/altimate/column-reach.test.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 5310be65ba..648fba3e9e 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -322,15 +322,15 @@ export function columnReach(manifestPath: string, model: string, column: string, uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) continue } + const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sqlText, col))) // No edge at all while the SQL names the column (a PIVOT, say) is the engine not seeing through // it, so its outputs are unknown. SQL that never names it (`select 1`, `count(*)`) reads nothing and // falls through. Below an unsure parent it is already reported as uncertain. - if (edges.length === 0 && !fromUnsure && parents.some((p) => [...tracked.get(p)!].some((col) => namesColumn(sqlText, col)))) { + if (edges.length === 0 && !fromUnsure && named.length > 0) { uncertain.push(entry(id, "names it, but the lineage engine found no column edges in its SQL")) unsure.add(id) continue } - const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sqlText, col))) const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) if (ambiguous || named.length > 0 || fromUnsure) { if (ambiguous || fromUnsure) unsure.add(id) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index c9f76fa1dd..1daac56afb 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -392,6 +392,7 @@ describe("columnReach on a model the engine cannot fully trace", () => { try { const reach = columnReach(manifestFile(constant), "fct_orders", "revenue") expect([...reach.untraced, ...reach.uncertain].map((m) => m.name)).not.toContain("rpt_ltv") + expect(reach.models.map((m) => m.name)).not.toContain("rpt_ltv") } finally { again.mockRestore() } From 353523d8e43af8fb10726b322aa81a6afa34b185 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:42:09 +0530 Subject: [PATCH 15/37] style: indent the passthrough condition as Prettier does Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/mcp-server/server.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 2f940e56e6..0e38525111 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -228,10 +228,10 @@ export async function createServer(options: ServeOptions = {}) { // Tools outside the declared set may still hand over a structured result of their own, held to // the same cap: past it only the text goes. ...(structured && - typeof structured === "object" && - !isError && - !STRUCTURED_TOOLS.has(request.params.name) && - Buffer.byteLength(JSON.stringify(structured)) <= STRUCTURED_MAX_BYTES + typeof structured === "object" && + !isError && + !STRUCTURED_TOOLS.has(request.params.name) && + Buffer.byteLength(JSON.stringify(structured)) <= STRUCTURED_MAX_BYTES ? { structuredContent: structured as Record } : {}), isError, From f8fb664fdad04fb3697fd4c8282e528c56bd6f63 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 16:46:01 +0530 Subject: [PATCH 16/37] refactor: drop the structured passthrough no tool reaches; dbt_lineage takes the declared, capped path Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/mcp-server/server.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts index 0e38525111..852df3fb56 100644 --- a/packages/opencode/src/altimate/mcp-server/server.ts +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -222,18 +222,8 @@ export async function createServer(options: ServeOptions = {}) { const clarified = request.params.name === "answer_commit" ? await clarifyReading(request.params.arguments, result, extra.signal) : undefined if (clarified) return { content: [{ type: "text" as const, text: clarified }], isError: true } const isError = resultFailed(result) - const structured = result.metadata?.structuredContent return { content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], - // Tools outside the declared set may still hand over a structured result of their own, held to - // the same cap: past it only the text goes. - ...(structured && - typeof structured === "object" && - !isError && - !STRUCTURED_TOOLS.has(request.params.name) && - Buffer.byteLength(JSON.stringify(structured)) <= STRUCTURED_MAX_BYTES - ? { structuredContent: structured as Record } - : {}), isError, ...(STRUCTURED_TOOLS.has(request.params.name) && { structuredContent: structuredResult(request.params.name, result.metadata, result.output), From 46bd1a0663f4f9ed36b2059b7e3f1e536c4d6033 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:24:52 +0530 Subject: [PATCH 17/37] fix: an uncompiled project's own manifest path keeps the dbt integration's answer, and whole-row reads are possible reads - a manifest_path not written yet is placed by its path (nearest existing folder's links resolved): inside the adapter's project it keeps the adapter's lineage; elsewhere it is still another project's - a parent's row used as a value (to_json(o), cast(o as varchar)) or a NATURAL JOIN makes a model a possible reader, never a confident non-reader - downstream entries carry unique_id; a SQL path matches with ./ or backslashes - count_big(*)/count_if(*) are not a select *; a backtick output alias is not a read - include_graph's description says the graph is left out of MCP structured results Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 46 ++++++++++++++++++- .../src/altimate/native/dbt/lineage.ts | 4 +- .../src/altimate/tools/dbt-lineage.ts | 20 +++++--- .../src/altimate/tools/impact-analysis.ts | 2 +- .../test/altimate/column-reach.test.ts | 27 ++++++++++- .../altimate/dbt-lineage-structured.test.ts | 8 ++++ 6 files changed, 95 insertions(+), 12 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 648fba3e9e..a8bee43cfd 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -126,7 +126,7 @@ export function selectsStar(sql: string): boolean { const prev = before.at(-1) ?? "" const next = after[0] ?? "" // count(*) reads no column; hash(*), object_construct(*), struct(*) read them all. - if (prev === "(" && next === ")" && /\bcount\s*\($/i.test(before)) continue + if (prev === "(" && next === ")" && /\bcount(?:_big|_if)?\s*\($/i.test(before)) continue if (prev === "." || next === "" || next === "," || next === ")" || ends.test(after)) return true const operand = /[\w"'`\])]/ // A sign on the right operand (`revenue * -1`) belongs to it. @@ -151,7 +151,42 @@ function namesColumn(sql: string, column: string): boolean { const ident = new RegExp(`(^|[^A-Za-z0-9_$])("${name}"|${name})(?![A-Za-z0-9_$])`, "gi") for (const match of code.matchAll(ident)) { const before = code.slice(0, match.index! + match[1]!.length) - if (/\bas\s*$/i.test(before)) continue + if (/\bas\s*`?$/i.test(before)) continue + return true + } + return false +} + +const NOT_ALIAS = /^(on|using|where|join|left|right|full|inner|outer|cross|natural|group|order|limit|union|having|qualify|window|lateral|as|select|with|pivot|unpivot)$/i +const PART = String.raw`(?:[A-Za-z_][\w$]*|"[^"]+"|` + "`[^`]+`" + String.raw`|\[[^\]]+\])` +const RELATION = new RegExp(String.raw`\b(?:from|join)\s+(${PART}(?:\s*\.\s*${PART})*)(?:\s+(?:as\s+)?([A-Za-z_][\w$]*))?`, "gi") + +/** + * Whether the SQL reads whole rows of a parent, which the lineage engine traces to no column: a + * NATURAL JOIN, or a parent's table name or alias used as a value (`to_json(o)`, `cast(o as varchar)`). + * `tables` are the parents' bare table names. Used only to mark a model as one that may read the column. + */ +export function readsWholeRow(sql: string, tables: string[]): boolean { + const code = stripSql(sql) + if (/\bnatural\s+(?:(?:left|right|full|inner)\s+(?:outer\s+)?)?join\b/i.test(code)) return true + const names = new Set(tables.map((one) => one.toLowerCase())) + const rows = new Set() + // The FROM and JOIN clauses that name a parent: where it is named, not used. + const declared: [number, number][] = [] + for (const match of code.matchAll(RELATION)) { + const table = match[1]!.split(".").pop()!.trim().replace(/^["`[]|["`\]]$/g, "").toLowerCase() + if (!names.has(table)) continue + declared.push([match.index!, match.index! + match[0].length]) + rows.add(table) + if (match[2] && !NOT_ALIAS.test(match[2])) rows.add(match[2].toLowerCase()) + } + if (rows.size === 0) return false + // A bare name that is not qualified (`o.x`), not qualifying (`o.`), not an output alias and not a call. + for (const match of code.matchAll(/(? at >= from && at < to)) continue + const after = code.slice(at + match[1]!.length) + if (/^\s*[.(]/.test(after) || /\bas\s*$/i.test(code.slice(0, at))) continue return true } return false @@ -331,6 +366,13 @@ export function columnReach(manifestPath: string, model: string, column: string, unsure.add(id) continue } + // A whole row of a parent read as one value (`to_json(o)`, a NATURAL JOIN) carries the column with + // no edge the engine can report. + if (!fromUnsure && !ambiguous && named.length === 0 && readsWholeRow(sqlText, parents.map(bareName))) { + unsure.add(id) + uncertain.push(entry(id, "reads whole rows of a parent (a NATURAL JOIN, or the row as a value) the lineage engine cannot trace by column")) + continue + } const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) if (ambiguous || named.length > 0 || fromUnsure) { if (ambiguous || fromUnsure) unsure.add(id) diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index 5522cb9972..8efe6658d0 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -43,9 +43,11 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { const sources = manifest.sources || {} // A model by name or unique_id; a snapshot by unique_id; either by its project-relative file path. + // `./models/x.sql` and `models\\x.sql` are the manifest's `models/x.sql`. + const filePath = (p: unknown) => String(p ?? "").replace(/\\/g, "/").replace(/^(?:\.\/)+/, "") const byPath = () => Object.values(nodes).find( - (node) => ["model", "snapshot"].includes(node?.resource_type) && node.original_file_path === params.model, + (node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model), ) const modelNode = findModel(nodes, params.model) ?? diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index d78fb89865..b9d761f400 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -53,7 +53,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { include_graph: z .boolean() .optional() - .describe("Also return the model's graph from the manifest (everything upstream and downstream, up to 500 nodes each side; the exposures and tests are those of the model and its downstream, upstream tests are not included; graph.upTruncated / graph.truncated are set when a side is cut) in the metadata; for lineage viewers"), + .describe("Also return the model's graph from the manifest (everything upstream and downstream, up to 500 nodes each side; the exposures and tests are those of the model and its downstream, upstream tests are not included; graph.upTruncated / graph.truncated are set when a side is cut) in the metadata; for lineage viewers. Over MCP the graph is left out of the structured result: read it with `altimate tool run dbt_lineage --json`"), }), async execute(args, ctx) { let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort) @@ -126,22 +126,28 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { /** * Why a named manifest is not the adapter's project's, or undefined when it is: it must lie in that * project's folder (links resolved) with no other dbt_project.yml between (a nested project or an - * installed package is another project). A path that cannot be resolved counts as not this one's. + * installed package is another project). A manifest not written yet (an uncompiled project) is placed + * by its path, its nearest existing folder's links resolved, so the project's own default path keeps + * the adapter's answer. */ async function manifestOutsideAdapterProject(manifestPath: string): Promise { const { getDbtProjectRoot } = await import("../native/connections/register") const root = await getDbtProjectRoot() if (!root) return "the dbt adapter has no project root" const real = (p: string) => { - try { - return fs.realpathSync(path.resolve(p)) - } catch { - return undefined + const rest: string[] = [] + for (let at = path.resolve(p); ; at = path.dirname(at)) { + try { + return path.join(fs.realpathSync(at), ...rest) + } catch { + if (path.dirname(at) === at) return undefined + rest.unshift(path.basename(at)) + } } } const projectDir = real(root) const manifest = real(manifestPath) - if (!projectDir || !manifest) return `the manifest ${manifestPath} cannot be read` + if (!projectDir || !manifest) return `the manifest ${manifestPath} cannot be placed` const inside = path.relative(projectDir, manifest) // Outside the project: the relative path climbs out (`..` as a whole segment, so `..data/` is inside). if (!inside || inside === ".." || inside.startsWith(`..${path.sep}`) || path.isAbsolute(inside)) return `the manifest ${manifestPath} is not in the dbt project at ${root}` diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index 507ee87918..02b579d23a 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -142,7 +142,7 @@ export const ImpactAnalysisTool = Tool.define("impact_analysis", { model: targetModel.name, column: args.column ?? null, change_type: args.change_type, - downstream: downstream.map((d) => ({ name: d.name, depth: d.depth, materialized: d.materialized ?? null })), + downstream: downstream.map((d) => ({ name: d.name, ...(d.unique_id && { unique_id: d.unique_id }), depth: d.depth, materialized: d.materialized ?? null })), exposures, ...(reach && { column_reach: reach }), ...(reachError && { column_reach_error: reachError }), diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 1daac56afb..fc6c58779f 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -3,7 +3,7 @@ import * as core from "@altimateai/altimate-core" import { mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { columnReach, exposuresOn, parentsNamed, selectsStar } from "../../src/altimate/native/dbt/column-reach" +import { columnReach, exposuresOn, parentsNamed, readsWholeRow, selectsStar } from "../../src/altimate/native/dbt/column-reach" import { lineageGraph } from "../../src/altimate/native/dbt/graph" import { structuredError, structuredResult } from "../../src/altimate/mcp-server/server" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" @@ -580,6 +580,9 @@ describe("dbtLineage", () => { expect(Object.keys(dbtLineage({ manifest_path: file, model: "mart_ltv", include_graph: true }).graph!.nodes)).toContain("model.shop.rpt_ltv") // By project-relative path too. expect(dbtLineage({ manifest_path: file, model: "models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + // `./` and Windows separators spell the same path. + expect(dbtLineage({ manifest_path: file, model: "./models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + expect(dbtLineage({ manifest_path: file, model: "models\\mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") const withSnapshot: any = manifest() withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" } expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap") @@ -589,3 +592,25 @@ describe("dbtLineage", () => { expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "test.shop.t1" }).model_unique_id).toBe("snapshot.shop.odd") }) }) + +describe("columnReach with whole-row reads", () => { + test("a parent's row used as a value, or a NATURAL JOIN, is a possible read, never a confident non-reader", () => { + for (const sql of [ + "select order_id, to_json(o) as j from fct_orders o", + "select order_id, md5(cast(o as varchar)) as h from fct_orders o", + "select a.order_id from fct_orders a natural join other b", + ]) { + const value = manifest() + value.nodes["model.shop.rpt_counts"].compiled_code = sql + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name).sort()).toEqual(["rpt_big_orders", "rpt_counts"]) + expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(["ops"]) + } + }) + + test("qualified columns and the FROM clause itself are not whole-row reads", () => { + expect(readsWholeRow('select o.order_id, o.revenue as fct_orders from "shop"."main"."fct_orders" o where o.x > 1', ["fct_orders"])).toBe(false) + expect(readsWholeRow("select fct_orders.order_id from fct_orders join x on fct_orders.id = x.id", ["fct_orders"])).toBe(false) + expect(readsWholeRow("select count(*) from fct_orders o left join other on o.id = other.id", ["fct_orders"])).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index d1c716570e..a2da3f4185 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -191,6 +191,14 @@ describe("dbt_lineage with the project's dbt adapter", () => { await writeFile(path.join(sibling, "target", "manifest.json"), empty) const next = await tool.execute({ model: "customers", manifest_path: path.join(sibling, "target", "manifest.json") }, ctx) expect(next.metadata.structuredContent).toBeUndefined() + // An uncompiled project's own default manifest, not written yet, keeps the adapter's answer. + const uncompiled = path.join(mine, "fresh", "target", "manifest.json") + const unwritten = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "nope", "manifest.json") }, ctx) + expect(unwritten.metadata.structuredContent).toEqual(TRACE) + expect((await tool.execute({ model: "customers", manifest_path: uncompiled }, ctx)).metadata.structuredContent).toEqual(TRACE) + // One not written yet in another folder is still another project's. + const elsewhere = await tool.execute({ model: "customers", manifest_path: path.join(other, "missing", "manifest.json") }, ctx) + expect(elsewhere.metadata.structuredContent).toBeUndefined() } finally { await rm(dir, { recursive: true, force: true }) } From 195435a2a87e9b926d61685c5b79827e70e7097a Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:32:41 +0530 Subject: [PATCH 18/37] fix: a whole row is a row source passed whole to a call, so import CTEs stay confident and quoted, CTE and subquery rows are caught - row sources: FROM/JOIN relations and aliases (quoted or not, never a keyword), CTE names and subquery aliases - a whole row read beside the column passes the doubt on to the model's children - a missing manifest with no dbt project above it names no other project; a link to a missing manifest cannot be placed Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 58 +++++++++---------- .../src/altimate/tools/dbt-lineage.ts | 18 +++++- .../test/altimate/column-reach.test.ts | 35 +++++++++-- .../altimate/dbt-lineage-structured.test.ts | 13 ++++- 4 files changed, 85 insertions(+), 39 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index a8bee43cfd..283ff0ab89 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -157,38 +157,34 @@ function namesColumn(sql: string, column: string): boolean { return false } -const NOT_ALIAS = /^(on|using|where|join|left|right|full|inner|outer|cross|natural|group|order|limit|union|having|qualify|window|lateral|as|select|with|pivot|unpivot)$/i -const PART = String.raw`(?:[A-Za-z_][\w$]*|"[^"]+"|` + "`[^`]+`" + String.raw`|\[[^\]]+\])` -const RELATION = new RegExp(String.raw`\b(?:from|join)\s+(${PART}(?:\s*\.\s*${PART})*)(?:\s+(?:as\s+)?([A-Za-z_][\w$]*))?`, "gi") +const KEYWORD = String.raw`(?:on|using|where|join|left|right|full|inner|outer|cross|natural|semi|anti|group|order|limit|union|except|intersect|minus|having|qualify|window|lateral|as|select|with|pivot|unpivot|tablesample|for|from)\b` +const NAME = String.raw`(?:[A-Za-z_][\w$]*|"[^"]+"|` + "`[^`]+`" + String.raw`|\[[^\]]+\])` +const unquote = (name: string) => name.replace(/^["`[]|["`\]]$/g, "").toLowerCase() /** - * Whether the SQL reads whole rows of a parent, which the lineage engine traces to no column: a - * NATURAL JOIN, or a parent's table name or alias used as a value (`to_json(o)`, `cast(o as varchar)`). - * `tables` are the parents' bare table names. Used only to mark a model as one that may read the column. + * Whether the SQL reads whole rows, which the lineage engine traces to no column: a NATURAL JOIN, or + * a row source (a table, its alias, a CTE or a subquery alias; quoted or not) passed whole to a call + * (`to_json(o)`, `md5(cast(o as varchar))`). Used only to mark a model as one that may read the column; + * a row source named in a select list by itself is not caught. */ -export function readsWholeRow(sql: string, tables: string[]): boolean { +export function readsWholeRow(sql: string): boolean { const code = stripSql(sql) if (/\bnatural\s+(?:(?:left|right|full|inner)\s+(?:outer\s+)?)?join\b/i.test(code)) return true - const names = new Set(tables.map((one) => one.toLowerCase())) - const rows = new Set() - // The FROM and JOIN clauses that name a parent: where it is named, not used. - const declared: [number, number][] = [] - for (const match of code.matchAll(RELATION)) { - const table = match[1]!.split(".").pop()!.trim().replace(/^["`[]|["`\]]$/g, "").toLowerCase() - if (!names.has(table)) continue - declared.push([match.index!, match.index! + match[0].length]) - rows.add(table) - if (match[2] && !NOT_ALIAS.test(match[2])) rows.add(match[2].toLowerCase()) - } - if (rows.size === 0) return false - // A bare name that is not qualified (`o.x`), not qualifying (`o.`), not an output alias and not a call. - for (const match of code.matchAll(/(? at >= from && at < to)) continue - const after = code.slice(at + match[1]!.length) - if (/^\s*[.(]/.test(after) || /\bas\s*$/i.test(code.slice(0, at))) continue - return true + const sources = new Set() + // FROM/JOIN relations (their last part) and aliases; an alias is never a keyword, so `join` that + // follows a relation is left for the next match. + const relation = new RegExp(String.raw`\b(?:from|join)\s+(${NAME}(?:\s*\.\s*${NAME})*)(?:\s+(?:as\s+)?(?!${KEYWORD})(${NAME}))?`, "gi") + for (const match of code.matchAll(relation)) { + sources.add(unquote(match[1]!.split(".").pop()!.trim())) + if (match[2]) sources.add(unquote(match[2])) } + // CTE names (`name as (`) and subquery aliases (`) name`, `) as name`). + for (const match of code.matchAll(new RegExp(String.raw`(${NAME})\s+as\s+(?:not\s+)?(?:materialized\s+)?\(`, "gi"))) sources.add(unquote(match[1]!)) + for (const match of code.matchAll(new RegExp(String.raw`\)\s*(?:as\s+)?(?!${KEYWORD})(${NAME})`, "gi"))) sources.add(unquote(match[1]!)) + // A row source alone as a call's argument: after `(` or `,`, before `)`, `,` or a cast's `as `. + // A CTE defined after a comma (`, b as (`, `, b as materialized (`) is not an argument. + const argument = new RegExp(String.raw`[(,]\s*(${NAME})\s*(?=\)|,|as\s+(?!(?:not\s+)?materialized\b)[A-Za-z])`, "gi") + for (const match of code.matchAll(argument)) if (sources.has(unquote(match[1]!))) return true return false } @@ -338,11 +334,13 @@ export function columnReach(manifestPath: string, model: string, column: string, // The engine has no schema for the tables inside an inlined ephemeral parent, so it reports no // edge through it: unknown, not unread, even when another parent's edges are traced. const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") + const wholeRow = readsWholeRow(sqlText) if (reads) { tracked.set(id, carried) trackedExact.set(id, carriedExact) // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. - if (ambiguous || fromUnsure || viaEphemeral || partial) unsure.add(id) + // A whole row read beside the column (`select revenue, to_json(o)`) may carry it on in another output. + if (ambiguous || fromUnsure || viaEphemeral || partial || wholeRow) unsure.add(id) reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) continue } @@ -366,9 +364,9 @@ export function columnReach(manifestPath: string, model: string, column: string, unsure.add(id) continue } - // A whole row of a parent read as one value (`to_json(o)`, a NATURAL JOIN) carries the column with - // no edge the engine can report. - if (!fromUnsure && !ambiguous && named.length === 0 && readsWholeRow(sqlText, parents.map(bareName))) { + // A whole row read as one value (`to_json(o)`, a NATURAL JOIN) carries the column with no edge the + // engine can report. + if (!fromUnsure && !ambiguous && named.length === 0 && wholeRow) { unsure.add(id) uncertain.push(entry(id, "reads whole rows of a parent (a NATURAL JOIN, or the row as a value) the lineage engine cannot trace by column")) continue diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index b9d761f400..3a0e59aa38 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -128,7 +128,8 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { * project's folder (links resolved) with no other dbt_project.yml between (a nested project or an * installed package is another project). A manifest not written yet (an uncompiled project) is placed * by its path, its nearest existing folder's links resolved, so the project's own default path keeps - * the adapter's answer. + * the adapter's answer; one with no dbt project above it at all (a default path resolved against + * another folder) names no other project either. A link whose target is missing cannot be placed. */ async function manifestOutsideAdapterProject(manifestPath: string): Promise { const { getDbtProjectRoot } = await import("../native/connections/register") @@ -140,7 +141,7 @@ async function manifestOutsideAdapterProject(manifestPath: string): Promise; error?: string } { try { diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index fc6c58779f..da2c5bcff9 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -608,9 +608,36 @@ describe("columnReach with whole-row reads", () => { } }) - test("qualified columns and the FROM clause itself are not whole-row reads", () => { - expect(readsWholeRow('select o.order_id, o.revenue as fct_orders from "shop"."main"."fct_orders" o where o.x > 1', ["fct_orders"])).toBe(false) - expect(readsWholeRow("select fct_orders.order_id from fct_orders join x on fct_orders.id = x.id", ["fct_orders"])).toBe(false) - expect(readsWholeRow("select count(*) from fct_orders o left join other on o.id = other.id", ["fct_orders"])).toBe(false) + test("a row through a quoted alias, a CTE, a subquery or after another join is a whole-row read", () => { + for (const sql of [ + 'select order_id, to_json("o") as j from fct_orders as "o"', + "select to_json(o) from other join fct_orders o on other.id = o.order_id", + "with c as (select * from fct_orders) select to_json(c) as j from c", + "select to_json(s) as j from (select * from fct_orders) s", + ]) + expect(readsWholeRow(sql)).toBe(true) + }) + + test("import CTEs, aliases and qualified columns are not whole-row reads", () => { + for (const sql of [ + 'with fct_orders as (select * from "shop"."main"."fct_orders") select order_id from fct_orders', + "with a as (select 1 as x), fct_orders as (select * from fct_orders) select order_id from fct_orders", + "with a as (select 1 as x), fct_orders as materialized (select * from fct_orders) select order_id from fct_orders", + "select n from (select count(*) as n from fct_orders) fct_orders", + "select count(*) fct_orders from fct_orders", + 'select o.order_id, o.revenue as fct_orders from "shop"."main"."fct_orders" o where o.x > 1', + "select fct_orders.order_id from fct_orders join x on fct_orders.id = x.id", + "select coalesce(o.a, o.b), cast(o.c as int) from fct_orders o left join other using (id)", + ]) + expect(readsWholeRow(sql)).toBe(false) + }) + + test("a model that reads the column and a whole row passes the doubt on to its children", () => { + const value = manifest() + value.nodes["model.shop.fct_daily"].compiled_code = "select sum(revenue) as revenue, to_json(o) as j from fct_orders o" + value.nodes["model.shop.rpt_json"] = model("rpt_json", ["model.shop.fct_daily"], "select j from fct_daily", ["j"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.models.map((m) => m.name)).toContain("fct_daily") + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_json") }) }) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index a2da3f4185..c2b7afe632 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -4,7 +4,7 @@ // adapter is up, and the manifest's lineage otherwise. The dbt adapter is replaced // through `dbtAdapterInternals`, so these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test" -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { Effect } from "effect" @@ -196,9 +196,18 @@ describe("dbt_lineage with the project's dbt adapter", () => { const unwritten = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "nope", "manifest.json") }, ctx) expect(unwritten.metadata.structuredContent).toEqual(TRACE) expect((await tool.execute({ model: "customers", manifest_path: uncompiled }, ctx)).metadata.structuredContent).toEqual(TRACE) - // One not written yet in another folder is still another project's. + // One not written yet in a folder no dbt project is above (a default path resolved against + // another folder) names no other project: the adapter answers. + const stray = await tool.execute({ model: "customers", manifest_path: path.join(other, "missing", "manifest.json") }, ctx) + expect(stray.metadata.structuredContent).toEqual(TRACE) + // One not written yet in another dbt project is that project's. + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") const elsewhere = await tool.execute({ model: "customers", manifest_path: path.join(other, "missing", "manifest.json") }, ctx) expect(elsewhere.metadata.structuredContent).toBeUndefined() + // A link to a manifest that is missing cannot be placed, wherever the link sits. + await symlink(path.join(other, "target", "gone.json"), path.join(mine, "target", "linked.json")) + const linked = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "linked.json") }, ctx) + expect(linked.metadata.structuredContent).toBeUndefined() } finally { await rm(dir, { recursive: true, force: true }) } From 68689f23dbb936692c1c359a9a2e6c78ece1c70b Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:38:45 +0530 Subject: [PATCH 19/37] fix: column reach flags a NATURAL JOIN only, and a manifest link is placed where it points A pattern cannot tell a row source from a column (sum(x) as x, extract(year from d)), so whole-row values are left to the lineage engine; a NATURAL JOIN is one keyword with no such ambiguity. A link to a manifest not written yet is followed, and a path through a file is placed instead of throwing. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 45 ++++---------- .../src/altimate/tools/dbt-lineage.ts | 23 ++++++-- .../test/altimate/column-reach.test.ts | 59 ++++++------------- .../altimate/dbt-lineage-structured.test.ts | 9 ++- 4 files changed, 54 insertions(+), 82 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 283ff0ab89..5f8c766361 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -157,35 +157,13 @@ function namesColumn(sql: string, column: string): boolean { return false } -const KEYWORD = String.raw`(?:on|using|where|join|left|right|full|inner|outer|cross|natural|semi|anti|group|order|limit|union|except|intersect|minus|having|qualify|window|lateral|as|select|with|pivot|unpivot|tablesample|for|from)\b` -const NAME = String.raw`(?:[A-Za-z_][\w$]*|"[^"]+"|` + "`[^`]+`" + String.raw`|\[[^\]]+\])` -const unquote = (name: string) => name.replace(/^["`[]|["`\]]$/g, "").toLowerCase() - /** - * Whether the SQL reads whole rows, which the lineage engine traces to no column: a NATURAL JOIN, or - * a row source (a table, its alias, a CTE or a subquery alias; quoted or not) passed whole to a call - * (`to_json(o)`, `md5(cast(o as varchar))`). Used only to mark a model as one that may read the column; - * a row source named in a select list by itself is not caught. + * Whether the SQL has a NATURAL JOIN, whose join columns the lineage engine reports no edge for. + * Limit: a parent's whole row passed as a value (`to_json(o)`) is not seen either; telling a row + * source from a column needs the engine's parse, not a pattern. */ -export function readsWholeRow(sql: string): boolean { - const code = stripSql(sql) - if (/\bnatural\s+(?:(?:left|right|full|inner)\s+(?:outer\s+)?)?join\b/i.test(code)) return true - const sources = new Set() - // FROM/JOIN relations (their last part) and aliases; an alias is never a keyword, so `join` that - // follows a relation is left for the next match. - const relation = new RegExp(String.raw`\b(?:from|join)\s+(${NAME}(?:\s*\.\s*${NAME})*)(?:\s+(?:as\s+)?(?!${KEYWORD})(${NAME}))?`, "gi") - for (const match of code.matchAll(relation)) { - sources.add(unquote(match[1]!.split(".").pop()!.trim())) - if (match[2]) sources.add(unquote(match[2])) - } - // CTE names (`name as (`) and subquery aliases (`) name`, `) as name`). - for (const match of code.matchAll(new RegExp(String.raw`(${NAME})\s+as\s+(?:not\s+)?(?:materialized\s+)?\(`, "gi"))) sources.add(unquote(match[1]!)) - for (const match of code.matchAll(new RegExp(String.raw`\)\s*(?:as\s+)?(?!${KEYWORD})(${NAME})`, "gi"))) sources.add(unquote(match[1]!)) - // A row source alone as a call's argument: after `(` or `,`, before `)`, `,` or a cast's `as `. - // A CTE defined after a comma (`, b as (`, `, b as materialized (`) is not an argument. - const argument = new RegExp(String.raw`[(,]\s*(${NAME})\s*(?=\)|,|as\s+(?!(?:not\s+)?materialized\b)[A-Za-z])`, "gi") - for (const match of code.matchAll(argument)) if (sources.has(unquote(match[1]!))) return true - return false +export function joinsNaturally(sql: string): boolean { + return /\bnatural\s+(?:(?:left|right|full|inner)\s+(?:outer\s+)?)?join\b/i.test(stripSql(sql)) } function childMapOf(manifest: any): Record { @@ -334,13 +312,13 @@ export function columnReach(manifestPath: string, model: string, column: string, // The engine has no schema for the tables inside an inlined ephemeral parent, so it reports no // edge through it: unknown, not unread, even when another parent's edges are traced. const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") - const wholeRow = readsWholeRow(sqlText) + const natural = joinsNaturally(sqlText) if (reads) { tracked.set(id, carried) trackedExact.set(id, carriedExact) // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. - // A whole row read beside the column (`select revenue, to_json(o)`) may carry it on in another output. - if (ambiguous || fromUnsure || viaEphemeral || partial || wholeRow) unsure.add(id) + // A NATURAL JOIN beside the column may join on it, or carry it on in another output. + if (ambiguous || fromUnsure || viaEphemeral || partial || natural) unsure.add(id) reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) continue } @@ -364,11 +342,10 @@ export function columnReach(manifestPath: string, model: string, column: string, unsure.add(id) continue } - // A whole row read as one value (`to_json(o)`, a NATURAL JOIN) carries the column with no edge the - // engine can report. - if (!fromUnsure && !ambiguous && named.length === 0 && wholeRow) { + // A NATURAL JOIN may join on the column with no edge the engine can report. + if (!fromUnsure && !ambiguous && named.length === 0 && natural) { unsure.add(id) - uncertain.push(entry(id, "reads whole rows of a parent (a NATURAL JOIN, or the row as a value) the lineage engine cannot trace by column")) + uncertain.push(entry(id, "uses a NATURAL JOIN, whose join columns the lineage engine cannot trace")) continue } const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 3a0e59aa38..5f7ea271ae 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -129,22 +129,35 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { * installed package is another project). A manifest not written yet (an uncompiled project) is placed * by its path, its nearest existing folder's links resolved, so the project's own default path keeps * the adapter's answer; one with no dbt project above it at all (a default path resolved against - * another folder) names no other project either. A link whose target is missing cannot be placed. + * another folder) names no other project either. A link to a manifest not written yet is placed where + * it points. */ async function manifestOutsideAdapterProject(manifestPath: string): Promise { const { getDbtProjectRoot } = await import("../native/connections/register") const root = await getDbtProjectRoot() if (!root) return "the dbt adapter has no project root" + // The path with its links resolved, the parts not written yet appended; a link to a missing file is + // followed to where it points. Undefined past 64 links (a loop). const real = (p: string) => { const rest: string[] = [] - for (let at = path.resolve(p); ; at = path.dirname(at)) { + let at = path.resolve(p) + for (let step = 0; step < 64; step++) { try { return path.join(fs.realpathSync(at), ...rest) - } catch { - if (path.dirname(at) === at || fs.lstatSync(at, { throwIfNoEntry: false })?.isSymbolicLink()) return undefined - rest.unshift(path.basename(at)) + } catch {} + let link: string | undefined + try { + if (fs.lstatSync(at).isSymbolicLink()) link = fs.readlinkSync(at) + } catch {} + if (link !== undefined) { + at = path.resolve(path.dirname(at), link) + continue } + if (path.dirname(at) === at) return undefined + rest.unshift(path.basename(at)) + at = path.dirname(at) } + return undefined } const projectDir = real(root) const manifest = real(manifestPath) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index da2c5bcff9..4ad21043b6 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -3,7 +3,7 @@ import * as core from "@altimateai/altimate-core" import { mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { columnReach, exposuresOn, parentsNamed, readsWholeRow, selectsStar } from "../../src/altimate/native/dbt/column-reach" +import { columnReach, exposuresOn, parentsNamed, joinsNaturally, selectsStar } from "../../src/altimate/native/dbt/column-reach" import { lineageGraph } from "../../src/altimate/native/dbt/graph" import { structuredError, structuredResult } from "../../src/altimate/mcp-server/server" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" @@ -593,51 +593,26 @@ describe("dbtLineage", () => { }) }) -describe("columnReach with whole-row reads", () => { - test("a parent's row used as a value, or a NATURAL JOIN, is a possible read, never a confident non-reader", () => { - for (const sql of [ - "select order_id, to_json(o) as j from fct_orders o", - "select order_id, md5(cast(o as varchar)) as h from fct_orders o", - "select a.order_id from fct_orders a natural join other b", - ]) { - const value = manifest() - value.nodes["model.shop.rpt_counts"].compiled_code = sql - const reach = columnReach(manifestFile(value), "fct_orders", "revenue") - expect(reach.uncertain.map((m) => m.name).sort()).toEqual(["rpt_big_orders", "rpt_counts"]) - expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(["ops"]) - } - }) - - test("a row through a quoted alias, a CTE, a subquery or after another join is a whole-row read", () => { - for (const sql of [ - 'select order_id, to_json("o") as j from fct_orders as "o"', - "select to_json(o) from other join fct_orders o on other.id = o.order_id", - "with c as (select * from fct_orders) select to_json(c) as j from c", - "select to_json(s) as j from (select * from fct_orders) s", - ]) - expect(readsWholeRow(sql)).toBe(true) - }) - - test("import CTEs, aliases and qualified columns are not whole-row reads", () => { - for (const sql of [ - 'with fct_orders as (select * from "shop"."main"."fct_orders") select order_id from fct_orders', - "with a as (select 1 as x), fct_orders as (select * from fct_orders) select order_id from fct_orders", - "with a as (select 1 as x), fct_orders as materialized (select * from fct_orders) select order_id from fct_orders", - "select n from (select count(*) as n from fct_orders) fct_orders", - "select count(*) fct_orders from fct_orders", - 'select o.order_id, o.revenue as fct_orders from "shop"."main"."fct_orders" o where o.x > 1', - "select fct_orders.order_id from fct_orders join x on fct_orders.id = x.id", - "select coalesce(o.a, o.b), cast(o.c as int) from fct_orders o left join other using (id)", - ]) - expect(readsWholeRow(sql)).toBe(false) +describe("columnReach with a NATURAL JOIN", () => { + test("a NATURAL JOIN is a possible read, never a confident non-reader", () => { + const value = manifest() + value.nodes["model.shop.rpt_counts"].compiled_code = "select a.order_id from fct_orders a natural join other b" + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name).sort()).toEqual(["rpt_big_orders", "rpt_counts"]) + expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(["ops"]) }) - test("a model that reads the column and a whole row passes the doubt on to its children", () => { + test("a model that reads the column through a NATURAL JOIN passes the doubt on to its children", () => { const value = manifest() - value.nodes["model.shop.fct_daily"].compiled_code = "select sum(revenue) as revenue, to_json(o) as j from fct_orders o" - value.nodes["model.shop.rpt_json"] = model("rpt_json", ["model.shop.fct_daily"], "select j from fct_daily", ["j"]) + value.nodes["model.shop.fct_daily"].compiled_code = "select sum(revenue) as revenue from fct_orders natural left join other" + value.nodes["model.shop.rpt_after"] = model("rpt_after", ["model.shop.fct_daily"], "select count(*) as n from fct_daily", ["n"]) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.models.map((m) => m.name)).toContain("fct_daily") - expect(reach.uncertain.map((m) => m.name)).toContain("rpt_json") + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") + }) + + test("aggregates and casts under their own names, and the word natural elsewhere, are not a NATURAL JOIN", () => { + for (const sql of ["select sum(revenue) as revenue, cast(d as date) as d from fct_orders", "select natural_key, 'natural join' as s from fct_orders -- natural join"]) + expect(joinsNaturally(sql)).toBe(false) }) }) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index c2b7afe632..dd0a0a0c06 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -204,10 +204,17 @@ describe("dbt_lineage with the project's dbt adapter", () => { await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") const elsewhere = await tool.execute({ model: "customers", manifest_path: path.join(other, "missing", "manifest.json") }, ctx) expect(elsewhere.metadata.structuredContent).toBeUndefined() - // A link to a manifest that is missing cannot be placed, wherever the link sits. + // A link to a manifest not written yet is placed where it points: in another project, that one's; + // in this one, the adapter answers. await symlink(path.join(other, "target", "gone.json"), path.join(mine, "target", "linked.json")) const linked = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "linked.json") }, ctx) expect(linked.metadata.structuredContent).toBeUndefined() + await symlink(path.join(mine, "target", "real.json"), path.join(mine, "target", "own.json")) + const own2 = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "own.json") }, ctx) + expect(own2.metadata.structuredContent).toEqual(TRACE) + // A path through a file is not a folder: it is placed, not thrown on. + const through = await tool.execute({ model: "customers", manifest_path: path.join(mine, "dbt_project.yml", "manifest.json") }, ctx) + expect(through.metadata.structuredContent).toEqual(TRACE) } finally { await rm(dir, { recursive: true, force: true }) } From 29b76e77028ec5ce24e93ee452b5c470f3ebe315 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:42:59 +0530 Subject: [PATCH 20/37] fix: every NATURAL join form counts and passes its doubt on, quoted names never do, and a relative link climbs from its real folder Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 6 ++++-- .../opencode/src/altimate/tools/dbt-lineage.ts | 3 ++- .../test/altimate/column-reach.test.ts | 18 ++++++++++++++++-- .../altimate/dbt-lineage-structured.test.ts | 9 +++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 5f8c766361..a79155f905 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -163,7 +163,9 @@ function namesColumn(sql: string, column: string): boolean { * source from a column needs the engine's parse, not a pattern. */ export function joinsNaturally(sql: string): boolean { - return /\bnatural\s+(?:(?:left|right|full|inner)\s+(?:outer\s+)?)?join\b/i.test(stripSql(sql)) + // Quoted names are emptied first: `as "natural join"` names a column. + const code = stripSql(sql).replace(/"(?:[^"]|"")*"|`[^`]*`|\[[^\]]*\]/g, '""') + return /\bnatural\s+(?:(?:left|right|full|inner|cross|semi|anti)\s+){0,2}(?:outer\s+)?join\b/i.test(code) } function childMapOf(manifest: any): Record { @@ -350,7 +352,7 @@ export function columnReach(manifestPath: string, model: string, column: string, } const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) if (ambiguous || named.length > 0 || fromUnsure) { - if (ambiguous || fromUnsure) unsure.add(id) + if (ambiguous || fromUnsure || natural) unsure.add(id) uncertain.push( entry( id, diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 5f7ea271ae..4f316c00f0 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -150,7 +150,8 @@ async function manifestOutsideAdapterProject(manifestPath: string): Promise { expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") }) - test("aggregates and casts under their own names, and the word natural elsewhere, are not a NATURAL JOIN", () => { - for (const sql of ["select sum(revenue) as revenue, cast(d as date) as d from fct_orders", "select natural_key, 'natural join' as s from fct_orders -- natural join"]) + test("every NATURAL join form counts; aggregates under their own names and the words in names, strings or comments do not", () => { + for (const sql of [ + "select sum(revenue) as revenue, cast(d as date) as d from fct_orders", + "select natural_key, 'natural join' as s from fct_orders -- natural join", + 'select a.id as "natural join", b.x as `natural join` from fct_orders a', + ]) expect(joinsNaturally(sql)).toBe(false) + for (const sql of ["select a.id from fct_orders a natural left semi join other b", "select * from a natural cross join b", "select * from a natural full outer join b"]) + expect(joinsNaturally(sql)).toBe(true) + }) + + test("a NATURAL JOIN model that names the column outside its select list also passes the doubt on", () => { + const value = manifest() + value.nodes["model.shop.rpt_big_orders"].compiled_code = "select a.order_id from fct_orders a natural join other b where a.revenue > 100" + value.nodes["model.shop.rpt_after"] = model("rpt_after", ["model.shop.rpt_big_orders"], "select count(*) as n from rpt_big_orders", ["n"]) + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") }) }) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index dd0a0a0c06..36ad1b6a2f 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -212,6 +212,15 @@ describe("dbt_lineage with the project's dbt adapter", () => { await symlink(path.join(mine, "target", "real.json"), path.join(mine, "target", "own.json")) const own2 = await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "own.json") }, ctx) expect(own2.metadata.structuredContent).toEqual(TRACE) + // A relative link climbs from its real folder: through a linked folder, `..` leaves this project. + const realOther = path.join(dir, "real") + await mkdir(path.join(realOther, "proj", "target"), { recursive: true }) + await mkdir(path.join(realOther, "otherproj"), { recursive: true }) + await writeFile(path.join(realOther, "otherproj", "dbt_project.yml"), "name: o\n") + await symlink(path.join(realOther, "proj", "target"), path.join(mine, "linked_target")) + await symlink("../../otherproj/target/manifest.json", path.join(realOther, "proj", "target", "manifest.json")) + const climbed = await tool.execute({ model: "customers", manifest_path: path.join(mine, "linked_target", "manifest.json") }, ctx) + expect(climbed.metadata.structuredContent).toBeUndefined() // A path through a file is not a folder: it is placed, not thrown on. const through = await tool.execute({ model: "customers", manifest_path: path.join(mine, "dbt_project.yml", "manifest.json") }, ctx) expect(through.metadata.structuredContent).toEqual(TRACE) From 46dc68a98ffe8cf1f68590c2d2902f8dda393bab Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:45:47 +0530 Subject: [PATCH 21/37] fix: a NATURAL JOIN inside array brackets or after an escaped quote still counts Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/column-reach.ts | 5 +++-- packages/opencode/test/altimate/column-reach.test.ts | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index a79155f905..1c09c14a9a 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -163,8 +163,9 @@ function namesColumn(sql: string, column: string): boolean { * source from a column needs the engine's parse, not a pattern. */ export function joinsNaturally(sql: string): boolean { - // Quoted names are emptied first: `as "natural join"` names a column. - const code = stripSql(sql).replace(/"(?:[^"]|"")*"|`[^`]*`|\[[^\]]*\]/g, '""') + // Double-quoted and backticked text is emptied first: `as "natural join"` names a column. Brackets + // are left alone, since they also hold array expressions (`array[(select …)]`). + const code = stripSql(sql).replace(/"(?:[^"\\]|\\.|"")*"|`[^`]*`/g, '""') return /\bnatural\s+(?:(?:left|right|full|inner|cross|semi|anti)\s+){0,2}(?:outer\s+)?join\b/i.test(code) } diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 5e48212e73..6565035ac0 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -618,7 +618,13 @@ describe("columnReach with a NATURAL JOIN", () => { 'select a.id as "natural join", b.x as `natural join` from fct_orders a', ]) expect(joinsNaturally(sql)).toBe(false) - for (const sql of ["select a.id from fct_orders a natural left semi join other b", "select * from a natural cross join b", "select * from a natural full outer join b"]) + for (const sql of [ + "select a.id from fct_orders a natural left semi join other b", + "select * from a natural cross join b", + "select * from a natural full outer join b", + "select array[(select max(a.id) from fct_orders a natural join other b)] as ids", + 'select "a\\"b" as c from a natural join b where t = "z"', + ]) expect(joinsNaturally(sql)).toBe(true) }) From ce71d5a6d7583ce23d45b542d77e65f65af1c541 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:48:08 +0530 Subject: [PATCH 22/37] fix: a NATURAL JOIN is found in the raw SQL, so no dialect's quoting can hide one Masking strings and quoted names to avoid a false positive let quoting rules that differ by dialect hide real joins. The words inside a string or comment now count too, which only adds doubt. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 15 ++++++++------- .../opencode/test/altimate/column-reach.test.ts | 14 ++++++++------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 1c09c14a9a..d0bdfae8ce 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -158,15 +158,16 @@ function namesColumn(sql: string, column: string): boolean { } /** - * Whether the SQL has a NATURAL JOIN, whose join columns the lineage engine reports no edge for. - * Limit: a parent's whole row passed as a value (`to_json(o)`) is not seen either; telling a row - * source from a column needs the engine's parse, not a pattern. + * Whether the SQL may have a NATURAL JOIN, whose join columns the lineage engine reports no edge for. + * Read from the raw SQL: the words inside a string, comment or quoted name count too, which only adds + * doubt, while masking quotes would let a dialect's quoting rules hide a real join. + * Limit: a parent's whole row passed as a value (`to_json(o)`) is not seen; telling a row source from + * a column needs the engine's parse, not a pattern. */ export function joinsNaturally(sql: string): boolean { - // Double-quoted and backticked text is emptied first: `as "natural join"` names a column. Brackets - // are left alone, since they also hold array expressions (`array[(select …)]`). - const code = stripSql(sql).replace(/"(?:[^"\\]|\\.|"")*"|`[^`]*`/g, '""') - return /\bnatural\s+(?:(?:left|right|full|inner|cross|semi|anti)\s+){0,2}(?:outer\s+)?join\b/i.test(code) + const natural = /\bnatural\s+(?:(?:left|right|full|inner|cross|semi|anti)\s+){0,2}(?:outer\s+)?join\b/i + // Also with block comments blanked, so `natural /* … */ join` counts: either match only adds doubt. + return natural.test(sql) || natural.test(sql.replace(/\/\*[\s\S]*?\*\//g, " ")) } function childMapOf(manifest: any): Record { diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 6565035ac0..25c0ff0996 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -611,12 +611,8 @@ describe("columnReach with a NATURAL JOIN", () => { expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") }) - test("every NATURAL join form counts; aggregates under their own names and the words in names, strings or comments do not", () => { - for (const sql of [ - "select sum(revenue) as revenue, cast(d as date) as d from fct_orders", - "select natural_key, 'natural join' as s from fct_orders -- natural join", - 'select a.id as "natural join", b.x as `natural join` from fct_orders a', - ]) + test("every NATURAL join form counts, whatever the quoting around it; aggregates under their own names do not", () => { + for (const sql of ["select sum(revenue) as revenue, cast(d as date) as d from fct_orders", "select natural_key, naturally from fct_orders"]) expect(joinsNaturally(sql)).toBe(false) for (const sql of [ "select a.id from fct_orders a natural left semi join other b", @@ -624,6 +620,12 @@ describe("columnReach with a NATURAL JOIN", () => { "select * from a natural full outer join b", "select array[(select max(a.id) from fct_orders a natural join other b)] as ids", 'select "a\\"b" as c from a natural join b where t = "z"', + `select "it's" as c from a natural join b where y = 'q'`, + "select 'a\\'b' as c from a natural join b where t = 'z'", + 'select "C:\\" as p from a natural join b where t = "z"', + "select * from a natural /* inner */ join b", + // The words in a string or comment count too: that only adds doubt. + "select 'natural join' as s from fct_orders", ]) expect(joinsNaturally(sql)).toBe(true) }) From a4536a89d2e51b95d3e5b4148fac4347eac3a2a2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:51:30 +0530 Subject: [PATCH 23/37] fix: comments between NATURAL and JOIN are read as gaps, with no pass over the whole SQL Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/native/dbt/column-reach.ts | 11 +++++++---- packages/opencode/test/altimate/column-reach.test.ts | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index d0bdfae8ce..072a942988 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -160,14 +160,17 @@ function namesColumn(sql: string, column: string): boolean { /** * Whether the SQL may have a NATURAL JOIN, whose join columns the lineage engine reports no edge for. * Read from the raw SQL: the words inside a string, comment or quoted name count too, which only adds - * doubt, while masking quotes would let a dialect's quoting rules hide a real join. + * doubt (a comment mentioning one marks the model uncertain), while masking quotes would let a dialect's + * quoting rules hide a real join. Comments between the keywords are read as gaps. * Limit: a parent's whole row passed as a value (`to_json(o)`) is not seen; telling a row source from * a column needs the engine's parse, not a pattern. */ +// Between the keywords: whitespace, or a comment (`--` or `#` to the line's end, `/* … */` to its first +// `*/`). Limit: a nested Postgres comment between them is not read as one. +const GAP = String.raw`(?:\s|--[^\n]*\n|#[^\n]*\n|\/\*(?:[^*]|\*(?!\/))*\*\/)+` +const NATURAL = new RegExp(String.raw`\bnatural${GAP}(?:(?:left|right|full|inner|cross|semi|anti)${GAP}){0,2}(?:outer${GAP})?join\b`, "i") export function joinsNaturally(sql: string): boolean { - const natural = /\bnatural\s+(?:(?:left|right|full|inner|cross|semi|anti)\s+){0,2}(?:outer\s+)?join\b/i - // Also with block comments blanked, so `natural /* … */ join` counts: either match only adds doubt. - return natural.test(sql) || natural.test(sql.replace(/\/\*[\s\S]*?\*\//g, " ")) + return NATURAL.test(sql) } function childMapOf(manifest: any): Record { diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 25c0ff0996..f70cbe4dbf 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -624,6 +624,11 @@ describe("columnReach with a NATURAL JOIN", () => { "select 'a\\'b' as c from a natural join b where t = 'z'", 'select "C:\\" as p from a natural join b where t = "z"', "select * from a natural /* inner */ join b", + "select a.id from a natural -- why\n join b", + "select a.id from a natural left -- x\nouter join b", + "select a.id from a natural # mysql\njoin b", + "select '/*' as s from a natural /* c */ join b", + "-- see /* note\nselect * from a natural /* c */ join b", // The words in a string or comment count too: that only adds doubt. "select 'natural join' as s from fct_orders", ]) From 2a4615570327a5eb3de64229ad96bd7c6ccb9438 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:55:15 +0530 Subject: [PATCH 24/37] fix: NATURAL JOIN lookup reads // and CR-ended comments, and stays bounded on any input Each natural word is checked in a 1000-character window, and past 100 of them the SQL counts as having one, which only adds doubt. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 19 ++++++++++++++----- .../test/altimate/column-reach.test.ts | 10 +++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 072a942988..e9328133f6 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -165,12 +165,21 @@ function namesColumn(sql: string, column: string): boolean { * Limit: a parent's whole row passed as a value (`to_json(o)`) is not seen; telling a row source from * a column needs the engine's parse, not a pattern. */ -// Between the keywords: whitespace, or a comment (`--` or `#` to the line's end, `/* … */` to its first -// `*/`). Limit: a nested Postgres comment between them is not read as one. -const GAP = String.raw`(?:\s|--[^\n]*\n|#[^\n]*\n|\/\*(?:[^*]|\*(?!\/))*\*\/)+` -const NATURAL = new RegExp(String.raw`\bnatural${GAP}(?:(?:left|right|full|inner|cross|semi|anti)${GAP}){0,2}(?:outer${GAP})?join\b`, "i") +// Between the keywords: whitespace, or a comment (`--`, `#` or `//` to the line's end, `/* … */` to its +// first `*/`). Limit: a nested Postgres comment between them is not read as one. +const GAP = String.raw`(?:\s|(?:--|#|\/\/)[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/)+` +const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|semi|anti)${GAP}){0,2}(?:outer${GAP})?join\b`, "i") +// How far past a `natural` the rest of the join is looked for, and how many are looked at: past that +// many the SQL may well have one, which only adds doubt. The work stays bounded on any input. +// Limit: comments longer than the window between the keywords hide the join. +const NATURAL_WINDOW = 1000 +const MAX_NATURALS = 100 export function joinsNaturally(sql: string): boolean { - return NATURAL.test(sql) + let seen = 0 + for (const match of sql.matchAll(/(? MAX_NATURALS || NATURAL.test(sql.slice(match.index!, match.index! + NATURAL_WINDOW))) return true + } + return false } function childMapOf(manifest: any): Record { diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index f70cbe4dbf..d9469a54cd 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -612,7 +612,11 @@ describe("columnReach with a NATURAL JOIN", () => { }) test("every NATURAL join form counts, whatever the quoting around it; aggregates under their own names do not", () => { - for (const sql of ["select sum(revenue) as revenue, cast(d as date) as d from fct_orders", "select natural_key, naturally from fct_orders"]) + for (const sql of [ + "select sum(revenue) as revenue, cast(d as date) as d from fct_orders", + "select natural_key, naturally from fct_orders", + "select 'natural ' as a, 'natural' as b from fct_orders natural_join", + ]) expect(joinsNaturally(sql)).toBe(false) for (const sql of [ "select a.id from fct_orders a natural left semi join other b", @@ -627,6 +631,10 @@ describe("columnReach with a NATURAL JOIN", () => { "select a.id from a natural -- why\n join b", "select a.id from a natural left -- x\nouter join b", "select a.id from a natural # mysql\njoin b", + "select a.id from a natural // snowflake\njoin b", + "select a.id from a natural -- old mac\rjoin b", + // Many `natural` words in a string before the real join: each is checked in a bounded window. + "select $$" + "natural /* ".repeat(4000) + "$$ as s from a natural join b", "select '/*' as s from a natural /* c */ join b", "-- see /* note\nselect * from a natural /* c */ join b", // The words in a string or comment count too: that only adds doubt. From 299bdf0daf5aab95491c7c27727b7b3e248d16a1 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 17:57:57 +0530 Subject: [PATCH 25/37] fix: Snowflake's DIRECTED NATURAL JOIN counts too Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/column-reach.ts | 3 ++- packages/opencode/test/altimate/column-reach.test.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index e9328133f6..3472849cc2 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -168,7 +168,8 @@ function namesColumn(sql: string, column: string): boolean { // Between the keywords: whitespace, or a comment (`--`, `#` or `//` to the line's end, `/* … */` to its // first `*/`). Limit: a nested Postgres comment between them is not read as one. const GAP = String.raw`(?:\s|(?:--|#|\/\/)[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/)+` -const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|semi|anti)${GAP}){0,2}(?:outer${GAP})?join\b`, "i") +// The join's modifiers in any dialect: LEFT/RIGHT/FULL/INNER/CROSS, OUTER, Spark's SEMI/ANTI, Snowflake's DIRECTED. +const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|outer|semi|anti|directed)${GAP}){0,4}join\b`, "i") // How far past a `natural` the rest of the join is looked for, and how many are looked at: past that // many the SQL may well have one, which only adds doubt. The work stays bounded on any input. // Limit: comments longer than the window between the keywords hide the join. diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index d9469a54cd..2ec1adc9ef 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -622,6 +622,8 @@ describe("columnReach with a NATURAL JOIN", () => { "select a.id from fct_orders a natural left semi join other b", "select * from a natural cross join b", "select * from a natural full outer join b", + "select a.id from a natural inner directed join b", + "select a.id from a natural left outer directed join b", "select array[(select max(a.id) from fct_orders a natural join other b)] as ids", 'select "a\\"b" as c from a natural join b where t = "z"', `select "it's" as c from a natural join b where y = 'q'`, From 5b670699a99a92172b3ae94bbbfec21b712220ad Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:07:45 +0530 Subject: [PATCH 26/37] fix: a manifest is placed where the system opens it, through Filesystem.realLocation; an empty path is no model's; NATURAL JOIN gaps up to 10,000 characters - links resolve before `..`, so `/link/../x` lands where the link points; the duplicate resolver is gone, and a link loop cannot be placed - a request that is empty once normalized never matches a model without a file path - the window per natural grows to 10,000 characters, still bounded with the 100-word cap - tests for count_big(*)/count_if(*), a backticked output alias, link/.. and the empty path Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 7 +++-- .../src/altimate/native/dbt/lineage.ts | 2 +- .../src/altimate/tools/dbt-lineage.ts | 31 ++++++------------- .../test/altimate/column-reach.test.ts | 20 +++++++++--- .../altimate/dbt-lineage-structured.test.ts | 6 ++++ 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 3472849cc2..21dba727c6 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -171,9 +171,10 @@ const GAP = String.raw`(?:\s|(?:--|#|\/\/)[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\ // The join's modifiers in any dialect: LEFT/RIGHT/FULL/INNER/CROSS, OUTER, Spark's SEMI/ANTI, Snowflake's DIRECTED. const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|outer|semi|anti|directed)${GAP}){0,4}join\b`, "i") // How far past a `natural` the rest of the join is looked for, and how many are looked at: past that -// many the SQL may well have one, which only adds doubt. The work stays bounded on any input. -// Limit: comments longer than the window between the keywords hide the join. -const NATURAL_WINDOW = 1000 +// many the SQL may well have one, which only adds doubt. An unclosed `/*` makes each check read to the +// window's end, so the window is what keeps crafted input cheap. +// Limit: more than 10,000 characters of comments between the keywords hide the join. +const NATURAL_WINDOW = 10_000 const MAX_NATURALS = 100 export function joinsNaturally(sql: string): boolean { let seen = 0 diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index 8efe6658d0..54892044ca 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -47,7 +47,7 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { const filePath = (p: unknown) => String(p ?? "").replace(/\\/g, "/").replace(/^(?:\.\/)+/, "") const byPath = () => Object.values(nodes).find( - (node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model), + (node) => !!filePath(params.model) && ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model), ) const modelNode = findModel(nodes, params.model) ?? diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 4f316c00f0..88716e638b 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -6,6 +6,7 @@ import { Dispatcher } from "../native" import type { DbtLineageResult } from "../native/types" import { loadRawManifest } from "../native/dbt/helpers" import { lineageGraph } from "../native/dbt/graph" +import { Filesystem } from "../../util/filesystem" /** The structured result of dbt-integration's `getModelLineage` (version 1), relayed as is. */ export type ModelLineage = { @@ -130,35 +131,21 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { * by its path, its nearest existing folder's links resolved, so the project's own default path keeps * the adapter's answer; one with no dbt project above it at all (a default path resolved against * another folder) names no other project either. A link to a manifest not written yet is placed where - * it points. + * it points; a link loop cannot be placed. */ async function manifestOutsideAdapterProject(manifestPath: string): Promise { const { getDbtProjectRoot } = await import("../native/connections/register") const root = await getDbtProjectRoot() if (!root) return "the dbt adapter has no project root" - // The path with its links resolved, the parts not written yet appended; a link to a missing file is - // followed to where it points. Undefined past 64 links (a loop). + // Where the system would open the path: links resolved segment by segment before `..` (so + // `/link/../x` lands where the link points), parts not written yet kept, a link to a missing + // file followed. A relative path is joined to the working folder as written, never normalized first. const real = (p: string) => { - const rest: string[] = [] - let at = path.resolve(p) - for (let step = 0; step < 64; step++) { - try { - return path.join(fs.realpathSync(at), ...rest) - } catch {} - let link: string | undefined - try { - if (fs.lstatSync(at).isSymbolicLink()) link = fs.readlinkSync(at) - } catch {} - if (link !== undefined) { - // From the link's real folder, as the system resolves it: `..` climbs from there. - at = path.resolve(fs.realpathSync(path.dirname(at)), link) - continue - } - if (path.dirname(at) === at) return undefined - rest.unshift(path.basename(at)) - at = path.dirname(at) + try { + return Filesystem.realLocation(path.isAbsolute(p) ? p : `${process.cwd()}${path.sep}${p}`) + } catch { + return undefined } - return undefined } const projectDir = real(root) const manifest = real(manifestPath) diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 2ec1adc9ef..2de1610e8c 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test" import * as core from "@altimateai/altimate-core" -import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { columnReach, exposuresOn, parentsNamed, joinsNaturally, selectsStar } from "../../src/altimate/native/dbt/column-reach" @@ -74,9 +74,14 @@ describe("columnReach", () => { test("an output alias or a comment naming the column is not a read", () => { const value = manifest() - value.nodes["model.shop.rpt_counts"].compiled_code = "select count(order_id) as revenue /* revenue */ from fct_orders -- revenue" - const reach = columnReach(manifestFile(value), "fct_orders", "revenue") - expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"]) + for (const sql of [ + "select count(order_id) as revenue /* revenue */ from fct_orders -- revenue", + "select count(order_id) as `revenue` from fct_orders", + ]) { + value.nodes["model.shop.rpt_counts"].compiled_code = sql + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"]) + } }) test("a downstream model without compiled SQL is reported untraced, and what reads it only may read the column", () => { @@ -262,6 +267,8 @@ describe("selectsStar", () => { for (const sql of [ "select revenue * 100 as cents from a", "select count(*) as n from a", + "select count_big(*) as n from a", + "select count_if(*) as n from a", "select a, b*c as d from t", "select '*' as s from a", "select (a + b) * c as d from t", @@ -583,6 +590,10 @@ describe("dbtLineage", () => { // `./` and Windows separators spell the same path. expect(dbtLineage({ manifest_path: file, model: "./models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") expect(dbtLineage({ manifest_path: file, model: "models\\mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + // An empty path is no model's, not the first one's without a file path. + const pathless = JSON.parse(readFileSync(file, "utf8")) + delete pathless.nodes["model.shop.fct_orders"].original_file_path + expect(dbtLineage({ manifest_path: manifestFile(pathless), model: "./" }).confidence_factors).toContain("Model './' not found in manifest") const withSnapshot: any = manifest() withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" } expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap") @@ -635,6 +646,7 @@ describe("columnReach with a NATURAL JOIN", () => { "select a.id from a natural # mysql\njoin b", "select a.id from a natural // snowflake\njoin b", "select a.id from a natural -- old mac\rjoin b", + "select a.id from a natural /* " + "x".repeat(5000) + " */ join b", // Many `natural` words in a string before the real join: each is checked in a bounded window. "select $$" + "natural /* ".repeat(4000) + "$$ as s from a natural join b", "select '/*' as s from a natural /* c */ join b", diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index 36ad1b6a2f..ab5eb7d636 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -221,6 +221,12 @@ describe("dbt_lineage with the project's dbt adapter", () => { await symlink("../../otherproj/target/manifest.json", path.join(realOther, "proj", "target", "manifest.json")) const climbed = await tool.execute({ model: "customers", manifest_path: path.join(mine, "linked_target", "manifest.json") }, ctx) expect(climbed.metadata.structuredContent).toBeUndefined() + // `..` after a link climbs from where the link points, as the system opens it: a manifest reached + // through `/out/..` is the other project's, though the path reads as this one's. + await mkdir(path.join(other, "deep"), { recursive: true }) + await symlink(path.join(other, "deep"), path.join(mine, "out")) + const viaDots = await tool.execute({ model: "customers", manifest_path: `${mine}/out/../target/manifest.json` }, ctx) + expect(viaDots.metadata.structuredContent).toBeUndefined() // A path through a file is not a folder: it is placed, not thrown on. const through = await tool.execute({ model: "customers", manifest_path: path.join(mine, "dbt_project.yml", "manifest.json") }, ctx) expect(through.metadata.structuredContent).toEqual(TRACE) From 6d71706bbe71571320ca78bd1f12b3e01e932ca3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:11:52 +0530 Subject: [PATCH 27/37] fix: a manifest is placed as the manifest reader opens it, and only link hops count toward the loop limit loadRawManifest opens a path through realpathSync, which collapses `..` before following links, so the project check does the same: the adapter's lineage and the graph read beside it always come from one project. Climbing to the nearest existing folder no longer counts toward the 64-link limit. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-lineage.ts | 34 ++++++++++++++----- .../altimate/dbt-lineage-structured.test.ts | 13 +++++-- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 88716e638b..4d4029a4e0 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -6,7 +6,6 @@ import { Dispatcher } from "../native" import type { DbtLineageResult } from "../native/types" import { loadRawManifest } from "../native/dbt/helpers" import { lineageGraph } from "../native/dbt/graph" -import { Filesystem } from "../../util/filesystem" /** The structured result of dbt-integration's `getModelLineage` (version 1), relayed as is. */ export type ModelLineage = { @@ -131,21 +130,38 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { * by its path, its nearest existing folder's links resolved, so the project's own default path keeps * the adapter's answer; one with no dbt project above it at all (a default path resolved against * another folder) names no other project either. A link to a manifest not written yet is placed where - * it points; a link loop cannot be placed. + * it points. */ async function manifestOutsideAdapterProject(manifestPath: string): Promise { const { getDbtProjectRoot } = await import("../native/connections/register") const root = await getDbtProjectRoot() if (!root) return "the dbt adapter has no project root" - // Where the system would open the path: links resolved segment by segment before `..` (so - // `/link/../x` lands where the link points), parts not written yet kept, a link to a missing - // file followed. A relative path is joined to the working folder as written, never normalized first. + // The path with its links resolved, the parts not written yet appended; a link to a missing file is + // followed to where it points. `..` is collapsed first, as the manifest reader (loadRawManifest's + // realpathSync) does, so this places the file that reader opens. Undefined past 64 links (a loop); + // climbing to the nearest existing folder is not counted. const real = (p: string) => { - try { - return Filesystem.realLocation(path.isAbsolute(p) ? p : `${process.cwd()}${path.sep}${p}`) - } catch { - return undefined + const rest: string[] = [] + let at = path.resolve(p) + for (let links = 0; links < 64; ) { + try { + return path.join(fs.realpathSync(at), ...rest) + } catch {} + let link: string | undefined + try { + if (fs.lstatSync(at).isSymbolicLink()) link = fs.readlinkSync(at) + } catch {} + if (link !== undefined) { + links++ + // From the link's real folder, as the system resolves it: `..` climbs from there. + at = path.resolve(fs.realpathSync(path.dirname(at)), link) + continue + } + if (path.dirname(at) === at) return undefined + rest.unshift(path.basename(at)) + at = path.dirname(at) } + return undefined } const projectDir = real(root) const manifest = real(manifestPath) diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index ab5eb7d636..0d2207edff 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -221,12 +221,19 @@ describe("dbt_lineage with the project's dbt adapter", () => { await symlink("../../otherproj/target/manifest.json", path.join(realOther, "proj", "target", "manifest.json")) const climbed = await tool.execute({ model: "customers", manifest_path: path.join(mine, "linked_target", "manifest.json") }, ctx) expect(climbed.metadata.structuredContent).toBeUndefined() - // `..` after a link climbs from where the link points, as the system opens it: a manifest reached - // through `/out/..` is the other project's, though the path reads as this one's. + // `..` after a link is collapsed first, as the manifest reader opens it: `/out/..` is read + // as this project's manifest, so the adapter answers, matching the graph include_graph reads. await mkdir(path.join(other, "deep"), { recursive: true }) await symlink(path.join(other, "deep"), path.join(mine, "out")) const viaDots = await tool.execute({ model: "customers", manifest_path: `${mine}/out/../target/manifest.json` }, ctx) - expect(viaDots.metadata.structuredContent).toBeUndefined() + expect(viaDots.metadata.structuredContent).toEqual(TRACE) + // Many folders not written yet below the project are climbed, not counted as links. + const deep = path.join(mine, ...Array.from({ length: 80 }, (_, i) => `d${i}`), "manifest.json") + expect((await tool.execute({ model: "customers", manifest_path: deep }, ctx)).metadata.structuredContent).toEqual(TRACE) + // A link loop cannot be placed. + await symlink(path.join(mine, "target", "loop-b"), path.join(mine, "target", "loop-a")) + await symlink(path.join(mine, "target", "loop-a"), path.join(mine, "target", "loop-b")) + expect((await tool.execute({ model: "customers", manifest_path: path.join(mine, "target", "loop-a") }, ctx)).metadata.structuredContent).toBeUndefined() // A path through a file is not a folder: it is placed, not thrown on. const through = await tool.execute({ model: "customers", manifest_path: path.join(mine, "dbt_project.yml", "manifest.json") }, ctx) expect(through.metadata.structuredContent).toEqual(TRACE) From ef2797a24ce9c1a553e52159ff058b4de72d9336 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:17:22 +0530 Subject: [PATCH 28/37] refactor: the requested SQL path is normalized once, and the link/.. test proves which manifest the graph reads Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/lineage.ts | 8 +++++--- .../opencode/test/altimate/dbt-lineage-structured.test.ts | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index 54892044ca..562800f981 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -45,10 +45,12 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { // A model by name or unique_id; a snapshot by unique_id; either by its project-relative file path. // `./models/x.sql` and `models\\x.sql` are the manifest's `models/x.sql`. const filePath = (p: unknown) => String(p ?? "").replace(/\\/g, "/").replace(/^(?:\.\/)+/, "") + // An empty request (`./`) is no model's path. + const wanted = filePath(params.model) const byPath = () => - Object.values(nodes).find( - (node) => !!filePath(params.model) && ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model), - ) + wanted + ? Object.values(nodes).find((node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === wanted) + : undefined const modelNode = findModel(nodes, params.model) ?? (nodes[params.model]?.resource_type === "snapshot" ? nodes[params.model] : null) ?? diff --git a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts index 0d2207edff..e73007fec8 100644 --- a/packages/opencode/test/altimate/dbt-lineage-structured.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-structured.test.ts @@ -225,8 +225,11 @@ describe("dbt_lineage with the project's dbt adapter", () => { // as this project's manifest, so the adapter answers, matching the graph include_graph reads. await mkdir(path.join(other, "deep"), { recursive: true }) await symlink(path.join(other, "deep"), path.join(mine, "out")) - const viaDots = await tool.execute({ model: "customers", manifest_path: `${mine}/out/../target/manifest.json` }, ctx) + // The other project's manifest is made unreadable: a graph read from it would fail to parse. + await writeFile(path.join(other, "target", "manifest.json"), "{ not json") + const viaDots = await tool.execute({ model: "customers", manifest_path: `${mine}/out/../target/manifest.json`, include_graph: true }, ctx) expect(viaDots.metadata.structuredContent).toEqual(TRACE) + expect(String(viaDots.metadata.graph_error ?? "")).not.toMatch(/json|parse/i) // Many folders not written yet below the project are climbed, not counted as links. const deep = path.join(mine, ...Array.from({ length: 80 }, (_, i) => `d${i}`), "manifest.json") expect((await tool.execute({ model: "customers", manifest_path: deep }, ctx)).metadata.structuredContent).toEqual(TRACE) From 315685b1f00653bda91d4243be30af498c528cb1 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:18:35 +0530 Subject: [PATCH 29/37] style: Prettier on the files this change left unformatted Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 54 ++- .../opencode/src/altimate/native/dbt/graph.ts | 6 +- .../src/altimate/native/dbt/lineage.ts | 19 +- .../test/altimate/column-reach.test.ts | 413 +++++++++++++++--- 4 files changed, 405 insertions(+), 87 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 21dba727c6..aeb91e78ac 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -56,18 +56,27 @@ const TRACED = new Set(["model", "snapshot"]) /** "db"."schema"."orders"."amount" → { parts: ["db","schema","orders"], column: "amount" }, lower-cased. */ function splitSource(source: unknown): { parts: string[]; column: string; exact: string } | null { - const text = typeof source === "string" ? source : typeof (source as any)?.source === "string" ? (source as any).source : null + const text = + typeof source === "string" ? source : typeof (source as any)?.source === "string" ? (source as any).source : null if (!text) return null // Split on dots outside quotes: "a"."net.amount" is table a, column net.amount. - const raw = [...text.matchAll(/"([^"]*)"|`([^`]*)`|\[([^\]]*)\]|([^."`\[\]]+)/g)].map((m) => m[1] ?? m[2] ?? m[3] ?? m[4] ?? "") + const raw = [...text.matchAll(/"([^"]*)"|`([^`]*)`|\[([^\]]*)\]|([^."`\[\]]+)/g)].map( + (m) => m[1] ?? m[2] ?? m[3] ?? m[4] ?? "", + ) if (raw.length < 2) return null // `exact` keeps the column's case, for parents that document two columns differing only by case. - return { parts: raw.slice(0, -1).map((part) => part.toLowerCase()), column: raw.at(-1)!.toLowerCase(), exact: raw.at(-1)! } + return { + parts: raw.slice(0, -1).map((part) => part.toLowerCase()), + column: raw.at(-1)!.toLowerCase(), + exact: raw.at(-1)!, + } } /** The ways a parent's relation can be written in a child's compiled SQL, most qualified first. */ function relationKeys(node: any): string[] { - const table = [node?.alias, node?.identifier, node?.name].filter((v): v is string => typeof v === "string" && v !== "") + const table = [node?.alias, node?.identifier, node?.name].filter( + (v): v is string => typeof v === "string" && v !== "", + ) const schema = typeof node?.schema === "string" ? node.schema : null const database = typeof node?.database === "string" ? node.database : null const keys = new Set() @@ -169,7 +178,10 @@ function namesColumn(sql: string, column: string): boolean { // first `*/`). Limit: a nested Postgres comment between them is not read as one. const GAP = String.raw`(?:\s|(?:--|#|\/\/)[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/)+` // The join's modifiers in any dialect: LEFT/RIGHT/FULL/INNER/CROSS, OUTER, Spark's SEMI/ANTI, Snowflake's DIRECTED. -const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|outer|semi|anti|directed)${GAP}){0,4}join\b`, "i") +const NATURAL = new RegExp( + String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|outer|semi|anti|directed)${GAP}){0,4}join\b`, + "i", +) // How far past a `natural` the rest of the join is looked for, and how many are looked at: past that // many the SQL may well have one, which only adds doubt. An unclosed `/*` makes each check read to the // window's end, so the window is what keeps crafted input cheap. @@ -283,7 +295,11 @@ export function columnReach(manifestPath: string, model: string, column: string, const edges: { source: unknown; target: string; lineage_type?: string }[] = lineage?.column_lineage?.length ? lineage.column_lineage : Object.entries(lineage?.column_dict ?? {}).flatMap(([out, sources]) => - (Array.isArray(sources) ? sources : [sources]).map((source) => ({ source, target: out, lineage_type: "direct" })), + (Array.isArray(sources) ? sources : [sources]).map((source) => ({ + source, + target: out, + lineage_type: "direct", + })), ) // Errors beside some edges: the engine traced only part of the SQL. const partial = Array.isArray(lineage?.errors) && lineage.errors.length > 0 @@ -298,11 +314,17 @@ export function columnReach(manifestPath: string, model: string, column: string, } const collides = parents.some((p) => allParents.some((other) => other !== p && bareName(other) === bareName(p))) const wildcard = - selectsStar(sqlText) && parents.some((p) => [...tracked.get(p)!].some((col) => !documentedColumns(lookup(p)).has(col))) + selectsStar(sqlText) && + parents.some((p) => [...tracked.get(p)!].some((col) => !documentedColumns(lookup(p)).has(col))) if (collides || wildcard) { unsure.add(id) uncertain.push( - entry(id, collides ? "reads a parent whose table name another parent shares" : "selects * from a parent whose schema.yml does not list the column"), + entry( + id, + collides + ? "reads a parent whose table name another parent shares" + : "selects * from a parent whose schema.yml does not list the column", + ), ) continue } @@ -366,7 +388,9 @@ export function columnReach(manifestPath: string, model: string, column: string, uncertain.push(entry(id, "uses a NATURAL JOIN, whose join columns the lineage engine cannot trace")) continue } - const shared = named.some((col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1) + const shared = named.some( + (col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1, + ) if (ambiguous || named.length > 0 || fromUnsure) { if (ambiguous || fromUnsure || natural) unsure.add(id) uncertain.push( @@ -399,7 +423,10 @@ export function exposuresOn(manifestPath: string, ids: Iterable): Column const manifest = loadRawManifest(manifestPath) const wanted = new Set(ids) // A metric reads its semantic model, which reads a model: whatever depends on one depends on the others. - const layers = [...Object.entries(manifest?.semantic_models ?? {}), ...Object.entries(manifest?.metrics ?? {})] + const layers = [ + ...Object.entries(manifest?.semantic_models ?? {}), + ...Object.entries(manifest?.metrics ?? {}), + ] for (let grew = true; grew; ) { grew = false for (const [id, node] of layers) { @@ -410,5 +437,10 @@ export function exposuresOn(manifestPath: string, ids: Iterable): Column } return Object.entries(manifest?.exposures ?? {}) .filter(([, node]) => (node?.depends_on?.nodes ?? []).some((dep: string) => wanted.has(dep))) - .map(([id, node]) => ({ unique_id: id, name: node?.name ?? id, label: node?.label || node?.name || id, type: node?.type ?? "exposure" })) + .map(([id, node]) => ({ + unique_id: id, + name: node?.name ?? id, + label: node?.label || node?.name || id, + type: node?.type ?? "exposure", + })) } diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts index 89ef133344..3edff2baf9 100644 --- a/packages/opencode/src/altimate/native/dbt/graph.ts +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -41,7 +41,8 @@ function lookup(manifest: any, id: string): any { function relationOf(node: any): string | null { // A manifest that carries relation_name with null (ephemeral models, analyses, tests) has no relation. if (node && "relation_name" in node && !node.relation_name) return null - if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name.replace(/[`"\[\]]/g, "").toLowerCase() + if (typeof node?.relation_name === "string" && node.relation_name) + return node.relation_name.replace(/[`"\[\]]/g, "").toLowerCase() const parts = [node?.database, node?.schema, node?.alias ?? node?.identifier ?? node?.name].filter( (part): part is string => typeof part === "string" && part !== "", ) @@ -77,7 +78,8 @@ function mapsOf(manifest: any): { parents: Record; children: R export function lineageGraph(manifest: any, id: string): LineageGraph { const { parents, children } = mapsOf(manifest) - const isTest = (one: string) => ["test", "unit_test"].includes(lookup(manifest, one)?.resource_type ?? one.split(".")[0]) + const isTest = (one: string) => + ["test", "unit_test"].includes(lookup(manifest, one)?.resource_type ?? one.split(".")[0]) const keep = new Set([id]) // Breadth first, so a cut keeps the nearest nodes. const walk = (links: Record, limit: number, seen = new Set([id])) => { diff --git a/packages/opencode/src/altimate/native/dbt/lineage.ts b/packages/opencode/src/altimate/native/dbt/lineage.ts index 562800f981..eac0cff688 100644 --- a/packages/opencode/src/altimate/native/dbt/lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/lineage.ts @@ -44,12 +44,17 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { // A model by name or unique_id; a snapshot by unique_id; either by its project-relative file path. // `./models/x.sql` and `models\\x.sql` are the manifest's `models/x.sql`. - const filePath = (p: unknown) => String(p ?? "").replace(/\\/g, "/").replace(/^(?:\.\/)+/, "") + const filePath = (p: unknown) => + String(p ?? "") + .replace(/\\/g, "/") + .replace(/^(?:\.\/)+/, "") // An empty request (`./`) is no model's path. const wanted = filePath(params.model) const byPath = () => wanted - ? Object.values(nodes).find((node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === wanted) + ? Object.values(nodes).find( + (node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === wanted, + ) : undefined const modelNode = findModel(nodes, params.model) ?? @@ -62,7 +67,9 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { } // The id of the node that was selected, not of whatever else in the manifest the request string spells. const uniqueId = - (typeof modelNode.unique_id === "string" && nodes[modelNode.unique_id] === modelNode ? modelNode.unique_id : undefined) ?? + (typeof modelNode.unique_id === "string" && nodes[modelNode.unique_id] === modelNode + ? modelNode.unique_id + : undefined) ?? Object.keys(nodes).find((id) => nodes[id] === modelNode) ?? params.model const graph = params.include_graph && uniqueId ? { graph: lineageGraph(manifest, uniqueId) } : {} @@ -103,7 +110,11 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { raw_lineage: rawLineage, // The engine reports SQL it cannot parse in `errors` with no edges, rather than throwing. confidence: rawLineage.error || unparsed(rawLineage) ? "low" : "high", - confidence_factors: rawLineage.error ? [String(rawLineage.error)] : unparsed(rawLineage) ? [String((rawLineage.errors as unknown[])[0])] : [], + confidence_factors: rawLineage.error + ? [String(rawLineage.error)] + : unparsed(rawLineage) + ? [String((rawLineage.errors as unknown[])[0])] + : [], upstream: upstreamIds, ...graph, } diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 2de1610e8c..fbb65c6422 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -3,7 +3,13 @@ import * as core from "@altimateai/altimate-core" import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { columnReach, exposuresOn, parentsNamed, joinsNaturally, selectsStar } from "../../src/altimate/native/dbt/column-reach" +import { + columnReach, + exposuresOn, + parentsNamed, + joinsNaturally, + selectsStar, +} from "../../src/altimate/native/dbt/column-reach" import { lineageGraph } from "../../src/altimate/native/dbt/graph" import { structuredError, structuredResult } from "../../src/altimate/mcp-server/server" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" @@ -32,19 +38,50 @@ const model = (name: string, deps: string[], sql: string, columns: string[], ext // rpt_ltv), filtered on by rpt_big_orders, and never read by rpt_counts. function manifest() { const nodes: Record = { - "model.shop.fct_orders": model("fct_orders", [], "select order_id, revenue, gross_revenue from raw.orders", ["order_id", "revenue", "gross_revenue"]), - "model.shop.fct_daily": model("fct_daily", ["model.shop.fct_orders"], "select sum(revenue) as revenue from fct_orders", ["revenue"]), - "model.shop.mart_ltv": model("mart_ltv", ["model.shop.fct_orders"], "select order_id, revenue as ltv from fct_orders", ["order_id", "ltv"]), - "model.shop.rpt_ltv": model("rpt_ltv", ["model.shop.mart_ltv"], "select max(ltv) as top_ltv from mart_ltv", ["top_ltv"]), - "model.shop.rpt_big_orders": model("rpt_big_orders", ["model.shop.fct_orders"], "select order_id from fct_orders where revenue > 100", ["order_id"]), - "model.shop.rpt_counts": model("rpt_counts", ["model.shop.fct_orders"], "select count(order_id) as orders from fct_orders", ["orders"]), + "model.shop.fct_orders": model("fct_orders", [], "select order_id, revenue, gross_revenue from raw.orders", [ + "order_id", + "revenue", + "gross_revenue", + ]), + "model.shop.fct_daily": model( + "fct_daily", + ["model.shop.fct_orders"], + "select sum(revenue) as revenue from fct_orders", + ["revenue"], + ), + "model.shop.mart_ltv": model( + "mart_ltv", + ["model.shop.fct_orders"], + "select order_id, revenue as ltv from fct_orders", + ["order_id", "ltv"], + ), + "model.shop.rpt_ltv": model("rpt_ltv", ["model.shop.mart_ltv"], "select max(ltv) as top_ltv from mart_ltv", [ + "top_ltv", + ]), + "model.shop.rpt_big_orders": model( + "rpt_big_orders", + ["model.shop.fct_orders"], + "select order_id from fct_orders where revenue > 100", + ["order_id"], + ), + "model.shop.rpt_counts": model( + "rpt_counts", + ["model.shop.fct_orders"], + "select count(order_id) as orders from fct_orders", + ["orders"], + ), } return { metadata: { adapter_type: "duckdb" }, nodes, sources: {}, exposures: { - "exposure.shop.exec_revenue": { name: "exec_revenue", label: "Exec Revenue", type: "dashboard", depends_on: { nodes: ["model.shop.fct_daily"] } }, + "exposure.shop.exec_revenue": { + name: "exec_revenue", + label: "Exec Revenue", + type: "dashboard", + depends_on: { nodes: ["model.shop.fct_daily"] }, + }, "exposure.shop.ops": { name: "ops", type: "application", depends_on: { nodes: ["model.shop.rpt_counts"] } }, }, } @@ -65,10 +102,16 @@ describe("columnReach", () => { expect(byName).toEqual({ fct_daily: ["revenue"], mart_ltv: ["ltv"], rpt_ltv: ["top_ltv"] }) // The engine reports no edge for a filter: the model is a possible reader, not a certain one. expect(reach.uncertain).toEqual([ - { name: "rpt_big_orders", unique_id: "model.shop.rpt_big_orders", reason: "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)" }, + { + name: "rpt_big_orders", + unique_id: "model.shop.rpt_big_orders", + reason: "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)", + }, ]) expect(reach.untraced).toEqual([]) - expect(reach.exposures).toEqual([{ unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }]) + expect(reach.exposures).toEqual([ + { unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }, + ]) expect(reach.truncated).toBe(false) }) @@ -92,19 +135,30 @@ describe("columnReach", () => { expect(reach.untraced).toEqual([ { name: "mart_ltv", unique_id: "model.shop.mart_ltv", reason: "No compiled SQL found — run `dbt compile` first" }, ]) - expect(reach.uncertain.map((m) => [m.name, m.reason])).toContainEqual(["rpt_ltv", "is downstream of a model that could not be traced or reads it ambiguously"]) + expect(reach.uncertain.map((m) => [m.name, m.reason])).toContainEqual([ + "rpt_ltv", + "is downstream of a model that could not be traced or reads it ambiguously", + ]) }) test("a model that only filters on the column passes nothing on to its children", () => { const value = manifest() - value.nodes["model.shop.rpt_after_filter"] = model("rpt_after_filter", ["model.shop.rpt_big_orders"], "select count(order_id) as n from rpt_big_orders", ["n"]) + value.nodes["model.shop.rpt_after_filter"] = model( + "rpt_after_filter", + ["model.shop.rpt_big_orders"], + "select count(order_id) as n from rpt_big_orders", + ["n"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"]) }) test("a join key another parent also documents is a possible read, not a dropped one", () => { const value = manifest() - value.nodes["model.shop.dim_orders"] = model("dim_orders", [], "select order_id, revenue from raw.dim", ["order_id", "revenue"]) + value.nodes["model.shop.dim_orders"] = model("dim_orders", [], "select order_id, revenue from raw.dim", [ + "order_id", + "revenue", + ]) value.nodes["model.shop.rpt_joined"] = model( "rpt_joined", ["model.shop.fct_orders", "model.shop.dim_orders"], @@ -120,7 +174,14 @@ describe("columnReach", () => { test("a source and a model with the same table name: a qualified source column is not the model's", () => { const value: any = manifest() value.sources = { - "source.shop.raw.fct_orders": { resource_type: "source", name: "fct_orders", identifier: "fct_orders", schema: "raw", database: "shop", columns: { revenue: column("revenue") } }, + "source.shop.raw.fct_orders": { + resource_type: "source", + name: "fct_orders", + identifier: "fct_orders", + schema: "raw", + database: "shop", + columns: { revenue: column("revenue") }, + }, } value.nodes["model.shop.rpt_src"] = model( "rpt_src", @@ -134,25 +195,56 @@ describe("columnReach", () => { test("a table name two parents share is ambiguous; a schema-qualified one is not", () => { const base = manifest() - const other = model("orders_copy", [], "select order_id, revenue from raw.copy", ["order_id", "revenue"], { alias: "fct_orders", schema: "archive" }) + const other = model("orders_copy", [], "select order_id, revenue from raw.copy", ["order_id", "revenue"], { + alias: "fct_orders", + schema: "archive", + }) const sql = (from: string) => `select b.revenue as kept from ${from}` const child = (from: string) => model("rpt_join", ["model.shop.fct_orders", "model.shop.orders_copy"], sql(from), ["kept"]) - const bare = { ...base, nodes: { ...base.nodes, "model.shop.orders_copy": other, "model.shop.rpt_join": child("fct_orders a join fct_orders b on a.order_id = b.order_id") } } + const bare = { + ...base, + nodes: { + ...base.nodes, + "model.shop.orders_copy": other, + "model.shop.rpt_join": child("fct_orders a join fct_orders b on a.order_id = b.order_id"), + }, + } const ambiguous = columnReach(manifestFile(bare), "fct_orders", "revenue") expect(ambiguous.models.map((m) => m.name)).not.toContain("rpt_join") - expect(ambiguous.uncertain.find((m) => m.name === "rpt_join")?.reason).toBe("reads a parent whose table name another parent shares") + expect(ambiguous.uncertain.find((m) => m.name === "rpt_join")?.reason).toBe( + "reads a parent whose table name another parent shares", + ) // Only archive.fct_orders (not traced) is read: no reader. - const qualified = { ...base, nodes: { ...base.nodes, "model.shop.orders_copy": other, "model.shop.rpt_join": child("main.fct_orders a join archive.fct_orders b on a.order_id = b.order_id") } } + const qualified = { + ...base, + nodes: { + ...base.nodes, + "model.shop.orders_copy": other, + "model.shop.rpt_join": child("main.fct_orders a join archive.fct_orders b on a.order_id = b.order_id"), + }, + } const reach = columnReach(manifestFile(qualified), "fct_orders", "revenue") expect(reach.models.map((m) => m.name)).not.toContain("rpt_join") }) test("a snapshot is traced and carries the column on", () => { const value = manifest() - value.nodes["snapshot.shop.snap_orders"] = { ...model("snap_orders", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", ["order_id", "revenue"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap_orders" } - value.nodes["model.shop.rpt_snap"] = model("rpt_snap", ["snapshot.shop.snap_orders"], "select sum(revenue) as snap_revenue from snap_orders", ["snap_revenue"]) + value.nodes["snapshot.shop.snap_orders"] = { + ...model("snap_orders", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", [ + "order_id", + "revenue", + ]), + resource_type: "snapshot", + unique_id: "snapshot.shop.snap_orders", + } + value.nodes["model.shop.rpt_snap"] = model( + "rpt_snap", + ["snapshot.shop.snap_orders"], + "select sum(revenue) as snap_revenue from snap_orders", + ["snap_revenue"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") const byName = Object.fromEntries(reach.models.map((m) => [m.name, m.columns])) expect(byName.snap_orders).toEqual(["revenue"]) @@ -167,7 +259,9 @@ describe("columnReach", () => { test("an unknown model or a missing manifest throws", () => { expect(() => columnReach(manifestFile(), "nope", "revenue")).toThrow(/not found/) - expect(() => columnReach(join(tmpdir(), "no-such-manifest.json"), "fct_orders", "revenue")).toThrow(/Manifest not found/) + expect(() => columnReach(join(tmpdir(), "no-such-manifest.json"), "fct_orders", "revenue")).toThrow( + /Manifest not found/, + ) }) }) @@ -179,15 +273,25 @@ describe("exposuresOn through metrics", () => { "metric.shop.order_count": { depends_on: { nodes: ["semantic_model.shop.orders"] } }, "metric.shop.double_count": { depends_on: { nodes: ["metric.shop.order_count"] } }, } - value.exposures["exposure.shop.kpi"] = { name: "kpi", type: "dashboard", depends_on: { nodes: ["metric.shop.double_count"] } } - expect(exposuresOn(manifestFile(value), ["model.shop.rpt_counts"]).map((e) => e.unique_id).sort()).toEqual(["exposure.shop.kpi", "exposure.shop.ops"]) + value.exposures["exposure.shop.kpi"] = { + name: "kpi", + type: "dashboard", + depends_on: { nodes: ["metric.shop.double_count"] }, + } + expect( + exposuresOn(manifestFile(value), ["model.shop.rpt_counts"]) + .map((e) => e.unique_id) + .sort(), + ).toEqual(["exposure.shop.kpi", "exposure.shop.ops"]) }) }) describe("exposuresOn", () => { test("lists exposures on any of the given nodes, by label when there is one", () => { const file = manifestFile() - expect(exposuresOn(file, ["model.shop.rpt_counts"])).toEqual([{ unique_id: "exposure.shop.ops", name: "ops", label: "ops", type: "application" }]) + expect(exposuresOn(file, ["model.shop.rpt_counts"])).toEqual([ + { unique_id: "exposure.shop.ops", name: "ops", label: "ops", type: "application" }, + ]) expect(exposuresOn(file, ["model.shop.fct_orders"])).toEqual([]) }) }) @@ -210,19 +314,40 @@ describe("columnReach where the engine's schema falls short", () => { test("select * from a parent whose schema.yml does not list the column is a possible read, and its children may read it", () => { const value = manifest() value.nodes["model.shop.fct_orders"].columns = { order_id: column("order_id") } - value.nodes["model.shop.rpt_star"] = model("rpt_star", ["model.shop.fct_orders"], 'select * from "shop"."main"."fct_orders"', []) - value.nodes["model.shop.rpt_after_star"] = model("rpt_after_star", ["model.shop.rpt_star"], "select sum(revenue) as total from rpt_star", ["total"]) + value.nodes["model.shop.rpt_star"] = model( + "rpt_star", + ["model.shop.fct_orders"], + 'select * from "shop"."main"."fct_orders"', + [], + ) + value.nodes["model.shop.rpt_after_star"] = model( + "rpt_after_star", + ["model.shop.rpt_star"], + "select sum(revenue) as total from rpt_star", + ["total"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") - expect(reach.uncertain.find((m) => m.name === "rpt_star")?.reason).toBe("selects * from a parent whose schema.yml does not list the column") + expect(reach.uncertain.find((m) => m.name === "rpt_star")?.reason).toBe( + "selects * from a parent whose schema.yml does not list the column", + ) expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after_star") }) test("a dashboard downstream of a wildcard reader in any dialect form is reported, not dropped", () => { - for (const sql of ["select distinct on (order_id) * from fct_orders", "select top 10 * from fct_orders", 'select "shop"."main"."fct_orders".* from "shop"."main"."fct_orders"']) { + for (const sql of [ + "select distinct on (order_id) * from fct_orders", + "select top 10 * from fct_orders", + 'select "shop"."main"."fct_orders".* from "shop"."main"."fct_orders"', + ]) { const value: any = manifest() value.nodes["model.shop.fct_orders"].columns = { order_id: column("order_id") } value.nodes["model.shop.rpt_star"] = model("rpt_star", ["model.shop.fct_orders"], sql, []) - value.exposures["exposure.shop.star_board"] = { name: "star_board", label: "Star Board", type: "dashboard", depends_on: { nodes: ["model.shop.rpt_star"] } } + value.exposures["exposure.shop.star_board"] = { + name: "star_board", + label: "Star Board", + type: "dashboard", + depends_on: { nodes: ["model.shop.rpt_star"] }, + } const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.uncertain.map((m) => m.name)).toContain("rpt_star") expect(reach.uncertain_exposures.map((e) => e.label)).toContain("Star Board") @@ -232,7 +357,12 @@ describe("columnReach where the engine's schema falls short", () => { test("a column name with a dot, quoted, is matched whole", () => { const value = manifest() value.nodes["model.shop.fct_orders"].columns = { "net.amount": column("net.amount") } - value.nodes["model.shop.rpt_net"] = model("rpt_net", ["model.shop.fct_orders"], 'select "net.amount" as net from fct_orders', ["net"]) + value.nodes["model.shop.rpt_net"] = model( + "rpt_net", + ["model.shop.fct_orders"], + 'select "net.amount" as net from fct_orders', + ["net"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "net.amount") expect(reach.models.find((m) => m.name === "rpt_net")?.columns).toEqual(["net"]) }) @@ -283,7 +413,12 @@ describe("selectsStar", () => { ]) expect(selectsStar(sql)).toBe(false) // A function over the whole row reads every column; only count(*) reads none. - for (const sql of ["select id, hash(*) as h from a", "select object_construct(*) as o from a", "select to_json(struct(*)) as j from a", "select columns(*) from a"]) + for (const sql of [ + "select id, hash(*) as h from a", + "select object_construct(*) as o from a", + "select to_json(struct(*)) as j from a", + "select columns(*) from a", + ]) expect(selectsStar(sql)).toBe(true) }) }) @@ -291,7 +426,10 @@ describe("selectsStar", () => { describe("columnReach through an ephemeral parent", () => { test("a model reading an ephemeral parent is a possible read, never a silent zero", () => { const value = manifest() - value.nodes["model.shop.eph"] = { ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", ["order_id", "revenue"]), config: { materialized: "ephemeral" } } + value.nodes["model.shop.eph"] = { + ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue from fct_orders", ["order_id", "revenue"]), + config: { materialized: "ephemeral" }, + } value.nodes["model.shop.rpt_eph"] = model( "rpt_eph", ["model.shop.eph"], @@ -309,7 +447,13 @@ describe("columnReach through an ephemeral parent", () => { describe("columnReach with an ephemeral and a regular parent", () => { test("a traced edge from the regular parent does not hide what flows through the ephemeral one", () => { const value = manifest() - value.nodes["model.shop.eph"] = { ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue as renamed from fct_orders", ["order_id", "renamed"]), config: { materialized: "ephemeral" } } + value.nodes["model.shop.eph"] = { + ...model("eph", ["model.shop.fct_orders"], "select order_id, revenue as renamed from fct_orders", [ + "order_id", + "renamed", + ]), + config: { materialized: "ephemeral" }, + } value.nodes["model.shop.mix"] = model( "mix", ["model.shop.fct_daily", "model.shop.eph"], @@ -327,7 +471,12 @@ describe("columnReach with columns that differ only by case", () => { test('"Revenue" is not revenue where the parent documents both', () => { const value: any = manifest() value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } - value.nodes["model.shop.rpt_case"] = model("rpt_case", ["model.shop.fct_orders"], 'select "Revenue" as other from fct_orders', ["other"]) + value.nodes["model.shop.rpt_case"] = model( + "rpt_case", + ["model.shop.fct_orders"], + 'select "Revenue" as other from fct_orders', + ["other"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.models.map((m) => m.name)).not.toContain("rpt_case") expect(reach.models.map((m) => m.name)).toContain("fct_daily") @@ -343,7 +492,12 @@ describe("columnReach case handling", () => { test("a spelling matching two columns that differ only by case leaves both columns' readers uncertain, never certain", () => { const value: any = manifest() value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } - value.nodes["model.shop.rpt_case"] = model("rpt_case", ["model.shop.fct_orders"], 'select "Revenue" as other from fct_orders', ["other"]) + value.nodes["model.shop.rpt_case"] = model( + "rpt_case", + ["model.shop.fct_orders"], + 'select "Revenue" as other from fct_orders', + ["other"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "REVENUE") expect(reach.models.map((m) => m.name)).not.toContain("fct_daily") expect(reach.models.map((m) => m.name)).not.toContain("rpt_case") @@ -353,7 +507,11 @@ describe("columnReach case handling", () => { test("with the spelling unsettled, a dashboard on the model itself is uncertain too", () => { const value: any = manifest() value.nodes["model.shop.fct_orders"].columns = { revenue: column("revenue"), Revenue: column("Revenue") } - value.exposures["exposure.shop.orders_board"] = { name: "orders_board", type: "dashboard", depends_on: { nodes: ["model.shop.fct_orders"] } } + value.exposures["exposure.shop.orders_board"] = { + name: "orders_board", + type: "dashboard", + depends_on: { nodes: ["model.shop.fct_orders"] }, + } const reach = columnReach(manifestFile(value), "fct_orders", "REVENUE") expect(reach.exposures.map((e) => e.unique_id)).not.toContain("exposure.shop.orders_board") expect(reach.uncertain_exposures.map((e) => e.unique_id)).toContain("exposure.shop.orders_board") @@ -378,13 +536,21 @@ describe("columnReach with an empty column_lineage", () => { describe("columnReach on a model the engine cannot fully trace", () => { const answer = (byModel: (sql: string) => unknown) => spyOn(core, "columnLineage").mockImplementation(((sql: string) => byModel(sql)) as any) - const direct = (sql: string) => ({ column_lineage: [{ source: '"shop"."main"."fct_orders"."revenue"', target: "revenue", lineage_type: "direct" }], column_dict: {} }) + const direct = (sql: string) => ({ + column_lineage: [{ source: '"shop"."main"."fct_orders"."revenue"', target: "revenue", lineage_type: "direct" }], + column_dict: {}, + }) test("no edge at all: SQL that names the column (a PIVOT) is uncertain and so is its child; SQL that never names it reads nothing", () => { const pivot = "select * from fct_orders pivot(sum(revenue) for status in ('paid' as paid_total))" const chain: any = manifest() chain.nodes["model.shop.fct_daily"].compiled_code = pivot - chain.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(paid_total) as top from fct_daily", ["top"]) + chain.nodes["model.shop.rpt_ltv"] = model( + "rpt_ltv", + ["model.shop.fct_daily"], + "select max(paid_total) as top from fct_daily", + ["top"], + ) const spy = answer((sql) => (sql === pivot ? { column_lineage: [], column_dict: {} } : direct(sql))) try { const reach = columnReach(manifestFile(chain), "fct_orders", "revenue") @@ -410,12 +576,23 @@ describe("columnReach on a model the engine cannot fully trace", () => { value.nodes["model.shop.fct_daily"].compiled_code = "select 3" value.nodes["model.shop.rpt_counts"].compiled_code = "select 4" // Its SQL never names the column: only its partly traced parent makes it a possible reader. - value.nodes["model.shop.rpt_ltv"] = model("rpt_ltv", ["model.shop.fct_daily"], "select max(total) as top from fct_daily", ["top"]) + value.nodes["model.shop.rpt_ltv"] = model( + "rpt_ltv", + ["model.shop.fct_daily"], + "select max(total) as top from fct_daily", + ["top"], + ) const spy = answer((sql) => sql === "select 3" ? { ...direct(sql), errors: ["partial"] } : sql === "select 4" - ? { column_lineage: [{ source: '"shop"."main"."fct_orders"."order_id"', target: "orders", lineage_type: "direct" }], column_dict: {}, errors: ["partial"] } + ? { + column_lineage: [ + { source: '"shop"."main"."fct_orders"."order_id"', target: "orders", lineage_type: "direct" }, + ], + column_dict: {}, + errors: ["partial"], + } : direct(sql), ) try { @@ -425,7 +602,9 @@ describe("columnReach on a model the engine cannot fully trace", () => { expect(reach.uncertain.map((m) => m.name)).toContain("rpt_ltv") expect(reach.models.map((m) => m.name)).not.toContain("rpt_ltv") // Its own child is checked though fct_daily carried the column on. - expect(reach.uncertain.find((m) => m.name === "rpt_counts")?.reason).toBe("the lineage engine traced only part of its SQL") + expect(reach.uncertain.find((m) => m.name === "rpt_counts")?.reason).toBe( + "the lineage engine traced only part of its SQL", + ) expect(reach.uncertain_exposures.map((e) => e.unique_id)).toContain("exposure.shop.ops") expect(reach.exposures.map((e) => e.unique_id)).not.toContain("exposure.shop.ops") } finally { @@ -442,23 +621,48 @@ describe("columnReach on SQL the engine cannot parse", () => { expect(reach.untraced.map((m) => m.name)).toEqual(["fct_daily"]) expect(reach.untraced[0]!.reason).toContain("Parse error") expect(reach.exposures).toEqual([]) - expect(reach.uncertain_exposures).toEqual([{ unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }]) + expect(reach.uncertain_exposures).toEqual([ + { unique_id: "exposure.shop.exec_revenue", name: "exec_revenue", label: "Exec Revenue", type: "dashboard" }, + ]) }) }) describe("lineageGraph", () => { test("a twelve-hop chain comes back whole to its source; upstream past 500 nodes is cut and marked", () => { const chain: Record = {} - for (let i = 1; i <= 12; i++) chain[`model.shop.m${i}`] = model(`m${i}`, [i === 1 ? "source.shop.raw.orders" : `model.shop.m${i - 1}`], "select 1", []) - chain["test.shop.m3_unique"] = { resource_type: "test", name: "m3_unique", depends_on: { nodes: ["model.shop.m3"] } } - const graph = lineageGraph({ nodes: chain, sources: { "source.shop.raw.orders": { resource_type: "source", name: "orders", source_name: "raw" } } }, "model.shop.m12") + for (let i = 1; i <= 12; i++) + chain[`model.shop.m${i}`] = model( + `m${i}`, + [i === 1 ? "source.shop.raw.orders" : `model.shop.m${i - 1}`], + "select 1", + [], + ) + chain["test.shop.m3_unique"] = { + resource_type: "test", + name: "m3_unique", + depends_on: { nodes: ["model.shop.m3"] }, + } + const graph = lineageGraph( + { + nodes: chain, + sources: { "source.shop.raw.orders": { resource_type: "source", name: "orders", source_name: "raw" } }, + }, + "model.shop.m12", + ) // An upstream model's tests are left out: only the target's and downstream tests are counted. expect(graph.nodes["test.shop.m3_unique"]).toBeUndefined() expect(graph.nodes["source.shop.raw.orders"]).toBeDefined() expect(Object.keys(graph.nodes)).toHaveLength(13) expect(graph.upTruncated).toBe(false) - const wide: Record = { "model.shop.top": model("top", Array.from({ length: 600 }, (_, i) => `model.shop.p${i}`), "select 1", []) } + const wide: Record = { + "model.shop.top": model( + "top", + Array.from({ length: 600 }, (_, i) => `model.shop.p${i}`), + "select 1", + [], + ), + } for (let i = 0; i < 600; i++) wide[`model.shop.p${i}`] = model(`p${i}`, [], "select 1", []) const cut = lineageGraph({ nodes: wide }, "model.shop.top") expect(cut.upTruncated).toBe(true) @@ -468,20 +672,32 @@ describe("lineageGraph", () => { test("downstream past 500 nodes is cut and marked; downstream tests are still counted", () => { const wide: Record = { "model.shop.top": model("top", [], "select 1", []) } for (let i = 0; i < 600; i++) wide[`model.shop.c${i}`] = model(`c${i}`, ["model.shop.top"], "select 1", []) - wide["test.shop.top_unique"] = { resource_type: "test", name: "top_unique", depends_on: { nodes: ["model.shop.top"] } } + wide["test.shop.top_unique"] = { + resource_type: "test", + name: "top_unique", + depends_on: { nodes: ["model.shop.top"] }, + } wide["test.shop.c0_unique"] = { resource_type: "test", name: "c0_unique", depends_on: { nodes: ["model.shop.c0"] } } const cut = lineageGraph({ nodes: wide }, "model.shop.top") expect(cut.truncated).toBe(true) expect(cut.upTruncated).toBe(false) const kept = Object.values(cut.nodes) expect(kept.filter((n) => n.type !== "test")).toHaveLength(501) - expect(kept.filter((n) => n.type === "test").map((n) => n.id).sort()).toEqual(["test.shop.c0_unique", "test.shop.top_unique"]) + expect( + kept + .filter((n) => n.type === "test") + .map((n) => n.id) + .sort(), + ).toEqual(["test.shop.c0_unique", "test.shop.top_unique"]) }) test("everything up and downstream, tests as ids, from dbt's child_map when present", () => { const value = { ...manifest(), - nodes: { ...manifest().nodes, "test.shop.t1": { resource_type: "test", name: "t1", depends_on: { nodes: ["model.shop.fct_daily"] } } }, + nodes: { + ...manifest().nodes, + "test.shop.t1": { resource_type: "test", name: "t1", depends_on: { nodes: ["model.shop.fct_daily"] } }, + }, } const graph = lineageGraph(value, "model.shop.mart_ltv") expect(graph.parents["model.shop.mart_ltv"]).toEqual(["model.shop.fct_orders"]) @@ -497,7 +713,14 @@ describe("lineageGraph", () => { columns: ["order_id", "ltv"], }) expect(graph.truncated).toBe(false) - const fromFct = lineageGraph({ ...value, child_map: { "model.shop.fct_orders": ["model.shop.fct_daily", "test.shop.t1"], "model.shop.fct_daily": [] }, parent_map: { "model.shop.fct_orders": [], "test.shop.t1": ["model.shop.fct_orders"] } }, "model.shop.fct_orders") + const fromFct = lineageGraph( + { + ...value, + child_map: { "model.shop.fct_orders": ["model.shop.fct_daily", "test.shop.t1"], "model.shop.fct_daily": [] }, + parent_map: { "model.shop.fct_orders": [], "test.shop.t1": ["model.shop.fct_orders"] }, + }, + "model.shop.fct_orders", + ) expect(Object.keys(fromFct.nodes).sort()).toEqual(["model.shop.fct_daily", "model.shop.fct_orders", "test.shop.t1"]) expect(fromFct.nodes["test.shop.t1"]!.type).toBe("test") expect(fromFct.parents["test.shop.t1"]).toEqual(["model.shop.fct_orders"]) @@ -522,15 +745,22 @@ describe("structuredError", () => { expect(Buffer.byteLength(out.report as string)).toBeLessThanOrEqual(60_000) }) test("a tool's own structured result sends the report's header, not its edge lists", () => { - const text = "Model: x (model.a.x) · table · confidence high\n\nUpstream (0):\n none\n\nColumn edges (1):\n x.a ← y.a\n\nNotes: n" + const text = + "Model: x (model.a.x) · table · confidence high\n\nUpstream (0):\n none\n\nColumn edges (1):\n x.a ← y.a\n\nNotes: n" expect(structuredResult("dbt_lineage", { structuredContent: { columnLineage: [1] } }, text)).toEqual({ columnLineage: [1], report: "Model: x (model.a.x) · table · confidence high\n\nUpstream (0):\n none", }) - expect((structuredResult("dbt_lineage", { structuredContent: {} }, "Model: x\n\nTrace of x.a (1 edges):\n e") as any).report).toBe("Model: x") + expect( + (structuredResult("dbt_lineage", { structuredContent: {} }, "Model: x\n\nTrace of x.a (1 edges):\n e") as any) + .report, + ).toBe("Model: x") }) test("exposures_error reaches the structured fields", () => { - expect(structuredResult("impact_analysis", { exposures_error: "bad json" }, "R")).toEqual({ exposures_error: "bad json", report: "R" }) + expect(structuredResult("impact_analysis", { exposures_error: "bad json" }, "R")).toEqual({ + exposures_error: "bad json", + report: "R", + }) }) }) @@ -553,10 +783,11 @@ describe("structuredError escaping", () => { }) }) - describe("structuredResult", () => { test("passes the lineage fields with the report, and only the report when they are too big", () => { - expect(structuredResult("impact_analysis", { severity: "LOW", findings: [{ category: "x" }], downstream: [] }, "Impact")).toEqual({ + expect( + structuredResult("impact_analysis", { severity: "LOW", findings: [{ category: "x" }], downstream: [] }, "Impact"), + ).toEqual({ severity: "LOW", downstream: [], report: "Impact", @@ -564,13 +795,21 @@ describe("structuredResult", () => { const huge = { edges: Array.from({ length: 5000 }, (_, i) => ({ source: `"t"."c${i}"`, target: `c${i}` })) } expect(structuredResult("dbt_lineage", huge, "Model: x")).toEqual({ report: "Model: x", truncated: true }) // dbt_lineage's text repeats the columns and edges: only its header goes with them. - expect(structuredResult("dbt_lineage", { columns: {}, edges: [], graph: { nodes: {} } }, "Model: x\nID: y\n\nColumn Mappings:\n a ← [\"t.a\"]")).toEqual({ + expect( + structuredResult( + "dbt_lineage", + { columns: {}, edges: [], graph: { nodes: {} } }, + 'Model: x\nID: y\n\nColumn Mappings:\n a ← ["t.a"]', + ), + ).toEqual({ columns: {}, edges: [], report: "Model: x\nID: y", }) // With no column_dict the text has no Column Mappings section: the edges are cut at their own header. - expect(structuredResult("dbt_lineage", { columns: {}, edges: [] }, "Model: x\nID: y\n\nLineage Edges:\n {}")).toEqual({ + expect( + structuredResult("dbt_lineage", { columns: {}, edges: [] }, "Model: x\nID: y\n\nLineage Edges:\n {}"), + ).toEqual({ columns: {}, edges: [], report: "Model: x\nID: y", @@ -584,23 +823,45 @@ describe("dbtLineage", () => { const result = dbtLineage({ manifest_path: file, model: "mart_ltv" }) expect(result.upstream).toEqual(["model.shop.fct_orders"]) expect(result.graph).toBeUndefined() - expect(Object.keys(dbtLineage({ manifest_path: file, model: "mart_ltv", include_graph: true }).graph!.nodes)).toContain("model.shop.rpt_ltv") + expect( + Object.keys(dbtLineage({ manifest_path: file, model: "mart_ltv", include_graph: true }).graph!.nodes), + ).toContain("model.shop.rpt_ltv") // By project-relative path too. - expect(dbtLineage({ manifest_path: file, model: "models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + expect(dbtLineage({ manifest_path: file, model: "models/mart_ltv.sql" }).model_unique_id).toBe( + "model.shop.mart_ltv", + ) // `./` and Windows separators spell the same path. - expect(dbtLineage({ manifest_path: file, model: "./models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") - expect(dbtLineage({ manifest_path: file, model: "models\\mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv") + expect(dbtLineage({ manifest_path: file, model: "./models/mart_ltv.sql" }).model_unique_id).toBe( + "model.shop.mart_ltv", + ) + expect(dbtLineage({ manifest_path: file, model: "models\\mart_ltv.sql" }).model_unique_id).toBe( + "model.shop.mart_ltv", + ) // An empty path is no model's, not the first one's without a file path. const pathless = JSON.parse(readFileSync(file, "utf8")) delete pathless.nodes["model.shop.fct_orders"].original_file_path - expect(dbtLineage({ manifest_path: manifestFile(pathless), model: "./" }).confidence_factors).toContain("Model './' not found in manifest") + expect(dbtLineage({ manifest_path: manifestFile(pathless), model: "./" }).confidence_factors).toContain( + "Model './' not found in manifest", + ) const withSnapshot: any = manifest() - withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" } - expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap") + withSnapshot.nodes["snapshot.shop.snap"] = { + ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), + resource_type: "snapshot", + unique_id: "snapshot.shop.snap", + } + expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe( + "snapshot.shop.snap", + ) // A snapshot named like another node's unique_id keeps its own id. withSnapshot.nodes["test.shop.t1"] = { resource_type: "test", name: "t1", depends_on: { nodes: [] } } - withSnapshot.nodes["snapshot.shop.odd"] = { ...withSnapshot.nodes["snapshot.shop.snap"], unique_id: "snapshot.shop.odd", name: "test.shop.t1" } - expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "test.shop.t1" }).model_unique_id).toBe("snapshot.shop.odd") + withSnapshot.nodes["snapshot.shop.odd"] = { + ...withSnapshot.nodes["snapshot.shop.snap"], + unique_id: "snapshot.shop.odd", + name: "test.shop.t1", + } + expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "test.shop.t1" }).model_unique_id).toBe( + "snapshot.shop.odd", + ) }) }) @@ -615,8 +876,14 @@ describe("columnReach with a NATURAL JOIN", () => { test("a model that reads the column through a NATURAL JOIN passes the doubt on to its children", () => { const value = manifest() - value.nodes["model.shop.fct_daily"].compiled_code = "select sum(revenue) as revenue from fct_orders natural left join other" - value.nodes["model.shop.rpt_after"] = model("rpt_after", ["model.shop.fct_daily"], "select count(*) as n from fct_daily", ["n"]) + value.nodes["model.shop.fct_daily"].compiled_code = + "select sum(revenue) as revenue from fct_orders natural left join other" + value.nodes["model.shop.rpt_after"] = model( + "rpt_after", + ["model.shop.fct_daily"], + "select count(*) as n from fct_daily", + ["n"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.models.map((m) => m.name)).toContain("fct_daily") expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") @@ -659,8 +926,14 @@ describe("columnReach with a NATURAL JOIN", () => { test("a NATURAL JOIN model that names the column outside its select list also passes the doubt on", () => { const value = manifest() - value.nodes["model.shop.rpt_big_orders"].compiled_code = "select a.order_id from fct_orders a natural join other b where a.revenue > 100" - value.nodes["model.shop.rpt_after"] = model("rpt_after", ["model.shop.rpt_big_orders"], "select count(*) as n from rpt_big_orders", ["n"]) + value.nodes["model.shop.rpt_big_orders"].compiled_code = + "select a.order_id from fct_orders a natural join other b where a.revenue > 100" + value.nodes["model.shop.rpt_after"] = model( + "rpt_after", + ["model.shop.rpt_big_orders"], + "select count(*) as n from rpt_big_orders", + ["n"], + ) const reach = columnReach(manifestFile(value), "fct_orders", "revenue") expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") }) From 3c6774bf8f0ce26b06f689a51e45d8ffe7174f6d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:30:29 +0530 Subject: [PATCH 30/37] fix: a column named through Snowflake IDENTIFIER() is a possible read, and a column reach that stopped early reports lower bounds - IDENTIFIER('col') with a traced column, or a bind variable, makes the model uncertain and passes the doubt on, as a NATURAL JOIN does - the text report says "at least" when column reach stopped early, and the exposures past the stop are unchecked, not non-readers - the model parameter says a snapshot is found by name too Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 36 ++++++++++++++++--- .../src/altimate/tools/dbt-lineage.ts | 2 +- .../src/altimate/tools/impact-analysis.ts | 6 ++-- .../test/altimate/column-reach.test.ts | 31 ++++++++++++++++ .../altimate/tools/impact-analysis.test.ts | 2 +- 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index aeb91e78ac..a086e1b26f 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -196,6 +196,21 @@ export function joinsNaturally(sql: string): boolean { return false } +/** + * Whether the SQL names one of `columns` through Snowflake's `IDENTIFIER('…')`, or passes it a bind + * variable (`IDENTIFIER($col)`): the lineage engine reads the argument as a string, so it reports no + * edge for the column. + */ +export function namesThroughIdentifier(sql: string, columns: string[]): boolean { + const wanted = new Set(columns.map((one) => one.toLowerCase())) + for (const match of sql.matchAll(/\bidentifier\s*\(\s*(?:'((?:[^']|'')*)'|([$:?]))/gi)) { + if (match[2]) return true + const name = match[1]!.replace(/''/g, "'").split(".").pop()!.trim().replace(/^"|"$/g, "") + if (wanted.has(name.toLowerCase())) return true + } + return false +} + function childMapOf(manifest: any): Record { if (manifest.child_map) return manifest.child_map const children: Record = {} @@ -353,12 +368,18 @@ export function columnReach(manifestPath: string, model: string, column: string, // edge through it: unknown, not unread, even when another parent's edges are traced. const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") const natural = joinsNaturally(sqlText) + // The column named as a string the engine cannot trace: its outputs are unknown, like a NATURAL JOIN's. + const viaIdentifier = namesThroughIdentifier( + sqlText, + parents.flatMap((parent) => [...tracked.get(parent)!]), + ) + const opaque = natural || viaIdentifier if (reads) { tracked.set(id, carried) trackedExact.set(id, carriedExact) // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. // A NATURAL JOIN beside the column may join on it, or carry it on in another output. - if (ambiguous || fromUnsure || viaEphemeral || partial || natural) unsure.add(id) + if (ambiguous || fromUnsure || viaEphemeral || partial || opaque) unsure.add(id) reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) continue } @@ -383,16 +404,23 @@ export function columnReach(manifestPath: string, model: string, column: string, continue } // A NATURAL JOIN may join on the column with no edge the engine can report. - if (!fromUnsure && !ambiguous && named.length === 0 && natural) { + if (!fromUnsure && !ambiguous && named.length === 0 && opaque) { unsure.add(id) - uncertain.push(entry(id, "uses a NATURAL JOIN, whose join columns the lineage engine cannot trace")) + uncertain.push( + entry( + id, + viaIdentifier + ? "names it through IDENTIFIER(), which the lineage engine cannot trace" + : "uses a NATURAL JOIN, whose join columns the lineage engine cannot trace", + ), + ) continue } const shared = named.some( (col) => allParents.filter((other) => documentedColumns(lookup(other)).has(col)).length > 1, ) if (ambiguous || named.length > 0 || fromUnsure) { - if (ambiguous || fromUnsure || natural) unsure.add(id) + if (ambiguous || fromUnsure || opaque) unsure.add(id) uncertain.push( entry( id, diff --git a/packages/opencode/src/altimate/tools/dbt-lineage.ts b/packages/opencode/src/altimate/tools/dbt-lineage.ts index 4d4029a4e0..7b27dd2a7e 100644 --- a/packages/opencode/src/altimate/tools/dbt-lineage.ts +++ b/packages/opencode/src/altimate/tools/dbt-lineage.ts @@ -41,7 +41,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", { parameters: z.object({ model: z .string() - .describe("Model name, unique_id or project-relative SQL path (e.g. 'my_model', 'model.project.my_model', 'models/my_model.sql'); a snapshot by unique_id or path"), + .describe("Model name, unique_id or project-relative SQL path (e.g. 'my_model', 'model.project.my_model', 'models/my_model.sql'); a snapshot by name, unique_id or path"), column: z.string().optional().describe("Trace only this column of the model, upstream and downstream"), manifest_path: z .string() diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index 02b579d23a..3b3ce5475d 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -280,7 +280,8 @@ export function formatImpactReport(data: { const reach = data.columnReach const dashboards = reach.exposures.length lines.push( - `Column Reach: ${data.model}.${data.column} is read by ${reach.models.length} model${reach.models.length === 1 ? "" : "s"} and ${dashboards} exposure${dashboards === 1 ? "" : "s"}` + + // Stopped early: the counts are lower bounds, and exposures past the stop may read the column too. + `Column Reach: ${data.model}.${data.column} is read by ${reach.truncated ? "at least " : ""}${reach.models.length} model${reach.models.length === 1 ? "" : "s"} and ${reach.truncated ? "at least " : ""}${dashboards} exposure${dashboards === 1 ? "" : "s"}` + `${reach.uncertain.length ? `; ${reach.uncertain.length} more may read it` : ""}` + `${reach.untraced.length ? `; ${reach.untraced.length} could not be traced` : ""}${reach.truncated ? " (stopped early)" : ""}`, ) @@ -304,7 +305,8 @@ export function formatImpactReport(data: { ) const others = (data.exposures ?? []).filter((e) => !listed.has(e.unique_id)) if (others.length > 0) { - lines.push(`${data.columnReach ? "Other exposures" : "Exposures"} (${others.length})`) + // When column reach stopped early, the rest are unchecked rather than known not to read the column. + lines.push(`${data.columnReach ? (data.columnReach.truncated ? "Other exposures (column reach stopped before checking them)" : "Other exposures") : "Exposures"} (${others.length})`) lines.push("".padEnd(40, "-")) for (const e of others) lines.push(` ${e.label} [${e.type}]`) lines.push("") diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index fbb65c6422..c48197ebfe 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -8,6 +8,7 @@ import { exposuresOn, parentsNamed, joinsNaturally, + namesThroughIdentifier, selectsStar, } from "../../src/altimate/native/dbt/column-reach" import { lineageGraph } from "../../src/altimate/native/dbt/graph" @@ -938,3 +939,33 @@ describe("columnReach with a NATURAL JOIN", () => { expect(reach.uncertain.map((m) => m.name)).toContain("rpt_after") }) }) + +describe("columnReach through Snowflake IDENTIFIER()", () => { + test("a column named as IDENTIFIER('…') is a possible read, and its child and dashboard are not dropped", () => { + const value = manifest() + value.nodes["model.shop.rpt_counts"].compiled_code = "select identifier('revenue') as copied from fct_orders" + value.nodes["model.shop.rpt_copy"] = model("rpt_copy", ["model.shop.rpt_counts"], "select copied from rpt_counts", [ + "copied", + ]) + const exposures: Record = value.exposures + exposures["exposure.shop.copy_board"] = { + name: "copy_board", + type: "dashboard", + depends_on: { nodes: ["model.shop.rpt_copy"] }, + } + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["rpt_counts", "rpt_copy"])) + expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(expect.arrayContaining(["ops", "copy_board"])) + }) + + test("IDENTIFIER() with the column, quoted or qualified, or a bind variable counts; another column does not", () => { + for (const sql of [ + "select identifier('revenue') from t", + `select IDENTIFIER('o."REVENUE"') from t`, + "select identifier($col) from t", + "select identifier(:col) from t", + ]) + expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) + expect(namesThroughIdentifier("select identifier('order_id') from t", ["revenue"])).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/tools/impact-analysis.test.ts b/packages/opencode/test/altimate/tools/impact-analysis.test.ts index 7ecf843b00..379bbc53d4 100644 --- a/packages/opencode/test/altimate/tools/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/tools/impact-analysis.test.ts @@ -219,7 +219,7 @@ describe("formatImpactReport", () => { }, }) expect(report).toContain( - "Column Reach: fct_orders.revenue is read by 1 model and 1 exposure; 1 more may read it; 1 could not be traced (stopped early)", + "Column Reach: fct_orders.revenue is read by at least 1 model and at least 1 exposure; 1 more may read it; 1 could not be traced (stopped early)", ) expect(report).toContain(" fct_daily → revenue") expect(report).toContain(" ◆ Exec Revenue [dashboard]") From 4123f0ce802d20c12c5928e2e73ed682ed4611b3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:35:34 +0530 Subject: [PATCH 31/37] fix: comments before or inside IDENTIFIER( are read as gaps, and its lookup is bounded like the NATURAL JOIN one Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 16 ++++++++++++---- .../opencode/test/altimate/column-reach.test.ts | 5 ++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index a086e1b26f..e0596193e9 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -196,16 +196,24 @@ export function joinsNaturally(sql: string): boolean { return false } +// `IDENTIFIER (`, comments allowed before and inside the parenthesis, then its string or bind variable. +const IDENTIFIER_CALL = new RegExp(String.raw`^identifier(?:${GAP})?\((?:${GAP})?(?:'((?:[^']|'')*)'|([$:?]))`, "i") + /** * Whether the SQL names one of `columns` through Snowflake's `IDENTIFIER('…')`, or passes it a bind * variable (`IDENTIFIER($col)`): the lineage engine reads the argument as a string, so it reports no - * edge for the column. + * edge for the column. Bounded like joinsNaturally: each `identifier` is read in a window, and past + * 100 of them the SQL counts as naming it. */ export function namesThroughIdentifier(sql: string, columns: string[]): boolean { const wanted = new Set(columns.map((one) => one.toLowerCase())) - for (const match of sql.matchAll(/\bidentifier\s*\(\s*(?:'((?:[^']|'')*)'|([$:?]))/gi)) { - if (match[2]) return true - const name = match[1]!.replace(/''/g, "'").split(".").pop()!.trim().replace(/^"|"$/g, "") + let seen = 0 + for (const match of sql.matchAll(/(? MAX_NATURALS) return true + const call = IDENTIFIER_CALL.exec(sql.slice(match.index!, match.index! + NATURAL_WINDOW)) + if (!call) continue + if (call[2]) return true + const name = call[1]!.replace(/''/g, "'").split(".").pop()!.trim().replace(/^"|"$/g, "") if (wanted.has(name.toLowerCase())) return true } return false diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index c48197ebfe..e772acb840 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -943,7 +943,8 @@ describe("columnReach with a NATURAL JOIN", () => { describe("columnReach through Snowflake IDENTIFIER()", () => { test("a column named as IDENTIFIER('…') is a possible read, and its child and dashboard are not dropped", () => { const value = manifest() - value.nodes["model.shop.rpt_counts"].compiled_code = "select identifier('revenue') as copied from fct_orders" + value.nodes["model.shop.rpt_counts"].compiled_code = + "select identifier(/* source column */ 'revenue') as copied from fct_orders" value.nodes["model.shop.rpt_copy"] = model("rpt_copy", ["model.shop.rpt_counts"], "select copied from rpt_counts", [ "copied", ]) @@ -962,6 +963,8 @@ describe("columnReach through Snowflake IDENTIFIER()", () => { for (const sql of [ "select identifier('revenue') from t", `select IDENTIFIER('o."REVENUE"') from t`, + "select identifier(/* source column */ 'revenue') as copied from t", + "select identifier /* c */ ( -- why\n 'revenue') from t", "select identifier($col) from t", "select identifier(:col) from t", ]) From 0ac91d3abf50ba896cdf7c93586dd1467227159d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:37:45 +0530 Subject: [PATCH 32/37] =?UTF-8?q?fix:=20any=20mention=20of=20a=20traced=20?= =?UTF-8?q?column=20inside=20IDENTIFIER('=E2=80=A6')=20counts,=20so=20a=20?= =?UTF-8?q?quoted=20name=20with=20a=20dot=20is=20never=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/column-reach.ts | 6 ++++-- packages/opencode/test/altimate/column-reach.test.ts | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index e0596193e9..b61a537a83 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -213,8 +213,10 @@ export function namesThroughIdentifier(sql: string, columns: string[]): boolean const call = IDENTIFIER_CALL.exec(sql.slice(match.index!, match.index! + NATURAL_WINDOW)) if (!call) continue if (call[2]) return true - const name = call[1]!.replace(/''/g, "'").split(".").pop()!.trim().replace(/^"|"$/g, "") - if (wanted.has(name.toLowerCase())) return true + // Any mention of the column in the string counts (qualified, quoted, with dots in a quoted name): + // telling those apart needs a parse, and a wider match only adds doubt. + const text = call[1]!.replace(/''/g, "'").toLowerCase() + if ([...wanted].some((column) => text.includes(column))) return true } return false } diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index e772acb840..a222a94ddd 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -964,11 +964,14 @@ describe("columnReach through Snowflake IDENTIFIER()", () => { "select identifier('revenue') from t", `select IDENTIFIER('o."REVENUE"') from t`, "select identifier(/* source column */ 'revenue') as copied from t", + `select identifier('o."net.revenue"') from t`, "select identifier /* c */ ( -- why\n 'revenue') from t", "select identifier($col) from t", "select identifier(:col) from t", ]) expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) + // A tracked column whose quoted name holds a dot. + expect(namesThroughIdentifier(`select identifier('"net.amount"') from t`, ["net.amount"])).toBe(true) expect(namesThroughIdentifier("select identifier('order_id') from t", ["revenue"])).toBe(false) }) }) From 72cc52daf6256fd3d05c64b88b3ea19b41eb2094 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:40:04 +0530 Subject: [PATCH 33/37] fix: a doubled quote inside an IDENTIFIER() name is read as one quote Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/native/dbt/column-reach.ts | 3 ++- packages/opencode/test/altimate/column-reach.test.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index b61a537a83..347d5c197a 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -215,7 +215,8 @@ export function namesThroughIdentifier(sql: string, columns: string[]): boolean if (call[2]) return true // Any mention of the column in the string counts (qualified, quoted, with dots in a quoted name): // telling those apart needs a parse, and a wider match only adds doubt. - const text = call[1]!.replace(/''/g, "'").toLowerCase() + // A doubled quote inside a quoted name (`"a""b"`) is one quote in the name. + const text = call[1]!.replace(/''/g, "'").replace(/""/g, '"').toLowerCase() if ([...wanted].some((column) => text.includes(column))) return true } return false diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index a222a94ddd..fc8a21e837 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -972,6 +972,8 @@ describe("columnReach through Snowflake IDENTIFIER()", () => { expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) // A tracked column whose quoted name holds a dot. expect(namesThroughIdentifier(`select identifier('"net.amount"') from t`, ["net.amount"])).toBe(true) + // A quote in a column's name, doubled inside the quoted identifier. + expect(namesThroughIdentifier(`select identifier('"a""b"') from t`, ['a"b'])).toBe(true) expect(namesThroughIdentifier("select identifier('order_id') from t", ["revenue"])).toBe(false) }) }) From 7890152ef222bfad8f36329962506da00ed65385 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:45:04 +0530 Subject: [PATCH 34/37] fix: an IDENTIFIER() string that is not a plain name counts as naming the column, so no escape can hide one Escapes (\x72), backslash-escaped quotes and other characters are not decoded; such a string marks the model uncertain, which only adds doubt. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 8 +++- .../test/altimate/column-reach.test.ts | 41 ++++++++++++------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 347d5c197a..100975ac03 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -197,7 +197,10 @@ export function joinsNaturally(sql: string): boolean { } // `IDENTIFIER (`, comments allowed before and inside the parenthesis, then its string or bind variable. -const IDENTIFIER_CALL = new RegExp(String.raw`^identifier(?:${GAP})?\((?:${GAP})?(?:'((?:[^']|'')*)'|([$:?]))`, "i") +const IDENTIFIER_CALL = new RegExp( + String.raw`^identifier(?:${GAP})?\((?:${GAP})?(?:'((?:[^'\\]|''|\\[\s\S])*)'|([$:?]))`, + "i", +) /** * Whether the SQL names one of `columns` through Snowflake's `IDENTIFIER('…')`, or passes it a bind @@ -215,6 +218,9 @@ export function namesThroughIdentifier(sql: string, columns: string[]): boolean if (call[2]) return true // Any mention of the column in the string counts (qualified, quoted, with dots in a quoted name): // telling those apart needs a parse, and a wider match only adds doubt. + // A string that is not a plain name (an escape such as `\x72`, any other character) is not decoded: + // it counts, which only adds doubt. + if (!/^[\w$."]*$/.test(call[1]!.replace(/''/g, "'"))) return true // A doubled quote inside a quoted name (`"a""b"`) is one quote in the name. const text = call[1]!.replace(/''/g, "'").replace(/""/g, '"').toLowerCase() if ([...wanted].some((column) => text.includes(column))) return true diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index fc8a21e837..093bfedf5a 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -941,22 +941,26 @@ describe("columnReach with a NATURAL JOIN", () => { }) describe("columnReach through Snowflake IDENTIFIER()", () => { - test("a column named as IDENTIFIER('…') is a possible read, and its child and dashboard are not dropped", () => { - const value = manifest() - value.nodes["model.shop.rpt_counts"].compiled_code = - "select identifier(/* source column */ 'revenue') as copied from fct_orders" - value.nodes["model.shop.rpt_copy"] = model("rpt_copy", ["model.shop.rpt_counts"], "select copied from rpt_counts", [ - "copied", - ]) - const exposures: Record = value.exposures - exposures["exposure.shop.copy_board"] = { - name: "copy_board", - type: "dashboard", - depends_on: { nodes: ["model.shop.rpt_copy"] }, + test("a column named as IDENTIFIER('…'), plain or escaped, is a possible read, and its child and dashboard are not dropped", () => { + for (const call of ["identifier(/* source column */ 'revenue')", String.raw`identifier('\x72evenue')`]) { + const value = manifest() + value.nodes["model.shop.rpt_counts"].compiled_code = `select ${call} as copied from fct_orders` + value.nodes["model.shop.rpt_copy"] = model( + "rpt_copy", + ["model.shop.rpt_counts"], + "select copied from rpt_counts", + ["copied"], + ) + const exposures: Record = value.exposures + exposures["exposure.shop.copy_board"] = { + name: "copy_board", + type: "dashboard", + depends_on: { nodes: ["model.shop.rpt_copy"] }, + } + const reach = columnReach(manifestFile(value), "fct_orders", "revenue") + expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["rpt_counts", "rpt_copy"])) + expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(expect.arrayContaining(["ops", "copy_board"])) } - const reach = columnReach(manifestFile(value), "fct_orders", "revenue") - expect(reach.uncertain.map((m) => m.name)).toEqual(expect.arrayContaining(["rpt_counts", "rpt_copy"])) - expect(reach.uncertain_exposures.map((e) => e.name)).toEqual(expect.arrayContaining(["ops", "copy_board"])) }) test("IDENTIFIER() with the column, quoted or qualified, or a bind variable counts; another column does not", () => { @@ -972,6 +976,13 @@ describe("columnReach through Snowflake IDENTIFIER()", () => { expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) // A tracked column whose quoted name holds a dot. expect(namesThroughIdentifier(`select identifier('"net.amount"') from t`, ["net.amount"])).toBe(true) + // An escape, or any text that is not a plain name, is not decoded: it counts. + for (const sql of [ + String.raw`select identifier('\x72evenue') from t`, + String.raw`select identifier('it\'s.revenue') from t`, + `select identifier('"my col"') from t`, + ]) + expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) // A quote in a column's name, doubled inside the quoted identifier. expect(namesThroughIdentifier(`select identifier('"a""b"') from t`, ['a"b'])).toBe(true) expect(namesThroughIdentifier("select identifier('order_id') from t", ["revenue"])).toBe(false) From 98c582c26700e96424daba159be2183faf71e528 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 18:48:55 +0530 Subject: [PATCH 35/37] fix: only an IDENTIFIER() whose whole argument is one plain string naming another column is cleared Concatenations, bind and session variables, functions and unterminated strings all count as possibly naming the column, so no argument form can produce a confident zero. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 29 ++++++++++--------- .../test/altimate/column-reach.test.ts | 12 ++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index 100975ac03..d3ad5af94f 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -196,17 +196,19 @@ export function joinsNaturally(sql: string): boolean { return false } -// `IDENTIFIER (`, comments allowed before and inside the parenthesis, then its string or bind variable. +// `IDENTIFIER (` with comments allowed around its parts; when its whole argument is one string +// literal, that string (group 1) and the closing parenthesis (group 2). const IDENTIFIER_CALL = new RegExp( - String.raw`^identifier(?:${GAP})?\((?:${GAP})?(?:'((?:[^'\\]|''|\\[\s\S])*)'|([$:?]))`, + String.raw`^identifier(?:${GAP})?\((?:${GAP})?(?:'((?:[^'\\]|''|\\[\s\S])*)'(?:${GAP})?(\))?)?`, "i", ) /** - * Whether the SQL names one of `columns` through Snowflake's `IDENTIFIER('…')`, or passes it a bind - * variable (`IDENTIFIER($col)`): the lineage engine reads the argument as a string, so it reports no - * edge for the column. Bounded like joinsNaturally: each `identifier` is read in a window, and past - * 100 of them the SQL counts as naming it. + * Whether the SQL may name one of `columns` through `IDENTIFIER(…)` (Snowflake, Databricks): the + * lineage engine reads the argument as a string, so it reports no edge for the column. Only an + * argument that is one plain string literal naming none of them is cleared; anything else (a bind or + * session variable, a concatenation, a function, an escape) counts, which only adds doubt. Bounded like + * joinsNaturally: each `identifier` is read in a window, and past 100 of them the SQL counts. */ export function namesThroughIdentifier(sql: string, columns: string[]): boolean { const wanted = new Set(columns.map((one) => one.toLowerCase())) @@ -214,15 +216,14 @@ export function namesThroughIdentifier(sql: string, columns: string[]): boolean for (const match of sql.matchAll(/(? MAX_NATURALS) return true const call = IDENTIFIER_CALL.exec(sql.slice(match.index!, match.index! + NATURAL_WINDOW)) + // `identifier` not called is a name, not the function. if (!call) continue - if (call[2]) return true - // Any mention of the column in the string counts (qualified, quoted, with dots in a quoted name): - // telling those apart needs a parse, and a wider match only adds doubt. - // A string that is not a plain name (an escape such as `\x72`, any other character) is not decoded: - // it counts, which only adds doubt. - if (!/^[\w$."]*$/.test(call[1]!.replace(/''/g, "'"))) return true - // A doubled quote inside a quoted name (`"a""b"`) is one quote in the name. - const text = call[1]!.replace(/''/g, "'").replace(/""/g, '"').toLowerCase() + if (call[1] === undefined || !call[2]) return true + const literal = call[1].replace(/''/g, "'") + if (!/^[\w$."]*$/.test(literal)) return true + // Any mention of the column in the string counts (qualified, quoted, with dots in a quoted name); + // a doubled quote inside a quoted name (`"a""b"`) is one quote. + const text = literal.replace(/""/g, '"').toLowerCase() if ([...wanted].some((column) => text.includes(column))) return true } return false diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 093bfedf5a..9bc1fc6d13 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -976,6 +976,18 @@ describe("columnReach through Snowflake IDENTIFIER()", () => { expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) // A tracked column whose quoted name holds a dot. expect(namesThroughIdentifier(`select identifier('"net.amount"') from t`, ["net.amount"])).toBe(true) + // Anything but one plain string literal counts: concatenations, variables, functions. + for (const sql of [ + "select identifier('o.' || :col) from t", + "select identifier('rev' || 'enue') from t", + "select identifier(session.col_name) from t", + "select identifier(col_var) from t", + "select identifier(concat('rev', 'enue')) from t", + "select identifier('order_id'", + ]) + expect(namesThroughIdentifier(sql, ["revenue"])).toBe(true) + // `identifier` as a plain name is not the function. + expect(namesThroughIdentifier("select identifier, other from t", ["revenue"])).toBe(false) // An escape, or any text that is not a plain name, is not decoded: it counts. for (const sql of [ String.raw`select identifier('\x72evenue') from t`, From b84dd265c999afb1904fd45a2ce365197f0c9b00 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 19:00:43 +0530 Subject: [PATCH 36/37] fix: a column reach that stopped early bounds every count, and the other exposures are not all called unchecked - uncertain and untraced counts say "at least" too when column reach stopped early - the other exposures' header says some of them were not checked, after the count - the parents' traced columns are built once Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/dbt/column-reach.ts | 9 ++++---- .../src/altimate/tools/impact-analysis.ts | 21 ++++++++++++------- .../altimate/tools/impact-analysis.test.ts | 9 ++++++-- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-reach.ts b/packages/opencode/src/altimate/native/dbt/column-reach.ts index d3ad5af94f..b1de28984b 100644 --- a/packages/opencode/src/altimate/native/dbt/column-reach.ts +++ b/packages/opencode/src/altimate/native/dbt/column-reach.ts @@ -386,11 +386,10 @@ export function columnReach(manifestPath: string, model: string, column: string, // edge through it: unknown, not unread, even when another parent's edges are traced. const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") const natural = joinsNaturally(sqlText) + // The traced columns this model's parents carry. + const trackedColumns = parents.flatMap((parent) => [...tracked.get(parent)!]) // The column named as a string the engine cannot trace: its outputs are unknown, like a NATURAL JOIN's. - const viaIdentifier = namesThroughIdentifier( - sqlText, - parents.flatMap((parent) => [...tracked.get(parent)!]), - ) + const viaIdentifier = namesThroughIdentifier(sqlText, trackedColumns) const opaque = natural || viaIdentifier if (reads) { tracked.set(id, carried) @@ -412,7 +411,7 @@ export function columnReach(manifestPath: string, model: string, column: string, uncertain.push(entry(id, "reads an ephemeral parent, inlined as a CTE the lineage engine cannot trace through")) continue } - const named = parents.flatMap((parent) => [...tracked.get(parent)!].filter((col) => namesColumn(sqlText, col))) + const named = trackedColumns.filter((col) => namesColumn(sqlText, col)) // No edge at all while the SQL names the column (a PIVOT, say) is the engine not seeing through // it, so its outputs are unknown. SQL that never names it (`select 1`, `count(*)`) reads nothing and // falls through. Below an unsure parent it is already reported as uncertain. diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index 3b3ce5475d..b4d5758e37 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -279,12 +279,14 @@ export function formatImpactReport(data: { if (data.column && data.columnReach) { const reach = data.columnReach const dashboards = reach.exposures.length - lines.push( - // Stopped early: the counts are lower bounds, and exposures past the stop may read the column too. - `Column Reach: ${data.model}.${data.column} is read by ${reach.truncated ? "at least " : ""}${reach.models.length} model${reach.models.length === 1 ? "" : "s"} and ${reach.truncated ? "at least " : ""}${dashboards} exposure${dashboards === 1 ? "" : "s"}` + - `${reach.uncertain.length ? `; ${reach.uncertain.length} more may read it` : ""}` + - `${reach.untraced.length ? `; ${reach.untraced.length} could not be traced` : ""}${reach.truncated ? " (stopped early)" : ""}`, - ) + // Stopped early: every count is a lower bound, since the models past the stop were never traced. + const atLeast = reach.truncated ? "at least " : "" + const plural = (n: number, word: string) => `${atLeast}${n} ${word}${n === 1 ? "" : "s"}` + const readers = `${plural(reach.models.length, "model")} and ${plural(dashboards, "exposure")}` + const maybe = reach.uncertain.length ? `; ${atLeast}${reach.uncertain.length} more may read it` : "" + const lost = reach.untraced.length ? `; ${atLeast}${reach.untraced.length} could not be traced` : "" + const stopped = reach.truncated ? " (stopped early)" : "" + lines.push(`Column Reach: ${data.model}.${data.column} is read by ${readers}${maybe}${lost}${stopped}`) lines.push("".padEnd(40, "-")) for (const m of reach.models) lines.push(` ${m.name}${m.columns.length ? ` → ${m.columns.join(", ")}` : ""}`) for (const e of reach.exposures) lines.push(` ◆ ${e.label} [${e.type}]`) @@ -305,8 +307,11 @@ export function formatImpactReport(data: { ) const others = (data.exposures ?? []).filter((e) => !listed.has(e.unique_id)) if (others.length > 0) { - // When column reach stopped early, the rest are unchecked rather than known not to read the column. - lines.push(`${data.columnReach ? (data.columnReach.truncated ? "Other exposures (column reach stopped before checking them)" : "Other exposures") : "Exposures"} (${others.length})`) + // When column reach stopped early, some of these may sit on models it never reached. + const title = data.columnReach ? "Other exposures" : "Exposures" + const stoppedNote = ": column reach stopped early, so some of these were not checked" + const unchecked = data.columnReach?.truncated ? stoppedNote : "" + lines.push(`${title} (${others.length})${unchecked}`) lines.push("".padEnd(40, "-")) for (const e of others) lines.push(` ${e.label} [${e.type}]`) lines.push("") diff --git a/packages/opencode/test/altimate/tools/impact-analysis.test.ts b/packages/opencode/test/altimate/tools/impact-analysis.test.ts index 379bbc53d4..9aa7a5e5c6 100644 --- a/packages/opencode/test/altimate/tools/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/tools/impact-analysis.test.ts @@ -208,7 +208,10 @@ describe("formatImpactReport", () => { transitive: [], affectedTestCount: 0, totalModels: 5, - exposures: [{ unique_id: "exposure.p.exec", name: "exec", label: "Exec Revenue", type: "dashboard" }], + exposures: [ + { unique_id: "exposure.p.exec", name: "exec", label: "Exec Revenue", type: "dashboard" }, + { unique_id: "exposure.p.far", name: "far", label: "Far Board", type: "dashboard" }, + ], columnReach: { models: [{ name: "fct_daily", unique_id: "model.p.fct_daily", columns: ["revenue"] }], uncertain: [{ name: "rpt_big", unique_id: "model.p.rpt_big", reason: "names it only outside its select list (WHERE, JOIN, GROUP BY or HAVING)" }], @@ -219,7 +222,7 @@ describe("formatImpactReport", () => { }, }) expect(report).toContain( - "Column Reach: fct_orders.revenue is read by at least 1 model and at least 1 exposure; 1 more may read it; 1 could not be traced (stopped early)", + "Column Reach: fct_orders.revenue is read by at least 1 model and at least 1 exposure; at least 1 more may read it; at least 1 could not be traced (stopped early)", ) expect(report).toContain(" fct_daily → revenue") expect(report).toContain(" ◆ Exec Revenue [dashboard]") @@ -227,6 +230,8 @@ describe("formatImpactReport", () => { expect(report).toContain(" ! mart_ltv not traced: No compiled SQL found — run `dbt compile` first") expect(report).toContain(" ? ◆ Ops [dashboard] on a model that may read it") expect(report).not.toContain("Exposures (") + // Stopped early: the other exposures are not claimed as non-readers. + expect(report).toContain("Other exposures (1): column reach stopped early, so some of these were not checked") }) test("percentage calculation with 0 total models does not produce NaN or Infinity", () => { From a36f4bee5f185bb8b96a230b4a03da3149e5fa29 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 3 Nov 2026 19:02:05 +0530 Subject: [PATCH 37/37] docs: the stopped-early exposures note says some may not have been checked Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/impact-analysis.ts | 2 +- packages/opencode/test/altimate/tools/impact-analysis.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/tools/impact-analysis.ts b/packages/opencode/src/altimate/tools/impact-analysis.ts index b4d5758e37..5a5e3de299 100644 --- a/packages/opencode/src/altimate/tools/impact-analysis.ts +++ b/packages/opencode/src/altimate/tools/impact-analysis.ts @@ -309,7 +309,7 @@ export function formatImpactReport(data: { if (others.length > 0) { // When column reach stopped early, some of these may sit on models it never reached. const title = data.columnReach ? "Other exposures" : "Exposures" - const stoppedNote = ": column reach stopped early, so some of these were not checked" + const stoppedNote = ": column reach stopped early, so some of these may not have been checked" const unchecked = data.columnReach?.truncated ? stoppedNote : "" lines.push(`${title} (${others.length})${unchecked}`) lines.push("".padEnd(40, "-")) diff --git a/packages/opencode/test/altimate/tools/impact-analysis.test.ts b/packages/opencode/test/altimate/tools/impact-analysis.test.ts index 9aa7a5e5c6..709407295a 100644 --- a/packages/opencode/test/altimate/tools/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/tools/impact-analysis.test.ts @@ -231,7 +231,7 @@ describe("formatImpactReport", () => { expect(report).toContain(" ? ◆ Ops [dashboard] on a model that may read it") expect(report).not.toContain("Exposures (") // Stopped early: the other exposures are not claimed as non-readers. - expect(report).toContain("Other exposures (1): column reach stopped early, so some of these were not checked") + expect(report).toContain("Other exposures (1): column reach stopped early, so some of these may not have been checked") }) test("percentage calculation with 0 total models does not produce NaN or Infinity", () => {