Repository navigation
feat(dbt_lineage): return the dbt integration's model lineage as structured content #1637
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
18e72ea
fc33c2e
06223f4
2f45bc4
7d5d12c
8822f29
bd1f98e
7f1230a
4e523b0
615f88b
39525e4
c8a46b9
f87d128
b139685
853bfa0
7109869
4652bed
fcc3c44
2bb5e67
fdeddbf
d500243
e7127db
6b54087
6dc6c60
20fb04e
88f03f4
4d6d77e
1311bf7
c9a9cf9
4b90f8e
b881a3e
8ee906a
bcf3981
2dc1c1a
7b5e148
6150c92
4900cc1
b654e92
a6412db
e0dc3da
977cebc
0197654
db33839
9ebb98b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,6 +43,81 @@ export async function serverInstructions(): Promise<string> { | |
| } | ||
| } | ||
|
|
||
| // altimate_change start — lineage tools answer with structured content as well as text | ||
| // Lineage tools also return their metadata as structured content, so a client that draws lineage | ||
| // (the Claude Code plugin's lineage pane) reads edges and downstream lists without parsing text. | ||
| // Claude Code shows structured content to the model in place of the text, so `report` carries the | ||
| // text (for dbt_lineage only its header: the mappings and edges travel as data), only the lineage | ||
| // fields are passed, and the whole is capped. | ||
| const STRUCTURED_FIELDS: Record<string, readonly string[]> = { | ||
| dbt_lineage: ["model_name", "model_unique_id", "confidence", "confidence_factors", "upstream", "columns", "edges", "error"], | ||
| impact_analysis: [ | ||
| "model", | ||
| "column", | ||
| "change_type", | ||
| "severity", | ||
| "direct_count", | ||
| "transitive_count", | ||
| "test_count", | ||
| "downstream", | ||
| "exposures", | ||
| "column_reach", | ||
| "column_reach_error", | ||
| "exposures_error", | ||
| "error", | ||
| ], | ||
| } | ||
| export const STRUCTURED_TOOLS: ReadonlySet<string> = new Set(Object.keys(STRUCTURED_FIELDS)) | ||
| const STRUCTURED_OUTPUT_SCHEMA = { | ||
| type: "object" as const, | ||
| properties: { | ||
| report: { type: "string", description: "The tool's text report" }, | ||
| truncated: { type: "boolean", description: "True when the lineage fields were left out for size; the report is kept, shortened if it is too large on its own" }, | ||
| }, | ||
| required: ["report"], | ||
| additionalProperties: true, | ||
| } | ||
| // The structured result reaches the model in Claude Code, so it stays within the same order of | ||
| // size as a text result: past this, only the report is sent. Claude Code caps an MCP result at | ||
| // 25k tokens by default and JSON of ids and edges runs ~3-4 characters a token, so this stays under that. | ||
| const STRUCTURED_MAX_BYTES = 60_000 | ||
|
|
||
| export function structuredResult(tool: string, metadata: Record<string, unknown>, report: string): Record<string, unknown> { | ||
| // A tool that builds its own structured result (dbt_lineage from the dbt integration) hands it | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents |
||
| // over whole; otherwise the lineage fields of its metadata are passed. | ||
| const own = metadata.structuredContent | ||
| if (own && typeof own === "object" && !Array.isArray(own)) { | ||
| // The text repeats every edge that `columnLineage` already carries: only its header goes with them. | ||
| const head = report.split(/\n(?:Column edges \(|Trace of )/)[0]!.trim() | ||
| const full = { ...(own as Record<string, unknown>), report: head } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [WARNING]: The two paths return disjoint With the dbt integration up this branch forwards |
||
| return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| } | ||
| const fields = Object.fromEntries((STRUCTURED_FIELDS[tool] ?? []).filter((key) => key in metadata).map((key) => [key, metadata[key]])) | ||
| // dbt_lineage's text prints the same mappings and edges as `columns` and `edges`: keep its header. | ||
| const head = tool === "dbt_lineage" && "columns" in fields ? report.split(/\n(?:Column Mappings|Lineage Edges):/)[0]!.trim() : report | ||
| const full = { ...fields, report: head } | ||
| return Buffer.byteLength(JSON.stringify(full)) <= STRUCTURED_MAX_BYTES ? full : capReport(report, true) | ||
| } | ||
| /** An error report held to the same cap as a result, so a huge exception message cannot exceed it. */ | ||
| export function structuredError(message: string): Record<string, unknown> { | ||
| return capReport(message, false) | ||
| } | ||
| // `dropped`: the result's other fields were left out, so it is marked truncated even when the | ||
| // report itself fits. | ||
| function capReport(message: string, dropped: boolean): Record<string, unknown> { | ||
| // Measured as sent: JSON escaping can grow the text (a newline is two bytes, a control | ||
| // character six), so the cap applies to the serialized result, shortening until it fits. | ||
| const size = (value: Record<string, unknown>) => Buffer.byteLength(JSON.stringify(value)) | ||
| const whole = dropped ? { report: message, truncated: true } : { report: message } | ||
| if (size(whole) <= STRUCTURED_MAX_BYTES) return whole | ||
| let report = message | ||
| while (report.length > 0 && size({ report, truncated: true }) > STRUCTURED_MAX_BYTES) { | ||
| report = report.slice(0, Math.floor(report.length * 0.9)) | ||
| } | ||
| return { report, truncated: true } | ||
| } | ||
| // altimate_change end | ||
|
|
||
| /** Spilled results: a resource_link next to the text, which keeps the preview, row count and path. */ | ||
| function resultLinks(metadata: Record<string, unknown>) { | ||
| // A batched sql_execute call reports one saved file per spilled query. | ||
|
|
@@ -97,6 +172,7 @@ export async function createServer(options: ServeOptions = {}) { | |
| name, | ||
| description: descriptions[name] ?? tool.description, | ||
| inputSchema: tool.jsonSchema as McpTool["inputSchema"], | ||
| ...(STRUCTURED_TOOLS.has(name) && { outputSchema: STRUCTURED_OUTPUT_SCHEMA }), | ||
| })), ...(workspace?.tools ?? [])], | ||
| })) | ||
| // Ask the user once per session when the model weighed two readings and the client can ask; any | ||
|
|
@@ -149,11 +225,16 @@ export async function createServer(options: ServeOptions = {}) { | |
| return { | ||
| content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], | ||
| isError, | ||
| ...(STRUCTURED_TOOLS.has(request.params.name) && { | ||
| structuredContent: structuredResult(request.params.name, result.metadata, result.output), | ||
| }), | ||
| } | ||
| } catch (error) { | ||
| const message = error instanceof Error ? error.message : String(error) | ||
| return { | ||
| content: [{ type: "text" as const, text: error instanceof Error ? error.message : String(error) }], | ||
| content: [{ type: "text" as const, text: message }], | ||
| isError: true, | ||
| ...(STRUCTURED_TOOLS.has(request.params.name) && { structuredContent: structuredError(message) }), | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -193,8 +274,13 @@ export async function serveStdio(options: ServeOptions = {}): Promise<void> { | |
| const onclose = server.onclose | ||
| server.onclose = async () => { | ||
| try { | ||
| await onclose?.() | ||
| await Registry.closeAll() | ||
| try { | ||
| await onclose?.() | ||
| await Registry.closeAll() | ||
| } finally { | ||
| // The dbt adapter's Python bridge is stopped even when closing the rest fails. | ||
| await (await import("../native/connections/register")).disposeDbtAdapter() | ||
| } | ||
| closed.resolve() | ||
| } catch (error) { | ||
| closed.reject(error) | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -55,11 +55,15 @@ let dbtAdapter: any | null | undefined = undefined | |||||
| // permanent negative cache (`dbtAdapter === null`) is unchanged — a project that | ||||||
| // becomes valid mid-session is still not retried. | ||||||
| let dbtAdapterInflight: Promise<any | null> | undefined | ||||||
| /** The project root the cached attempt read from the dbt config; null when it found none. */ | ||||||
| let dbtProjectRoot: string | null = null | ||||||
|
|
||||||
| /** Test seams for the single-flight. Production leaves `readConfig` unset. */ | ||||||
| export const dbtAdapterInternals: { | ||||||
| /** Replaces the dbt config read, so a test can hold an attempt open. */ | ||||||
| readConfig?: () => Promise<unknown> | ||||||
| /** Replaces dbt-tools' adapter creation, so a test can see which integration modes were tried. */ | ||||||
| create?: (config: any) => Promise<any> | ||||||
| /** Adapter creation attempts since the last reset — what single-flight bounds. */ | ||||||
| attempts: number | ||||||
| /** Cache writes since the last reset: only the attempt that owns the slot writes. */ | ||||||
|
|
@@ -101,6 +105,9 @@ async function ensureDbtAdapter(): Promise<any | null> { | |||||
| const dbtConfig = dbtAdapterInternals.readConfig | ||||||
| ? ((await dbtAdapterInternals.readConfig()) as Awaited<ReturnType<typeof readDbtConfig>>) | ||||||
| : await readDbtConfig() | ||||||
| if (dbtAdapterInflight === mine) { | ||||||
| dbtProjectRoot = typeof dbtConfig?.projectRoot === "string" ? dbtConfig.projectRoot : null | ||||||
| } | ||||||
| if (!dbtConfig) return publish(null) | ||||||
|
|
||||||
| // Check if dbt_project.yml exists | ||||||
|
|
@@ -113,7 +120,7 @@ async function ensureDbtAdapter(): Promise<any | null> { | |||||
| } | ||||||
|
|
||||||
| // Create the adapter | ||||||
| const { create } = await import("../../../../../dbt-tools/src/adapter") | ||||||
| const create = dbtAdapterInternals.create ?? (await import("../../../../../dbt-tools/src/adapter")).create | ||||||
| return publish(await create(dbtConfig), dbtConfig.projectRoot) | ||||||
| } catch { | ||||||
| // dbt-tools not available or config invalid — fall back to native | ||||||
|
|
@@ -261,6 +268,38 @@ async function tryExecuteViaDbt( | |||||
| } | ||||||
| } | ||||||
|
|
||||||
| /** The project's dbt adapter, created on first use; null when there is no usable dbt project. */ | ||||||
| export function getDbtAdapter(): Promise<any | null> { | ||||||
| return ensureDbtAdapter() | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Stops the cached dbt adapter (its Python bridge and file watchers), for a server that is shutting | ||||||
| * down. An attempt still starting is stopped once it settles. | ||||||
| */ | ||||||
| export async function disposeDbtAdapter(): Promise<void> { | ||||||
| const settled = dbtAdapter | ||||||
| const inflight = dbtAdapterInflight | ||||||
| resetDbtAdapter() | ||||||
| void inflight?.then((adapter) => adapter?.dispose?.()).catch(() => {}) | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The in-flight adapter is disposed fire-and-forget, so Prompt for AI agents
Suggested change
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not changing; recorded as R15 on the review log. Awaiting an attempt that is still starting would hold the stdio server's shutdown for a whole dbt parse. The in-flight adapter is disposed when its start settles, if the process is still alive. A process that exits first leaves that start to end on its own, as before this PR: the base disposed nothing on shutdown.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: unchanged after the reshape (b654e92); this residual is now R11 on the review log. |
||||||
| await settled?.dispose?.().catch(() => {}) | ||||||
| } | ||||||
|
|
||||||
| /** The dbt project's root as the adapter finds it: a dbt.json, or a walk up from the working directory. */ | ||||||
| export async function getDbtProjectRoot(): Promise<string | undefined> { | ||||||
| // Once an adapter attempt has settled, reuse the root it read rather than reading the config again. | ||||||
| if (dbtAdapter !== undefined) return dbtProjectRoot ?? undefined | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [SUGGESTION]: The read-only path never fills this cache, so every
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not changing in this PR; disclosed as R14. The probe is |
||||||
| try { | ||||||
| const config = dbtAdapterInternals.readConfig | ||||||
| ? await dbtAdapterInternals.readConfig() | ||||||
| : await (await import("../../../../../dbt-tools/src/config")).read() | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: This re-reads the dbt config that On the fallback path the config is read twice: once at lines 100-105 and again here. When no
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed in 6dc6c60: |
||||||
| const root = (config as { projectRoot?: unknown } | null)?.projectRoot | ||||||
| return typeof root === "string" ? root : undefined | ||||||
| } catch { | ||||||
| return undefined | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /** Reset dbt adapter (for testing). */ | ||||||
| export function resetDbtAdapter(): void { | ||||||
| dbtAdapter = undefined | ||||||
|
|
@@ -270,6 +309,7 @@ export function resetDbtAdapter(): void { | |||||
| dbtAdapterInflight = undefined | ||||||
| dbtAdapterInternals.attempts = 0 | ||||||
| dbtAdapterInternals.writes = 0 | ||||||
| dbtProjectRoot = null | ||||||
| } | ||||||
|
|
||||||
| // --------------------------------------------------------------------------- | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
SUGGESTION:
dispose()is assumed to return a promise; a synchronousdisposereplaces the originalinitialize()failure with acatch is not a functionTypeError and skipsthrow e.packages/dbt-tools/src/index.ts:206already cleans up with the shape-agnostictry { await adapter.dispose() } catch {}. Using that here keeps the initialize error the caller depends on:register.tslogs it and falls back to the configured mode, and losing it turns the whole attempt into a cachednull.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not an issue:
@altimateai/dbt-integrationdeclaresDBTProjectIntegrationAdapter.dispose()asasync dispose(): Promise<void>, so it always returns a promise and never throws synchronously;.catchcan't replace theinitialize()error.