From c74c1aad3033e5a5b96a6015e5bc8fd384972e08 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 13 Nov 2026 13:48:08 -0500 Subject: [PATCH 1/2] feat: add data source and connection variable types to dashboards --- .changeset/source-vars-1.md | 7 + packages/api/src/controllers/alerts.ts | 6 + .../api/__tests__/dashboard.int.test.ts | 124 ++++ packages/api/src/routers/api/dashboards.ts | 32 +- .../__tests__/provisionDashboards.int.test.ts | 35 + .../src/tasks/provisionDashboards/index.ts | 11 +- .../src/utils/dashboardVariableReferences.ts | 24 + packages/api/src/utils/externalApi.ts | 8 +- .../DashboardFiltersList.tsx | 34 +- .../__tests__/DashboardFiltersModal.test.tsx | 1 + .../__tests__/filterFormState.test.ts | 2 +- .../src/__tests__/referenceVariables.test.ts | 657 ++++++++++++++++++ packages/common-utils/src/core/utils.ts | 63 +- .../common-utils/src/dashboardFilterValues.ts | 16 + .../common-utils/src/dashboardValidation.ts | 129 +++- packages/common-utils/src/filters.ts | 56 +- .../common-utils/src/referenceVariables.ts | 366 ++++++++++ packages/common-utils/src/types.ts | 69 +- 18 files changed, 1586 insertions(+), 54 deletions(-) create mode 100644 .changeset/source-vars-1.md create mode 100644 packages/api/src/utils/dashboardVariableReferences.ts create mode 100644 packages/common-utils/src/__tests__/referenceVariables.test.ts create mode 100644 packages/common-utils/src/referenceVariables.ts diff --git a/.changeset/source-vars-1.md b/.changeset/source-vars-1.md new file mode 100644 index 0000000000..eeacef4d13 --- /dev/null +++ b/.changeset/source-vars-1.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': patch +--- + +feat: add data source and connection variable types to dashboards diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index f1c69d0ee5..f49cf85a6f 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -9,6 +9,7 @@ import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { configUsesReferenceVariable } from '@hyperdx/common-utils/dist/referenceVariables'; import { isRangeThresholdType } from '@hyperdx/common-utils/dist/types'; import { groupBy } from 'lodash'; import { type FilterQuery, Types } from 'mongoose'; @@ -97,6 +98,11 @@ export const validateAlertInput = async ( // The alert checker cannot evaluate these tiles, so the alert would never run. if (tile.config != null && !savedChartConfigSupportsAlerts(tile.config)) { + if (configUsesReferenceVariable(tile.config)) { + throw new Api400Error( + 'Alerts are not supported on tiles that use a data source or connection variable', + ); + } if (isPromqlSavedChartConfig(tile.config)) { throw new Api400Error('Alerts are not supported on PromQL tiles'); } diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index 2abf2baf64..b0d983335b 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1518,6 +1518,130 @@ describe('dashboard router', () => { ); }); + describe('data source variables', () => { + const dataSourceFilter = { + id: new Types.ObjectId().toString(), + type: 'DATA_SOURCE' as const, + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + it('persists a tile and a filter that reference the variable', async () => { + const tile = makeTile({ sourceId: '$logs' }); + const queriedFilter = { + id: new Types.ObjectId().toString(), + type: 'QUERY_EXPRESSION' as const, + name: 'Service', + expression: 'ServiceName', + source: '$logs', + }; + + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [tile], + filters: [dataSourceFilter, queriedFilter], + }) + .expect(200); + + expect(created.body.tiles[0].config.source).toBe('$logs'); + expect(created.body.filters).toEqual([dataSourceFilter, queriedFilter]); + }); + + it('rejects a reference to an unknown variable', async () => { + const res = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, tiles: [makeTile({ sourceId: '$nope' })] }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch( + /does not name a data source variable/, + ); + }); + + it('rejects a variable offering kinds the reference cannot use', async () => { + const res = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + ...dataSourceFilter, + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }, + ], + }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch(/promql/); + }); + + it('validates a PATCH of tiles against the stored filters', async () => { + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [dataSourceFilter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ tiles: [makeTile({ sourceId: '$logs' })] }) + .expect(200); + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [] }) + .expect(400); + }); + + it('drops an alert on a tile that uses the variable', async () => { + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [ + makeTile({ + sourceId: '$logs', + alert: makeMockAlert(webhook._id.toString()), + }), + ], + filters: [dataSourceFilter], + }) + .expect(200); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects a tile alert on a tile that uses the variable', async () => { + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [dataSourceFilter], + }) + .expect(200); + + const res = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: created.body.id, + tileId: created.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(400); + + expect(res.body.message).toMatch(/data source or connection variable/); + }); + }); + describe('promql-label filters', () => { const createPromqlSource = () => Source.create({ diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 544679ef92..8ec73d04b6 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -2,12 +2,14 @@ import { validateDashboardFilterModes, validateDashboardFilterOptionUniqueness, validateDashboardFilterVariableNames, + validateDashboardVariableReferences, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { getAdhocFilterSourceKinds, isAdhocFilter, isPrometheusLabelFilter, } from '@hyperdx/common-utils/dist/filters'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter, DashboardSchema, @@ -39,6 +41,7 @@ import { import { getSources } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import type { ObjectId } from '@/models'; +import { getVariableReferenceError } from '@/utils/dashboardVariableReferences'; import { objectIdSchema } from '@/utils/zod'; // create routes that will get and update dashboards @@ -101,9 +104,12 @@ async function validateFilterSources( new Set(getFilterSourceRefs(filter).map(refKey)), ]), ); + // Variable references are checked by validateDashboardVariableReferences const newRefs = filters.flatMap(filter => getFilterSourceRefs(filter).filter( - ref => !existingRefsById.get(filter.id)?.has(refKey(ref)), + ref => + parseVariableReference(ref.id) == null && + !existingRefsById.get(filter.id)?.has(refKey(ref)), ), ); if (newRefs.length === 0) return null; @@ -164,7 +170,14 @@ router.post( '/', migrateLegacyDashboardTileColors, validateRequest({ - body: DashboardWithoutIdSchema.superRefine(addFilterIssues), + body: DashboardWithoutIdSchema.superRefine(addFilterIssues).superRefine( + (data, ctx) => + validateDashboardVariableReferences( + data.tiles, + data.filters ?? [], + ctx, + ), + ), }), async (req, res, next) => { try { @@ -223,11 +236,16 @@ router.patch( // null still signals field removal downstream. const updates = DashboardWithoutIdSchema.partial().parse(req.body); - const sourceError = await validateFilterSources( - teamId, - req.body.filters, - dashboard.filters, - ); + const sourceError = + getVariableReferenceError( + req.body.tiles ?? dashboard.tiles, + req.body.filters ?? dashboard.filters, + ) ?? + (await validateFilterSources( + teamId, + req.body.filters, + dashboard.filters, + )); if (sourceError != null) { return res.status(400).json({ message: sourceError }); } diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 3e17594813..c91e79b5b2 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -143,6 +143,41 @@ describe('provisionDashboards', () => { // rejects the legacy enum and `readDashboardFiles` skips the file // outright, so a provisioned dashboard authored against #2265 // wouldn't ship at all. + it('parses a tile that references a declared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'source-variable.json'), + JSON.stringify({ + name: 'Source Variable', + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: ['log'], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }), + ); + expect(readDashboardFiles(tmpDir)).toHaveLength(1); + }); + + it('skips a tile that references an undeclared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'unknown-variable.json'), + JSON.stringify({ + name: 'Unknown Variable', + tiles: [makeTile({ sourceId: '$logs' })], + }), + ); + expect(readDashboardFiles(tmpDir)).toEqual([]); + }); + it('migrates legacy chart-N tile colors before schema validation', () => { const tile = makeTile(); (tile.config as any).color = 'chart-1'; diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cf10427c72..4199d96182 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -1,4 +1,7 @@ -import { validateDashboardFilterOptionUniqueness } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { + validateDashboardFilterOptionUniqueness, + validateDashboardVariableReferences, +} from '@hyperdx/common-utils/dist/dashboardValidation'; import { DashboardWithoutId, DashboardWithoutIdSchema, @@ -30,8 +33,10 @@ function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown { } const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( - (data, ctx) => - validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), + (data, ctx) => { + validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); + validateDashboardVariableReferences(data.tiles, data.filters ?? [], ctx); + }, ); export function readDashboardFiles(dir: string): DashboardWithoutId[] { diff --git a/packages/api/src/utils/dashboardVariableReferences.ts b/packages/api/src/utils/dashboardVariableReferences.ts new file mode 100644 index 0000000000..9e1eea3442 --- /dev/null +++ b/packages/api/src/utils/dashboardVariableReferences.ts @@ -0,0 +1,24 @@ +import { + collectRefinementIssues, + validateDashboardVariableReferences, +} from '@hyperdx/common-utils/dist/dashboardValidation'; +import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; + +/** + * Rejects data source and connection variable references (`$name` in source + * and connection fields) that name no variable of the right type, or a data + * source variable offering kinds the field cannot use. Takes internal-shape + * tiles and filters, as they will be stored, so a partial update can be + * checked against the stored remainder. Returns an error message, or null. + */ +export function getVariableReferenceError( + tiles: { config: object }[] = [], + filters: DashboardFilter[] = [], +): string | null { + const issues = collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + return issues.length > 0 + ? issues.map(issue => issue.message).join('; ') + : null; +} diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..cb63a42da8 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -5,6 +5,7 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, + isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig( } /** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract; +type InternalOnlyDashboardFilter = Extract< + DashboardFilter, + { type: 'ADHOC' | 'DATA_SOURCE' | 'CONNECTION' } +>; export type ExternalCompatibleDashboardFilter = Exclude< DashboardFilter, InternalOnlyDashboardFilter @@ -253,7 +257,7 @@ export function getExternalCompatibleFilters( ): ExternalCompatibleDashboardFilter[] { return (filters ?? []).filter( (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter), + !isAdhocFilter(filter) && !isReferenceVariableFilter(filter), ); } diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx index 22e99756ed..2703363e6a 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx @@ -5,6 +5,7 @@ import { isFilterRequired, isFilterVariableEnabled, } from '@hyperdx/common-utils/dist/filters'; +import { getObjectName } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; import { Button, @@ -17,15 +18,19 @@ import { import { IconAsterisk, IconBuildingBroadcastTower, + IconDatabase, IconFilter, IconLabel, IconList, IconPencil, IconRefresh, IconSearch, + IconServer, IconTrash, } from '@tabler/icons-react'; +import { SOURCE_KIND_LABELS } from '@/components/Sources/SourceForm/constants'; +import { useConnections } from '@/connection'; import { useSources } from '@/source'; import { MODAL_SIZE } from './constants'; @@ -53,7 +58,7 @@ function getSourceNames( sources?: { id: string; name: string }[], ) { return ids - .map(id => sources?.find(s => s.id === id)?.name) + .map(id => getObjectName(id, sources)) .filter((name): name is string => !!name) .join(', '); } @@ -62,19 +67,20 @@ function getSourceNames( function getValuesAttribute( filter: DashboardFilter, sources?: { id: string; name: string }[], + connections?: { id: string; name: string }[], ): DashboardFilterAttribute { switch (filter.type) { case 'QUERY_EXPRESSION': return { icon: , tooltip: 'Source the dropdown values are queried from', - label: sources?.find(s => s.id === filter.source)?.name ?? '', + label: getObjectName(filter.source, sources) ?? '', }; case 'PROMETHEUS_LABEL': return { icon: , tooltip: 'PromQL source and label the dropdown values come from', - label: [sources?.find(s => s.id === filter.source)?.name, filter.label] + label: [getObjectName(filter.source, sources), filter.label] .filter(Boolean) .join(' · '), }; @@ -91,6 +97,25 @@ function getValuesAttribute( tooltip: 'Sources the condition keys and values come from', label: getSourceNames(filter.sources, sources), }; + case 'DATA_SOURCE': + return { + icon: , + tooltip: + 'Kinds of sources offered, and the connection they are limited to', + label: [ + filter.allowedSourceKinds + .map(kind => SOURCE_KIND_LABELS[kind]) + .join(', '), + filter.connection && getObjectName(filter.connection, connections), + ] + .filter(Boolean) + .join(' · '), + }; + case 'CONNECTION': + return { + icon: , + label: 'Connections', + }; default: filter satisfies never; // exhaustive check return { icon: , label: '' }; @@ -122,6 +147,7 @@ export const DashboardFiltersList = ({ onAddNew, }: DashboardFiltersListProps) => { const { data: sources } = useSources(); + const { data: connections } = useConnections(); return ( {filters.map(filter => { - const attributes = [getValuesAttribute(filter, sources)]; + const attributes = [getValuesAttribute(filter, sources, connections)]; const broadcastTarget = getBroadcastTargetDisplay(filter, sources); if (!hideAppliesTo && broadcastTarget != null) { diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx index 91d55296f3..04b1d99c62 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx @@ -12,6 +12,7 @@ import { FilterFormControl } from '@/components/DashboardFiltersModal/filterForm jest.mock('@/useConfirm', () => ({ useConfirm: () => jest.fn() })); jest.mock('@/source', () => ({ useSources: () => ({ data: [] }) })); +jest.mock('@/connection', () => ({ useConnections: () => ({ data: [] }) })); // Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. window.HTMLElement.prototype.scrollIntoView = jest.fn(); diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index d0e3131fbe..9f2ea9c142 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -326,7 +326,7 @@ describe('toSavedFilter', () => { /** The minimum a form needs to save, per filter type the form supports. */ const byType: Record< - Exclude, + Exclude, Partial > = { QUERY_EXPRESSION: { diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts new file mode 100644 index 0000000000..feb32581dd --- /dev/null +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -0,0 +1,657 @@ +import { + convertToDashboardTemplate, + savedChartConfigSupportsAlerts, +} from '@/core/utils'; +import { getBlockingRequiredFilterNames } from '@/dashboardFilterValues'; +import { + collectRefinementIssues, + validateDashboardVariableReferences, +} from '@/dashboardValidation'; +import { + getDashboardVariableDeclarations, + isReferenceVariableFilterType, + validateVariableName, +} from '@/filters'; +import { + configUsesReferenceVariable, + getConfigReferenceVariables, + getConnectionVariableNames, + getObjectName, + getSourceVariableDeclarations, + isSourceOfferedByVariable, + isSourceVariableReferenceAccepted, + parseVariableReference, + resolveConfigReferences, + resolveFilterReferences, + resolveVariableReference, + restoreConfigReferences, +} from '@/referenceVariables'; +import { + AdhocDashboardFilter, + ConnectionDashboardFilter, + DashboardFilter, + DashboardFilterSchema, + DataSourceDashboardFilter, + DisplayType, + QueryExpressionDashboardFilter, + RawSqlSavedChartConfig, + SavedChartConfig, + SourceKind, +} from '@/types'; + +const dataSourceFilter = ( + overrides: Partial = {}, +): DataSourceDashboardFilter => ({ + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const queriedFilter = ( + overrides: Partial = {}, +): QueryExpressionDashboardFilter => ({ + id: 'q', + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + source: '$logs', + ...overrides, +}); + +const connectionFilter = ( + overrides: Partial = {}, +): ConnectionDashboardFilter => ({ + id: 'conn-filter', + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const rawSqlConfig = ( + overrides: Partial = {}, +): RawSqlSavedChartConfig => ({ + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + ...overrides, +}); + +const lineConfig = (source: string): SavedChartConfig => ({ + source, + displayType: DisplayType.Line, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'lucene', +}); + +describe('DataSourceDashboardFilterSchema', () => { + it('accepts a data source filter', () => { + expect(DashboardFilterSchema.safeParse(dataSourceFilter()).success).toBe( + true, + ); + }); + + it('requires at least one allowed kind', () => { + expect( + DashboardFilterSchema.safeParse( + dataSourceFilter({ allowedSourceKinds: [] }), + ).success, + ).toBe(false); + }); + + it('is always single-select and required', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + maxSelections: undefined, + }).success, + ).toBe(false); + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + minSelections: 0, + }).success, + ).toBe(false); + }); + + it('rejects unknown keys', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + isGlobalRequirement: true, + }).success, + ).toBe(false); + }); +}); + +describe('parseVariableReference', () => { + it.each([ + ['$logs', 'logs'], + ['$Logs_2', 'Logs_2'], + ['65f1c0ffee0000000000abcd', undefined], + ['$', undefined], + ['$1abc', undefined], + ['x$logs', undefined], + ['$logs ', undefined], + [undefined, undefined], + ])('parses %p as %p', (source, expected) => { + expect(parseVariableReference(source)).toBe(expected); + }); +}); + +describe('resolveVariableReference', () => { + it('passes source IDs through and resolves references', () => { + expect(resolveVariableReference('abc', new Map([['logs', 'x']]))).toBe( + 'abc', + ); + expect(resolveVariableReference('$logs', new Map([['logs', 'x']]))).toBe( + 'x', + ); + expect(resolveVariableReference('$logs', new Map())).toBeUndefined(); + expect(resolveVariableReference(undefined, new Map())).toBeUndefined(); + }); +}); + +describe('resolveConfigReferences', () => { + const values = new Map([ + ['logs', 'abc'], + ['conn', 'conn-1'], + ]); + + it('replaces a source reference with the selected source', () => { + expect(resolveConfigReferences(lineConfig('$logs'), values)).toEqual( + lineConfig('abc'), + ); + }); + + it('replaces a connection reference with the selected connection', () => { + expect( + resolveConfigReferences(rawSqlConfig({ connection: '$conn' }), values), + ).toEqual(rawSqlConfig({ connection: 'conn-1' })); + }); + + it('leaves unselected references and IDs as written', () => { + expect(resolveConfigReferences(lineConfig('$logs'), new Map())).toEqual( + lineConfig('$logs'), + ); + expect(resolveConfigReferences(lineConfig('abc'), values)).toEqual( + lineConfig('abc'), + ); + }); +}); + +describe('getConfigReferenceVariables', () => { + it('names the variables a config source and connection reference', () => { + expect( + getConfigReferenceVariables( + rawSqlConfig({ connection: '$conn', source: '$logs' }), + ), + ).toEqual(['logs', 'conn']); + expect(getConfigReferenceVariables(lineConfig('abc'))).toEqual([]); + }); +}); + +describe('isSourceVariableReferenceAccepted', () => { + const declarations = [ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + { name: 'any', allowedSourceKinds: [SourceKind.Log, SourceKind.Promql] }, + ]; + + it('accepts a variable whose every kind the field accepts', () => { + expect( + isSourceVariableReferenceAccepted('$logs', declarations, [ + SourceKind.Log, + SourceKind.Trace, + ]), + ).toBe(true); + }); + + it('rejects a variable offering a kind the field does not accept', () => { + expect( + isSourceVariableReferenceAccepted('$any', declarations, [SourceKind.Log]), + ).toBe(false); + }); + + it('rejects an unknown variable and a source ID', () => { + expect( + isSourceVariableReferenceAccepted('$nope', declarations, [ + SourceKind.Log, + ]), + ).toBe(false); + expect( + isSourceVariableReferenceAccepted('abc', declarations, [SourceKind.Log]), + ).toBe(false); + }); +}); + +describe('resolveFilterReferences', () => { + it('resolves source and appliesToSourceIds', () => { + const { filter, unresolvedVariables } = resolveFilterReferences( + queriedFilter({ appliesToSourceIds: ['$logs', 'other'] }), + new Map([['logs', 'abc']]), + ); + expect(filter.source).toBe('abc'); + expect(filter.appliesToSourceIds).toEqual(['abc', 'other']); + expect(unresolvedVariables).toEqual([]); + }); + + it('leaves unselected references as written so they match no tile', () => { + const adhoc: AdhocDashboardFilter = { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['$logs'], + isVariableEnabled: true, + }; + const { filter, unresolvedVariables } = resolveFilterReferences( + adhoc, + new Map(), + ); + expect(filter.sources).toEqual(['$logs']); + expect(unresolvedVariables).toEqual(['logs']); + }); +}); + +describe('dashboard variable declarations', () => { + it('keeps the first of two variables sharing a name', () => { + expect( + getSourceVariableDeclarations([ + dataSourceFilter(), + dataSourceFilter({ + id: 'ds2', + allowedSourceKinds: [SourceKind.Trace], + }), + ]), + ).toEqual([ + { + name: 'logs', + allowedSourceKinds: [SourceKind.Log], + connection: undefined, + }, + ]); + }); + + it('keeps reference variables out of text variables', () => { + const filters: DashboardFilter[] = [dataSourceFilter(), connectionFilter()]; + expect(getDashboardVariableDeclarations(filters)).toEqual([]); + expect(getSourceVariableDeclarations(filters)).toEqual([ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + ]); + expect(getConnectionVariableNames(filters)).toEqual(['conn']); + }); + + it('shares the variable name namespace with other filters', () => { + expect( + validateVariableName({ + value: 'logs', + otherFilters: [dataSourceFilter()], + }), + ).toMatch(/used by another filter/); + }); +}); + +describe('alerts', () => { + it('are not supported on tiles that use a connection variable', () => { + expect( + savedChartConfigSupportsAlerts( + rawSqlConfig({ displayType: DisplayType.Line, connection: '$conn' }), + ), + ).toBe(false); + }); + + it('are not supported on tiles that use a data source variable', () => { + expect(configUsesReferenceVariable(lineConfig('$logs'))).toBe(true); + expect(savedChartConfigSupportsAlerts(lineConfig('$logs'))).toBe(false); + expect(savedChartConfigSupportsAlerts(lineConfig('abc'))).toBe(true); + }); +}); + +describe('getBlockingRequiredFilterNames (connections)', () => { + it('blocks a tile until its connection variable has a selection', () => { + expect( + getBlockingRequiredFilterNames({ + config: rawSqlConfig({ connection: '$conn' }), + sourceId: undefined, + unsatisfiedRequiredFilters: [connectionFilter()], + referencedVariables: [], + referenceVariableNames: ['conn'], + }), + ).toEqual(['Connection']); + }); +}); + +describe('getBlockingRequiredFilterNames', () => { + it('blocks a tile until its data source variable has a selection', () => { + const unsatisfied = [dataSourceFilter()]; + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('$logs'), + sourceId: undefined, + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + referenceVariableNames: ['logs'], + }), + ).toEqual(['Logs source']); + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('abc'), + sourceId: 'abc', + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + }), + ).toEqual([]); + }); +}); + +describe('validateDashboardVariableReferences', () => { + const collectIssues = ( + tiles: { config: SavedChartConfig }[], + filters: DashboardFilter[], + ) => + collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + + it('accepts references to declared variables of compatible kinds', () => { + expect( + collectIssues( + [{ config: lineConfig('$logs') }], + [dataSourceFilter(), queriedFilter()], + ), + ).toEqual([]); + }); + + it('accepts connection references to a declared connection variable', () => { + expect( + collectIssues( + [{ config: rawSqlConfig({ connection: '$conn' }) }], + [connectionFilter(), dataSourceFilter({ connection: '$conn' })], + ), + ).toEqual([]); + }); + + it('rejects a connection reference that names no connection variable', () => { + const issues = collectIssues( + [{ config: rawSqlConfig({ connection: '$logs' }) }], + [dataSourceFilter({ connection: '$nope' })], + ); + expect(issues.map(issue => issue.path)).toEqual([ + ['tiles', 0, 'config', 'connection'], + ['filters', 0, 'connection'], + ]); + }); + + it('rejects a source reference to a connection variable', () => { + expect( + collectIssues([{ config: lineConfig('$conn') }], [connectionFilter()]), + ).toHaveLength(1); + }); + + it('rejects a reference to an unknown variable', () => { + const issues = collectIssues([{ config: lineConfig('$nope') }], []); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['tiles', 0, 'config', 'source']); + }); + + it.each<[string, { config: SavedChartConfig }[], DashboardFilter[]]>([ + [ + 'a PromQL tile', + [ + { + config: { + configType: 'promql', + source: '$logs', + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + }, + }, + ], + [], + ], + [ + 'a PromQL label filter', + [], + [ + { + id: 'p', + type: 'PROMETHEUS_LABEL', + name: 'Job', + source: '$logs', + label: 'job', + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + ], + [ + 'an ad hoc filter of another source type', + [], + [ + { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'promql', + sources: ['$logs'], + isVariableEnabled: true, + }, + ], + ], + [ + "a queried filter's applies-to list", + [], + [ + queriedFilter({ + source: 'abc', + appliesToSourceIds: ['$logs'], + }), + ], + ], + ])( + 'rejects a logs-or-PromQL variable used by %s', + (_label, tiles, filters) => { + const issues = collectIssues(tiles, [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + ...filters, + ]); + expect(issues.length).toBeGreaterThan(0); + }, + ); + + it('rejects a variable offering kinds the field does not accept', () => { + const issues = collectIssues( + [], + [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + queriedFilter(), + ], + ); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['filters', 1, 'source']); + expect(issues[0].message).toMatch(/promql/); + }); +}); + +describe('convertToDashboardTemplate (connections)', () => { + it('keeps connection references and names a data source connection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { + id: 't', + x: 0, + y: 0, + w: 6, + h: 4, + config: rawSqlConfig({ connection: '$conn' }), + }, + ], + filters: [ + connectionFilter(), + dataSourceFilter({ connection: 'conn-1' }), + ], + savedFilterValues: [{ type: 'variable', name: 'conn', values: ['c'] }], + }, + [], + [{ id: 'conn-1', name: 'Primary', host: '', username: '' }], + ); + expect(template.tiles[0].config).toMatchObject({ connection: '$conn' }); + expect(template.filters?.[1]).toMatchObject({ connection: 'Primary' }); + expect(template.savedFilterValues).toBeUndefined(); + }); +}); + +describe('convertToDashboardTemplate', () => { + it('keeps references and drops the saved data source selection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { id: 't', x: 0, y: 0, w: 6, h: 4, config: lineConfig('$logs') }, + ], + filters: [dataSourceFilter(), queriedFilter()], + savedFilterValues: [ + { type: 'variable', name: 'logs', values: ['abc'] }, + { type: 'variable', name: 'env', values: ['prod'] }, + ], + }, + [], + ); + expect(template.tiles[0].config).toMatchObject({ source: '$logs' }); + expect(template.filters?.[1]).toMatchObject({ source: '$logs' }); + expect(template.savedFilterValues).toEqual([ + { type: 'variable', name: 'env', values: ['prod'] }, + ]); + }); +}); + +describe('resolveFilterReferences (connections)', () => { + it("resolves a data source variable's connection reference", () => { + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map([['conn', 'conn-1']]), + ), + ).toEqual({ + filter: dataSourceFilter({ connection: 'conn-1' }), + unresolvedVariables: [], + }); + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map(), + ).unresolvedVariables, + ).toEqual(['conn']); + }); +}); + +describe('restoreConfigReferences', () => { + it('puts stored source and connection references back', () => { + const stored = rawSqlConfig({ connection: '$conn', source: '$logs' }); + const resolved = resolveConfigReferences( + stored, + new Map([ + ['conn', 'conn-1'], + ['logs', 'abc'], + ]), + ); + expect( + restoreConfigReferences( + { ...resolved, displayType: DisplayType.Number }, + stored, + ), + ).toEqual({ ...stored, displayType: DisplayType.Number }); + }); + + it('keeps fields the stored config holds as IDs', () => { + expect( + restoreConfigReferences( + rawSqlConfig({ connection: 'conn-2' }), + rawSqlConfig({ connection: 'conn-1' }), + ), + ).toEqual(rawSqlConfig({ connection: 'conn-2' })); + }); +}); + +describe('isSourceOfferedByVariable', () => { + const source = { kind: SourceKind.Log, connection: 'conn-1' }; + + it('offers an enabled source of an allowed kind on the variable connection', () => { + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + connection: 'conn-1', + }), + ).toBe(true); + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + }), + ).toBe(true); + }); + + it.each([ + ['another kind', source, { allowedSourceKinds: [SourceKind.Trace] }], + [ + 'another connection', + source, + { allowedSourceKinds: [SourceKind.Log], connection: 'conn-2' }, + ], + [ + 'an unresolved connection reference', + source, + { allowedSourceKinds: [SourceKind.Log], connection: '$conn' }, + ], + [ + 'a disabled source', + { ...source, disabled: true }, + { allowedSourceKinds: [SourceKind.Log] }, + ], + ])('does not offer %s', (_label, candidate, variable) => { + expect(isSourceOfferedByVariable(candidate, variable)).toBe(false); + }); +}); + +describe('getObjectName', () => { + const objects = [{ id: 'abc', name: 'Logs' }]; + + it('names an object, keeps a reference, and is undefined for a missing ID', () => { + expect(getObjectName('abc', objects)).toBe('Logs'); + expect(getObjectName('$logs', objects)).toBe('$logs'); + expect(getObjectName('nope', objects)).toBeUndefined(); + }); +}); + +describe('isReferenceVariableFilterType', () => { + it.each([ + ['DATA_SOURCE', true], + ['CONNECTION', true], + ['STATIC_LIST', false], + [undefined, false], + ] as const)('%p is %p', (type, expected) => { + expect(isReferenceVariableFilterType(type)).toBe(expected); + }); +}); diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 7c93078458..422eecb1d0 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -8,6 +8,8 @@ export { default as objectHash } from 'object-hash'; import { isAdhocFilter, + isConnectionFilter, + isDataSourceFilter, isQueryExpressionFilter, isStaticListFilter, } from '@/filters'; @@ -26,6 +28,11 @@ import { replaceMacros, } from '@/macros'; import { QUERY_PARAMS, RawSqlQueryParam } from '@/rawSqlParams'; +import { + configUsesReferenceVariable, + getObjectName, + getReferenceVariableFilters, +} from '@/referenceVariables'; import { BuilderChartConfig, BuilderChartConfigWithDateRange, @@ -914,44 +921,40 @@ export function convertToDashboardTemplate( }; }; + // A variable reference is portable as written; '' marks a missing object + const getSourceName = (id: string) => getObjectName(id, sources) ?? ''; + const getConnectionName = (id: string) => + getObjectName(id, connections) ?? ''; + const convertToTileTemplate = ( input: Dashboard['tiles'][0], - sources: TSource[], - connections: Connection[], ): TileTemplate => { const tile = TileTemplateSchema.strip().parse(structuredClone(input)); // Extract name from source/connection or default to '' if not found const tileConfig = tile.config; if (isBuilderSavedChartConfig(tileConfig)) { - tileConfig.source = ( - sources.find(source => source.id === tileConfig.source) ?? { name: '' } - ).name; + tileConfig.source = getSourceName(tileConfig.source); } else if (isRawSqlSavedChartConfig(tileConfig)) { - tileConfig.connection = ( - connections.find(conn => conn.id === tileConfig.connection) ?? { - name: '', - } - ).name; + tileConfig.connection = getConnectionName(tileConfig.connection); if (tileConfig.source) { - tileConfig.source = - sources.find(source => source.id === tileConfig.source)?.name ?? ''; + tileConfig.source = getSourceName(tileConfig.source); } } convertOnClickTargetIdToName(tileConfig); return tile; }; - const convertToFilterTemplate = ( - input: DashboardFilter, - sources: TSource[], - ): DashboardFilter => { + const convertToFilterTemplate = (input: DashboardFilter): DashboardFilter => { const filter = DashboardFilterSchema.parse(structuredClone(input)); - // A static filter references nothing in the workspace - if (isStaticListFilter(filter)) return filter; - - const getSourceName = (id: string) => - sources.find(source => source.id === id)?.name ?? ''; + // These reference nothing in the workspace + if (isStaticListFilter(filter) || isConnectionFilter(filter)) return filter; + if (isDataSourceFilter(filter)) { + if (filter.connection) { + filter.connection = getConnectionName(filter.connection) || undefined; + } + return filter; + } if (isAdhocFilter(filter)) { filter.sources = filter.sources.map(getSourceName); @@ -969,13 +972,13 @@ export function convertToDashboardTemplate( }; for (const tile of input.tiles) { - output.tiles.push(convertToTileTemplate(tile, sources, connections)); + output.tiles.push(convertToTileTemplate(tile)); } if (input.filters) { output.filters = []; for (const filter of input.filters ?? []) { - output.filters.push(convertToFilterTemplate(filter, sources)); + output.filters.push(convertToFilterTemplate(filter)); } } @@ -988,8 +991,16 @@ export function convertToDashboardTemplate( output.savedQueryLanguage = input.savedQueryLanguage ?? null; } - if (input.savedFilterValues && input.savedFilterValues.length > 0) { - output.savedFilterValues = structuredClone(input.savedFilterValues); + // A reference variable's saved value is an ID from this workspace + const referenceVariableNames = new Set( + getReferenceVariableFilters(input.filters).map(({ name }) => name), + ); + const savedFilterValues = (input.savedFilterValues ?? []).filter( + value => + !(value.type === 'variable' && referenceVariableNames.has(value.name)), + ); + if (savedFilterValues.length > 0) { + output.savedFilterValues = structuredClone(savedFilterValues); } return output; @@ -1728,6 +1739,8 @@ export function displayTypeSupportsPromQLAlerts( export function savedChartConfigSupportsAlerts( config: SavedChartConfig, ): boolean { + // An alert must query one fixed source, not the viewer's current selection. + if (configUsesReferenceVariable(config)) return false; if (isPromqlSavedChartConfig(config)) { return displayTypeSupportsPromQLAlerts(config.displayType); } diff --git a/packages/common-utils/src/dashboardFilterValues.ts b/packages/common-utils/src/dashboardFilterValues.ts index 2ea039b02b..a0e7922d12 100644 --- a/packages/common-utils/src/dashboardFilterValues.ts +++ b/packages/common-utils/src/dashboardFilterValues.ts @@ -14,6 +14,7 @@ import { } from '@/filters'; import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig } from '@/guards'; import { hasMacro, isMissingFiltersMacro } from '@/macros'; +import { getReferenceVariableFilters } from '@/referenceVariables'; import { AdhocDashboardFilter, AdhocFilterValue, @@ -269,6 +270,8 @@ export function getAdhocFilterTargetsForTile( * 2. The tile references the filter's variable. * 3. The filter applies via broadcast to the tile's source, and the tile * applies broadcast conditions (`consumesBroadcastFilters`). + * 4. The filter is a data source or connection variable the tile's stored + * config references. */ export function getBlockingRequiredFilters( unsatisfiedRequiredFilters: DashboardFilter[], @@ -277,8 +280,16 @@ export function getBlockingRequiredFilters( referencedVariableNames?: readonly string[]; /** Whether the tile's query applies broadcast filter conditions. */ consumesBroadcastFilters: boolean; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }, ): DashboardFilter[] { + const referenceVariableNames = new Set(tile.referenceVariableNames ?? []); + const referenceVariableFilterIds = new Set( + getReferenceVariableFilters(unsatisfiedRequiredFilters) + .filter(({ name }) => referenceVariableNames.has(name)) + .map(({ filter }) => filter.id), + ); const referencedNamesSet = new Set(tile.referencedVariableNames ?? []); const referencedFilterIds = new Set( getDashboardVariableFilters(unsatisfiedRequiredFilters) @@ -290,6 +301,7 @@ export function getBlockingRequiredFilters( filter => isFilterGlobalRequirement(filter) || referencedFilterIds.has(filter.id) || + referenceVariableFilterIds.has(filter.id) || (tile.consumesBroadcastFilters && doesFilterApplyToSource(filter, tile.sourceId)), ); @@ -301,15 +313,19 @@ export function getBlockingRequiredFilterNames({ sourceId, unsatisfiedRequiredFilters, referencedVariables, + referenceVariableNames, }: { config: SavedChartConfig | ChartConfigWithOptDateRange; sourceId: string | undefined; unsatisfiedRequiredFilters: DashboardFilter[] | undefined; /** The tile's variables, already narrowed to the ones it references. */ referencedVariables: readonly { name: string }[] | undefined; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }): string[] { return getBlockingRequiredFilters(unsatisfiedRequiredFilters ?? [], { sourceId, + referenceVariableNames, referencedVariableNames: referencedVariables?.map( variable => variable.name, ), diff --git a/packages/common-utils/src/dashboardValidation.ts b/packages/common-utils/src/dashboardValidation.ts index 8a88bf999e..f08a52584e 100644 --- a/packages/common-utils/src/dashboardValidation.ts +++ b/packages/common-utils/src/dashboardValidation.ts @@ -7,7 +7,22 @@ import { isFilterBroadcastEnabled, isFilterVariableEnabled, } from './filters'; -import { DashboardContainerSchema, DisplayType } from './types'; +import { + getConfigReferenceFields, + getConnectionVariableNames, + getFilterReferenceFields, + getReferenceFieldValues, + getSourceVariableFilters, + getStringField, + getUnacceptedSourceKinds, + parseVariableReference, + ReferenceField, +} from './referenceVariables'; +import { + DashboardContainerSchema, + DashboardFilter, + DisplayType, +} from './types'; // Inputs shared by the internal `DashboardSchema` refinement and the // external API body schema: the validation only depends on the @@ -374,3 +389,115 @@ export function validateChartConfigFormulas( }); } } + +/** The issues a refinement raises, run on its own outside a schema. */ +export function collectRefinementIssues( + refine: (ctx: z.RefinementCtx) => void, +): z.ZodIssue[] { + const result = z + .object({}) + .superRefine((_value, ctx) => refine(ctx)) + .safeParse({}); + return result.success ? [] : result.error.issues; +} + +/** + * Every reference variable reference must name a variable of the right type + * on the dashboard: `$name` in a source field names a data source variable + * offering only kinds the field accepts; `$name` in a connection field names a + * connection variable. The fields are those of `getConfigReferenceFields` and + * `getFilterReferenceFields`. + * + * Issues raised: + * - An unknown variable, or one of the wrong type (path `[i].config.` or `[i].[j]`). + * - A data source variable offering kinds the field does not accept (same paths). + */ +export function validateDashboardVariableReferences( + // Structural so stored documents (whose alerts carry server fields) fit + tiles: { config: object }[], + filters: DashboardFilter[], + ctx: z.RefinementCtx, + paths?: { + tilesPath?: (string | number)[]; + filtersPath?: (string | number)[]; + }, +): void { + const tilesPath = paths?.tilesPath ?? ['tiles']; + const filtersPath = paths?.filtersPath ?? ['filters']; + const kindsByName = new Map( + getSourceVariableFilters(filters).map(({ filter, name }) => [ + name, + filter.allowedSourceKinds, + ]), + ); + const connectionVariableNames = new Set(getConnectionVariableNames(filters)); + + const check = ( + value: string, + field: ReferenceField, + path: (string | number)[], + ) => { + const name = parseVariableReference(value); + if (name == null) return; + if (field.target === 'connection') { + if (connectionVariableNames.has(name)) return; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a connection variable on this dashboard`, + path, + }); + return; + } + const kinds = kindsByName.get(name); + if (kinds == null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a data source variable on this dashboard`, + path, + }); + return; + } + const unaccepted = getUnacceptedSourceKinds( + { allowedSourceKinds: kinds }, + field.acceptedKinds, + ); + if (unaccepted.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Data source variable "${value}" offers ${unaccepted.join(', ')} sources, which cannot be used here`, + path, + }); + } + }; + + const checkFields = ( + object: object, + fields: ReferenceField[], + path: (string | number)[], + ) => { + for (const field of fields) { + const value = getStringField(object, field.key); + if (value != null) { + check(value, field, [...path, field.key]); + } else { + getReferenceFieldValues(object, field.key).forEach((entry, i) => + check(entry, field, [...path, field.key, i]), + ); + } + } + }; + + tiles.forEach((tile, tileIdx) => + checkFields(tile.config, getConfigReferenceFields(tile.config), [ + ...tilesPath, + tileIdx, + 'config', + ]), + ); + filters.forEach((filter, filterIdx) => + checkFields(filter, getFilterReferenceFields(filter), [ + ...filtersPath, + filterIdx, + ]), + ); +} diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 3412e16d8c..5115216b59 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -938,6 +938,37 @@ export function isAdhocFilter( return filter.type === 'ADHOC'; } +/** Type guard for DATA_SOURCE type filters. */ +export function isDataSourceFilter( + filter: T, +): filter is Extract { + return filter.type === 'DATA_SOURCE'; +} + +/** Type guard for CONNECTION type filters. */ +export function isConnectionFilter( + filter: T, +): filter is Extract { + return filter.type === 'CONNECTION'; +} + +/** + * Whether a filter type is a reference variable: a data source or connection + * variable, whose value is an ID that tiles and filters use in place of one. + */ +export function isReferenceVariableFilterType( + type: DashboardFilterKind | undefined, +): type is 'DATA_SOURCE' | 'CONNECTION' { + return type === 'DATA_SOURCE' || type === 'CONNECTION'; +} + +/** Type guard for reference variable filters; see `isReferenceVariableFilterType`. */ +export function isReferenceVariableFilter< + T extends { type: DashboardFilterKind }, +>(filter: T): filter is Extract { + return isReferenceVariableFilterType(filter.type); +} + /** * The sources an ADHOC filter broadcasts its conditions to: its * `appliesToSourceIds`, or its own `sources` when that is unset or empty. @@ -1211,6 +1242,7 @@ export type DashboardVariableDeclaration = Pick< /** Minimal projection of fields necessary to extract the variables a dashboard declares. */ export type FilterForVariableDeclaration = { + type?: DashboardFilterKind; name: string; expression?: string; variableName?: string; @@ -1220,17 +1252,33 @@ export type FilterForVariableDeclaration = { /** * The variable-enabled filters a dashboard declares, paired with the name each - * one answers to, in filter order. + * one answers to, in filter order. Data source variables are left out: their + * value selects a tile's source or connection and is never substituted into + * query text. */ export function getDashboardVariableFilters< T extends FilterForVariableDeclaration, >(filters: T[] | undefined): { filter: T; name: string }[] { + return pairWithVariableNames( + (filters ?? []).filter( + filter => + isFilterVariableEnabled(filter) && + !isReferenceVariableFilterType(filter.type), + ), + ); +} + +/** + * Each filter paired with the name it answers to, skipping filters with no + * usable name. + */ +export function pairWithVariableNames< + T extends { name: string; variableName?: string }, +>(filters: T[]): { filter: T; name: string }[] { const results: { filter: T; name: string }[] = []; const takenNames = new Set(); - for (const filter of filters ?? []) { - if (!isFilterVariableEnabled(filter)) continue; - + for (const filter of filters) { // There shouldn't be any duplicate names, but if there are then the first one wins. const name = getFilterVariableName(filter); if (!name || takenNames.has(name)) continue; diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts new file mode 100644 index 0000000000..7dd476b3da --- /dev/null +++ b/packages/common-utils/src/referenceVariables.ts @@ -0,0 +1,366 @@ +/** + * Reference variables are dashboard variables whose value is the ID of a + * workspace object: a data source variable selects a source, a connection + * variable selects a connection. Tiles and filters reference them as `$name` + * in the fields that otherwise hold that ID. + */ +import { + getAdhocFilterSourceKinds, + isConnectionFilter, + isDataSourceFilter, + isReferenceVariableFilter, + pairWithVariableNames, + QUERY_EXPRESSION_FILTER_SOURCE_KINDS, +} from '@/filters'; +import { + ConnectionDashboardFilter, + DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, + DashboardFilter, + DataSourceDashboardFilter, + NON_PROMQL_SOURCE_KINDS, + SourceKind, +} from '@/types'; + +/** + * The variable name a field references, if it holds a `$name` reference + * rather than an ID. The whole field is the token. + */ +export function parseVariableReference( + value: string | undefined | null, +): string | undefined { + if (!value?.startsWith('$')) return undefined; + const name = value.slice(1); + return DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED.test(name) ? name : undefined; +} + +export function toVariableReference(name: string): string { + return `$${name}`; +} + +/** A field's value, read without assuming the object's shape. */ +function getField(object: object, field: string): unknown { + return field in object ? Reflect.get(object, field) : undefined; +} + +/** A field's value when it is a string; undefined when absent or not a string. */ +export function getStringField( + object: object, + field: string, +): string | undefined { + const value = getField(object, field); + return typeof value === 'string' ? value : undefined; +} + +/** + * An object's name, or the reference itself for a variable reference. + * Undefined when no object has the ID. + */ +export function getObjectName( + id: string, + objects: readonly { id: string; name: string }[] | undefined, +): string | undefined { + if (parseVariableReference(id) != null) return id; + return objects?.find(object => object.id === id)?.name; +} + +/** Selected ID per reference variable name; undefined when unselected. */ +export type ReferenceVariableValues = ReadonlyMap; + +/** A data source variable's identity, as offered by source pickers. */ +export type SourceVariableDeclaration = { + name: string; + allowedSourceKinds: SourceKind[]; + /** A connection ID or connection variable reference the sources are limited to. */ + connection?: string; +}; + +/** + * The kinds a data source variable can resolve to that a field accepting + * `acceptedKinds` cannot use. + */ +export function getUnacceptedSourceKinds( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): SourceKind[] { + return variable.allowedSourceKinds.filter( + kind => !acceptedKinds.includes(kind), + ); +} + +/** + * Whether a field accepting `acceptedKinds` can use a data source variable, + * whatever source it selects. + */ +export function isSourceVariableAccepted( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): boolean { + return getUnacceptedSourceKinds(variable, acceptedKinds).length === 0; +} + +/** `isSourceVariableAccepted` for the variable that `source` references. */ +export function isSourceVariableReferenceAccepted( + source: string | undefined, + declarations: SourceVariableDeclaration[], + acceptedKinds: readonly SourceKind[], +): boolean { + const name = parseVariableReference(source); + const variable = declarations.find(v => v.name === name); + return variable != null && isSourceVariableAccepted(variable, acceptedKinds); +} + +/** + * Whether a data source variable can select `source`. `connection` is the + * variable's connection already resolved; a still unresolved reference + * matches no source. + */ +export function isSourceOfferedByVariable( + source: { kind: SourceKind; connection: string; disabled?: boolean }, + variable: Pick< + SourceVariableDeclaration, + 'allowedSourceKinds' | 'connection' + >, +): boolean { + return ( + !source.disabled && + variable.allowedSourceKinds.includes(source.kind) && + (!variable.connection || source.connection === variable.connection) + ); +} + +/** The data source variables a dashboard declares, in filter order. */ +export function getSourceVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: DataSourceDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isDataSourceFilter)); +} + +/** The connection variables a dashboard declares, in filter order. */ +export function getConnectionVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: ConnectionDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isConnectionFilter)); +} + +/** + * The data source and connection variables a dashboard declares, in filter + * order. Structural so the external API's filter shape fits too. + */ +export function getReferenceVariableFilters< + T extends { + type: DashboardFilter['type']; + name: string; + variableName?: string; + }, +>( + filters: T[] | undefined, +): { + filter: Extract; + name: string; +}[] { + return pairWithVariableNames( + (filters ?? []).filter(isReferenceVariableFilter), + ); +} + +export function getSourceVariableDeclarations( + filters: DashboardFilter[] | undefined, +): SourceVariableDeclaration[] { + return getSourceVariableFilters(filters).map(({ filter, name }) => ({ + name, + allowedSourceKinds: filter.allowedSourceKinds, + connection: filter.connection, + })); +} + +export function getConnectionVariableNames( + filters: DashboardFilter[] | undefined, +): string[] { + return getConnectionVariableFilters(filters).map(({ name }) => name); +} + +/** + * Resolve a field: an ID passes through, a reference resolves to the + * variable's selection (undefined while unselected). + */ +export function resolveVariableReference( + value: string | undefined, + values: ReferenceVariableValues | undefined, +): string | undefined { + const name = parseVariableReference(value); + if (name == null) return value; + return values?.get(name); +} + +/** + * A field that can hold a reference variable reference: what it may reference, + * and, for a source field, the source kinds it accepts. + */ +export type ReferenceField = + | { key: K; target: 'source'; acceptedKinds: readonly SourceKind[] } + | { key: K; target: 'connection' }; + +/** The fields of a chart config that can hold a reference. */ +export function getConfigReferenceFields( + config: object, +): ReferenceField<'source' | 'connection'>[] { + const isPromql = getStringField(config, 'configType') === 'promql'; + return [ + { + key: 'source', + target: 'source', + acceptedKinds: isPromql ? [SourceKind.Promql] : NON_PROMQL_SOURCE_KINDS, + }, + { key: 'connection', target: 'connection' }, + ]; +} + +/** The fields of a filter that can hold a reference. */ +export function getFilterReferenceFields( + filter: DashboardFilter, +): ReferenceField< + 'source' | 'sources' | 'appliesToSourceIds' | 'connection' +>[] { + switch (filter.type) { + case 'QUERY_EXPRESSION': + return (['source', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: QUERY_EXPRESSION_FILTER_SOURCE_KINDS, + })); + case 'PROMETHEUS_LABEL': + return [ + { key: 'source', target: 'source', acceptedKinds: [SourceKind.Promql] }, + ]; + case 'ADHOC': + return (['sources', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: getAdhocFilterSourceKinds(filter.sourceType), + })); + case 'DATA_SOURCE': + return [{ key: 'connection', target: 'connection' }]; + case 'STATIC_LIST': + case 'CONNECTION': + return []; + default: + filter satisfies never; + return []; + } +} + +/** A field's references and IDs: one for a string field, each entry of a list. */ +export function getReferenceFieldValues(object: object, key: string): string[] { + const value = getField(object, key); + if (typeof value === 'string') return [value]; + return Array.isArray(value) + ? value.filter((v): v is string => typeof v === 'string') + : []; +} + +/** Each field's value with `resolve` applied, keeping strings and lists as such. */ +function mapReferenceFields( + object: T, + keys: readonly string[], + resolve: (value: string) => string, +): T { + let mapped = object; + for (const key of keys) { + const value = getField(object, key); + if (typeof value === 'string') { + const next = resolve(value); + if (next !== value) mapped = { ...mapped, [key]: next }; + } else if (Array.isArray(value)) { + const next = value.map(v => (typeof v === 'string' ? resolve(v) : v)); + if (next.some((v, i) => v !== value.at(i))) { + mapped = { ...mapped, [key]: next }; + } + } + } + return mapped; +} + +/** The variables named by references in the given fields, deduplicated. */ +function getReferencedVariables( + object: object, + keys: readonly string[], +): string[] { + return Array.from( + new Set( + keys + .flatMap(key => getReferenceFieldValues(object, key)) + .map(parseVariableReference) + .filter((name): name is string => name != null), + ), + ); +} + +/** The reference variables a chart config's `source` and `connection` name. */ +export function getConfigReferenceVariables(config: object): string[] { + return getReferencedVariables( + config, + getConfigReferenceFields(config).map(({ key }) => key), + ); +} + +/** Whether a chart config's source or connection is a variable reference. */ +export function configUsesReferenceVariable(config: object): boolean { + return getConfigReferenceVariables(config).length > 0; +} + +/** + * Replace the references in a config's `source` and `connection` with the + * selected IDs. An unselected reference is left as written. Returns `config` + * itself when nothing changes. + */ +export function resolveConfigReferences( + config: T, + values: ReferenceVariableValues | undefined, +): T { + return mapReferenceFields( + config, + getConfigReferenceFields(config).map(({ key }) => key), + value => resolveVariableReference(value, values) || value, + ); +} + +/** + * The inverse of `resolveConfigReferences`: put the references `stored` holds + * back on a config derived from its resolved form. + */ +export function restoreConfigReferences( + config: T, + stored: object, +): T { + let restored = config; + for (const { key } of getConfigReferenceFields(stored)) { + const value = getStringField(stored, key); + if (parseVariableReference(value) != null && key in config) { + restored = { ...restored, [key]: value }; + } + } + return restored; +} + +/** + * Replace the references in a filter's source and connection fields with the + * selected IDs. An unselected reference is left as written, so it matches no + * tile's source rather than widening a list to every tile, and is reported in + * `unresolvedVariables`. + */ +export function resolveFilterReferences( + filter: T, + values: ReferenceVariableValues | undefined, +): { filter: T; unresolvedVariables: string[] } { + const keys = getFilterReferenceFields(filter).map(({ key }) => key); + return { + filter: mapReferenceFields( + filter, + keys, + value => resolveVariableReference(value, values) || value, + ), + unresolvedVariables: getReferencedVariables(filter, keys).filter( + name => !values?.get(name), + ), + }; +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dba9c7854e..30a4c90f6a 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2295,12 +2295,27 @@ export const DashboardContainerSchema = z.object({ export type DashboardContainer = z.infer; +export enum SourceKind { + Log = 'log', + Trace = 'trace', + Session = 'session', + Metric = 'metric', + Promql = 'promql', +} + +/** The source kinds builder and raw SQL charts can query. */ +export const NON_PROMQL_SOURCE_KINDS: readonly SourceKind[] = Object.values( + SourceKind, +).filter(kind => kind !== SourceKind.Promql); + /** Type of dashboard filter, determining how its dropdown values are populated. */ export const DashboardFilterType = z.enum([ 'QUERY_EXPRESSION', 'STATIC_LIST', 'PROMETHEUS_LABEL', 'ADHOC', + 'DATA_SOURCE', + 'CONNECTION', ]); /** Allowed variable names for dashboard filters. Alphanumeric + underscore, must start with a letter. */ @@ -2440,11 +2455,52 @@ export const AdhocDashboardFilterSchema = dashboardFilterBaseSchema }) .strict(); +/** + * A variable whose single selected value is a source ID. Tiles and other + * filters reference it as `$variableName` in place of a source ID. Always + * single-select and required, since a tile cannot query without a source. + */ +export const DataSourceDashboardFilterSchema = dashboardFilterBaseSchema + .omit({ isGlobalRequirement: true }) + .extend({ + type: z.literal(DashboardFilterType.enum.DATA_SOURCE), + /** The kinds of sources offered as values. */ + allowedSourceKinds: z.array(z.nativeEnum(SourceKind)).min(1), + /** + * Limits the offered sources to one connection: a connection ID, or a + * `$name` reference to a connection variable. + */ + connection: z.string().min(1).optional(), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + +/** + * A variable whose single selected value is a connection ID. Raw SQL tiles + * and data source variables reference it as `$variableName` in place of a + * connection ID. + */ +export const ConnectionDashboardFilterSchema = dashboardFilterBaseSchema + .omit({ isGlobalRequirement: true }) + .extend({ + type: z.literal(DashboardFilterType.enum.CONNECTION), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + export const DashboardFilterSchema = z.discriminatedUnion('type', [ QueryExpressionDashboardFilterSchema, StaticListDashboardFilterSchema, PromqlLabelDashboardFilterSchema, AdhocDashboardFilterSchema, + DataSourceDashboardFilterSchema, + ConnectionDashboardFilterSchema, ]); export type QueryExpressionDashboardFilter = z.infer< @@ -2457,6 +2513,12 @@ export type PromqlLabelDashboardFilter = z.infer< typeof PromqlLabelDashboardFilterSchema >; export type AdhocDashboardFilter = z.infer; +export type DataSourceDashboardFilter = z.infer< + typeof DataSourceDashboardFilterSchema +>; +export type ConnectionDashboardFilter = z.infer< + typeof ConnectionDashboardFilterSchema +>; export type DashboardFilter = z.infer; export enum PresetDashboard { @@ -2612,13 +2674,6 @@ export type Team = z.infer; // -------------------------- // TABLE SOURCES // -------------------------- -export enum SourceKind { - Log = 'log', - Trace = 'trace', - Session = 'session', - Metric = 'metric', - Promql = 'promql', -} // -------------------------- // TABLE SOURCE FORM VALIDATION From e3a89419fec8a165ea5644470ce51c437f138f3d Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 13 Nov 2026 13:54:23 -0500 Subject: [PATCH 2/2] feat: support data source and connection variables in the external API and MCP --- .changeset/source-vars-2.md | 5 + packages/api/openapi.json | 124 ++++++++++++++--- .../dashboards/patchDashboard.int.test.ts | 72 ++++++++++ .../dashboards/queryTile.int.test.ts | 82 ++++++++++++ .../dashboards/saveDashboard.int.test.ts | 73 ++++++++++ .../__tests__/dashboards/validation.test.ts | 29 ++++ .../api/src/mcp/prompts/dashboards/content.ts | 2 + .../mcp/tools/dashboards/patchDashboard.ts | 1 + .../api/src/mcp/tools/dashboards/queryTile.ts | 21 ++- .../src/mcp/tools/dashboards/queryTiles.ts | 27 +++- .../src/mcp/tools/dashboards/saveDashboard.ts | 1 + .../api/src/mcp/tools/dashboards/schemas.ts | 126 +++++++++++++++--- .../src/mcp/tools/dashboards/validation.ts | 16 ++- .../api/src/mcp/tools/dashboards/variables.ts | 85 +++++++++++- .../__tests__/dashboards.int.test.ts | 96 +++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 109 +++++++++++++-- .../external-api/v2/utils/dashboards.ts | 71 ++++++---- packages/api/src/utils/externalApi.ts | 26 +++- packages/api/src/utils/zod.ts | 66 +++++++-- 19 files changed, 923 insertions(+), 109 deletions(-) create mode 100644 .changeset/source-vars-2.md diff --git a/.changeset/source-vars-2.md b/.changeset/source-vars-2.md new file mode 100644 index 0000000000..32fd79c38c --- /dev/null +++ b/.changeset/source-vars-2.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': minor +--- + +feat: support data source and connection variables in the external API and MCP diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3faea..a4b2dd232e 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -1809,7 +1809,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -1906,7 +1906,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -1983,7 +1983,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2070,7 +2070,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2153,7 +2153,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2214,7 +2214,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2269,7 +2269,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2363,7 +2363,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2421,7 +2421,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2485,7 +2485,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2525,7 +2525,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to mine patterns from.", + "description": "ID of the data source to mine patterns from, or `$name` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2590,7 +2590,7 @@ }, "connectionId": { "type": "string", - "description": "ID of the ClickHouse connection to execute the query against.", + "description": "ID of the ClickHouse connection to execute the query against, or\n`$name` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, "sqlTemplate": { @@ -2601,7 +2601,7 @@ }, "sourceId": { "type": "string", - "description": "Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.", + "description": "Optional ID of the data source associated with this Raw SQL chart, or `$name` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters.", "example": "65f5e4a3b9e77c001a567890" }, "numberFormat": { @@ -3951,6 +3951,12 @@ }, { "$ref": "#/components/schemas/PrometheusLabelFilterInput" + }, + { + "$ref": "#/components/schemas/DataSourceFilterInput" + }, + { + "$ref": "#/components/schemas/ConnectionFilterInput" } ], "discriminator": { @@ -3958,7 +3964,95 @@ "mapping": { "QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput", "STATIC_LIST": "#/components/schemas/StaticListFilterInput", - "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput" + "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", + "DATA_SOURCE": "#/components/schemas/DataSourceFilterInput", + "CONNECTION": "#/components/schemas/ConnectionFilterInput" + } + } + }, + "DataSourceFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a source. Tiles reference it\nas `$variableName` in their sourceId, and other filters in sourceId or\nappliesToSourceIds, so viewers can switch the source those read. Tiles\nthat use it do not load until a source is selected, and cannot have\nalerts.\n", + "required": [ + "type", + "name", + "allowedSourceKinds" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "DATA_SOURCE" + ], + "description": "Filter type discriminator. Must be \"DATA_SOURCE\".", + "example": "DATA_SOURCE" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Logs source" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token tiles and filters reference the selected source by, as `$variableName`.", + "example": "logs" + }, + "allowedSourceKinds": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "log", + "trace", + "session", + "metric", + "promql" + ] + }, + "description": "Kinds of sources offered. A field can only reference the variable\nif it accepts every listed kind.\n", + "example": [ + "log" + ] + }, + "connectionId": { + "type": "string", + "description": "Optional connection the offered sources are limited to: a\nconnection ID, or `$name` of a CONNECTION filter on the dashboard.\n", + "example": "$conn" + } + } + }, + "ConnectionFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables in theirs. Tiles that use it do not load until a connection is\nselected, and cannot have alerts.\n", + "required": [ + "type", + "name" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "CONNECTION" + ], + "description": "Filter type discriminator. Must be \"CONNECTION\".", + "example": "CONNECTION" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Connection" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token raw SQL tiles reference the selected connection by, as `$variableName`.", + "example": "conn" } } }, @@ -3994,7 +4088,7 @@ }, "sourceId": { "type": "string", - "description": "Source ID this dashboard filter key applies to", + "description": "Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard", "example": "65f5e4a3b9e77c001a111111" }, "sourceMetricType": { diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts index eef1d08387..21e4e6b12e 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts @@ -1400,4 +1400,76 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { expect(tile.config.showOperandSeries).toBe(false); }); }); + + describe('reference variables', () => { + const createDashboard = () => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Tile', + source: ctx.traceSource._id.toString(), + displayType: 'line', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + team: ctx.team._id, + }).save(); + + const patchSource = (dashboardId: string, sourceId: string) => + callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId, + tileId: 'tile', + tile: { + name: 'Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + }); + + it('patches a tile to read from a declared data source variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$traces'); + + expect(result.isError).toBeFalsy(); + const stored = await Dashboard.findById(dashboard._id).lean(); + expect(stored?.tiles[0].config).toMatchObject({ source: '$traces' }); + }); + + it('rejects a patch referencing an undeclared variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$nope'); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts index 48de1b8f67..5042fc502c 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts @@ -11,6 +11,7 @@ import { seedExponentialHistogramMetric, } from '@/fixtures'; import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Dashboard from '@/models/dashboard'; import { Source } from '@/models/source'; import { setupDashboardTests } from './setup'; @@ -1373,4 +1374,85 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => { expect(rows.map(r => r.ServiceName)).toEqual(['svc-e', 'svc-d', 'svc-c']); }); }); + + describe('reference variables', () => { + const createVariableDashboard = (savedFilterValues: unknown[] = []) => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'variable-tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Count', + source: '$traces', + displayType: 'number', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + savedFilterValues, + team: ctx.team._id, + }).save(); + + it("runs a tile on its data source variable's saved selection", async () => { + const dashboard = await createVariableDashboard([ + { + type: 'variable', + name: 'traces', + values: [ctx.traceSource._id.toString()], + }, + ]); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBeFalsy(); + }); + + it('runs a tile on a selection passed in variableValues', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + variableValues: [ + { name: 'traces', values: [ctx.traceSource._id.toString()] }, + ], + }); + + expect(result.isError).toBeFalsy(); + }); + + it('reports a data source variable with no selection', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('$traces variable'); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 4940948fd0..cc60d29e48 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -4027,4 +4027,77 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { expect(getFirstText(patchResult)).toContain('savedFilterValues'); }); }); + + describe('reference variables', () => { + const filters = [ + { type: 'CONNECTION', name: 'Connection', variableName: 'conn' }, + { + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }, + ]; + + it('saves and returns variables and tiles that reference them', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Raw SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }, + ], + filters, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.filters).toEqual([ + expect.objectContaining(filters[0]), + expect.objectContaining(filters[1]), + ]); + expect( + output.tiles.map((t: ExternalDashboardTileWithId) => t.config), + ).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index 260e5fd650..0afb76619b 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -11,6 +11,7 @@ import { getRawSqlMissingSourceError, getRawSqlTileMacroWarnings, getRawSqlTilesMissingRequiredSource, + getSavedFilterValueWarnings, getTileVariableWarnings, } from '@/mcp/tools/dashboards/validation'; import type { @@ -922,3 +923,31 @@ describe('getFilterVariableWarnings', () => { expect(joined).toContain('Available variables: service'); }); }); + +describe('getSavedFilterValueWarnings', () => { + const filters = [ + { type: 'DATA_SOURCE' as const, name: 'Traces', variableName: 'traces' }, + { type: 'CONNECTION' as const, name: 'Cluster', variableName: 'conn' }, + ]; + + it('accepts saved values for data source and connection variables', () => { + expect( + getSavedFilterValueWarnings( + [ + { type: 'variable', name: 'traces', values: ['s1'] }, + { type: 'variable', name: 'conn', values: ['c1'] }, + ], + filters, + ), + ).toEqual([]); + }); + + it('warns about a saved value no filter declares', () => { + expect( + getSavedFilterValueWarnings( + [{ type: 'variable', name: 'nope', values: ['x'] }], + filters, + ), + ).toHaveLength(1); + }); +}); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 84c8199bb3..a778a2b3d7 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -135,6 +135,8 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 9g. SET savedFilterValues TO GIVE THE DASHBOARD A DEFAULT SELECTION. Pass savedFilterValues on clickstack_save_dashboard or clickstack_patch_dashboard to pre-select filter values whenever the dashboard opens. Pair this with a required filter (9e) so the dashboard opens scoped rather than blocked. +9h. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). + 10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. 11. GROUP RELATED TILES INTO CONTAINERS. REQUIRED at five or more tiles, no exceptions. An ungrouped wall of nine or ten tiles is a readability failure even when each tile is correct in isolation. Containers are the right way to introduce structure; markdown tiles for section labels are not. diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index aeea54d7bd..3ab90c136e 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -202,6 +202,7 @@ export function registerPatchDashboard({ teamId, tiles: [mergedTile], existingTiles: existingDashboard.tiles ?? [], + storedFilters: existingDashboard.filters, containers: existingDashboard.containers ?? [], }); if (validationError) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b57..4517a374e3 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -13,7 +13,11 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; export function registerQueryTile({ context, @@ -79,8 +83,8 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), }); - const tile = externalDashboard.tiles.find(t => t.id === tileId); - if (!tile) { + const storedTile = externalDashboard.tiles.find(t => t.id === tileId); + if (!storedTile) { return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => t.id).join(', ')}`, ); @@ -94,6 +98,17 @@ export function registerQueryTile({ return mcpUserError(resolvedVariables.error); } + const resolvedTile = resolveTileVariableReferences( + storedTile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + ); + if ('error' in resolvedTile) { + return mcpUserError(resolvedTile.error); + } + const { tile } = resolvedTile; + const result = await runConfigTile( teamId.toString(), tile, diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c950..6940ed7974 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -19,7 +19,11 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; /** * How many tiles to query against ClickHouse at once. Kept lowish so a batch @@ -251,6 +255,16 @@ export function registerQueryTiles({ return { ...base, status: 'skipped' }; } + const resolvedTile = resolveTileVariableReferences( + tile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + ); + if ('error' in resolvedTile) { + return { ...base, status: 'error', error: resolvedTile.error }; + } + // Run the query under the shared deadline. It's passed as a thunk // so withDeadline can skip issuing it entirely once the budget is // spent (see withDeadline). withDeadline hands the thunk an @@ -263,10 +277,13 @@ export function registerQueryTiles({ try { const result = await withDeadline( signal => - runConfigTile(teamId.toString(), tile, startDate, endDate, { - abortSignal: signal, - variables, - }), + runConfigTile( + teamId.toString(), + resolvedTile.tile, + startDate, + endDate, + { abortSignal: signal, variables }, + ), deadlineAt, ); diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index ba0c0d21e3..58722d33de 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -324,6 +324,7 @@ async function updateDashboard({ tiles: tilesWithId, filters, existingTiles: existingDashboard.tiles ?? [], + storedFilters: existingDashboard.filters, containers: effectiveContainers, }); if (validationError) { diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index f9e40efe58..9eaf2bc670 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -23,6 +23,7 @@ import { PromqlQueryTypeSchema, PromqlReducerSchema, SearchConditionTrimmedLanguageSchema, + SourceKind, SQLIntervalSchema, StaticListDashboardFilterSchema, } from '@hyperdx/common-utils/dist/types'; @@ -36,10 +37,18 @@ import { QUERYABLE_METRIC_KINDS } from '@/mcp/tools/sources/metricKinds'; import { externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + objectIdOrVariableReferenceSchema, objectIdSchema, tagsSchema, } from '@/utils/zod'; +const SOURCE_VARIABLE_REFERENCE_NOTE = + 'May instead be `$name`, a reference to a DATA_SOURCE filter (variable) on this ' + + 'dashboard, so viewers can switch the source; the variable must only offer kinds ' + + 'this field accepts.'; + +const SOURCE_ID_DESCRIPTION = `Source ID – call clickstack_list_sources. ${SOURCE_VARIABLE_REFERENCE_NOTE}`; + /** * Metric type values exposed on dashboard tile select items. Restricted to * the kinds the query renderer can translate today; summary is intentionally @@ -646,7 +655,7 @@ const mcpTimeChartDisplayFields = { }; const mcpRowListSourceFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), where: z .string() .optional() @@ -734,7 +743,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('line').describe('Line chart over time'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -771,7 +780,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ }); const mcpStackedTimeChartConfigFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -823,7 +832,7 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('table').describe('Tabular aggregated data'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -879,7 +888,7 @@ export const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('number') .describe('Single aggregate scalar value'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -920,7 +929,7 @@ const mcpPieTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('pie').describe('Pie chart'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -968,7 +977,7 @@ const mcpCategoricalBarTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('bar') .describe('Bar chart — one bar per group value (not a time series)'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -1056,7 +1065,8 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ .string() .describe( 'Source ID. Must be a Trace source today; use clickstack_list_sources and ' + - 'pick one whose kind is "trace".', + 'pick one whose kind is "trace". ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpHeatmapSelectItemSchema) @@ -1101,7 +1111,8 @@ const mcpHeatmapSeriesTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources', + 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpTileSelectItemSchema) @@ -1198,7 +1209,9 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ connectionId: z .string() .describe( - 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections', + 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections. ' + + 'May instead be `$name`, a reference to a CONNECTION filter (variable) on this ' + + 'dashboard, so viewers can switch the connection the query runs on.', ), sourceId: z .string() @@ -1208,7 +1221,8 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ 'ALWAYS set this for raw SQL tiles UNLESS the query reads from multiple tables ' + '(e.g. JOINs or sub-queries spanning several sources), in which case omit it. ' + 'sourceId is REQUIRED by two macros: $__filters and $__sourceTable. ' + - 'The sourceId must belong to the same connection as connectionId.', + 'The sourceId must belong to the same connection as connectionId. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sqlTemplate: z.string().describe(` Raw ClickHouse SQL query. SQL guidelines: @@ -1336,7 +1350,8 @@ const mcpPromqlTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'ID of a source whose kind is "promql" – call clickstack_list_sources', + 'ID of a source whose kind is "promql" – call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), expressions: z .array(mcpPromqlExpressionSchema) @@ -1665,9 +1680,10 @@ const mcpQueryExpressionFilterSchema = z "not declared in any of this dashboard's filters is silently dropped at click time. " + 'Declare an expression here for every column you plan to drive via row-click.', ), - sourceId: objectIdSchema.describe( + sourceId: objectIdOrVariableReferenceSchema.describe( 'Source the filter values are pulled from (for the dropdown). ' + - 'Get IDs from clickstack_list_sources.', + 'Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sourceMetricType: z .nativeEnum(MetricsDataType) @@ -1696,7 +1712,7 @@ const mcpQueryExpressionFilterSchema = z 'Query language of `where`. Set it explicitly.', ), appliesToSourceIds: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .optional() .describe( 'Source IDs (from clickstack_list_sources) that this filter is applied to. ' + @@ -1773,8 +1789,9 @@ const mcpPromqlLabelFilterSchema = z .describe( "Filter type discriminator for a filter whose dropdown lists a Prometheus label's values.", ), - sourceId: objectIdSchema.describe( - 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources.', + sourceId: objectIdOrVariableReferenceSchema.describe( + 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), label: PromqlLabelDashboardFilterSchema.shape.label.describe( 'Label whose values fill the dropdown, e.g. "job". ' + @@ -1791,15 +1808,70 @@ const mcpPromqlLabelFilterSchema = z 'default multi-select, or {job="$job"} with maxSelections: 1.', ); +// Reference variables are always single-select and required, so they take +// none of the selection-limit fields. +const mcpReferenceVariableBaseShape = { + id: mcpDashboardFilterBaseShape.id, + name: mcpDashboardFilterBaseShape.name, + variableName: mcpDashboardFilterBaseShape.variableName, +}; + +const mcpDataSourceFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.DATA_SOURCE) + .describe( + 'Filter type discriminator for a variable whose value is a source.', + ), + allowedSourceKinds: z + .array(z.nativeEnum(SourceKind)) + .min(1) + .describe( + 'Kinds of sources offered in the dropdown. A tile can only use the variable ' + + 'if it accepts every listed kind, so keep kinds that charts treat alike ' + + '(log + trace, or a single kind).', + ), + connectionId: objectIdOrVariableReferenceSchema + .optional() + .describe( + 'Optional connection the offered sources are limited to: a connection ID, or ' + + '`$name` of a CONNECTION filter on this dashboard.', + ), + }) + .describe( + 'A variable whose single, required value is a source. Tiles and other filters ' + + 'reference it as `$variableName` in sourceId (or appliesToSourceIds) instead of a ' + + 'source ID; tiles stay blocked until a source is selected.', + ); + +const mcpConnectionFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.CONNECTION) + .describe( + 'Filter type discriminator for a variable whose value is a connection.', + ), + }) + .describe( + 'A variable whose single, required value is a connection. Raw SQL tiles reference ' + + 'it as `$variableName` in connectionId, and DATA_SOURCE filters in connectionId.', + ); + const mcpDashboardFilterSchema = z .discriminatedUnion('type', [ mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + - 'Two types: QUERY_EXPRESSION queries its dropdown values from a source column; ' + - 'STATIC_LIST declares them inline and is always variable-only.', + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + + 'STATIC_LIST declares them inline and is always variable-only; ' + + 'DATA_SOURCE and CONNECTION select the source or connection that tiles referencing ' + + 'them as `$variableName` use.', ); const mcpPromqlDashboardFilterSchema = z @@ -1807,13 +1879,16 @@ const mcpPromqlDashboardFilterSchema = z mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, mcpPromqlLabelFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + 'STATIC_LIST declares them inline and is always variable-only; ' + 'PROMETHEUS_LABEL lists the values of a label on a promql source and is ' + - 'always variable-only.', + 'always variable-only; DATA_SOURCE and CONNECTION select the source or ' + + 'connection that tiles referencing them as `$variableName` use.', ); export type McpDashboardFilter = z.infer; @@ -1879,7 +1954,16 @@ const mcpFiltersParam = z ']\n' + "A filter's `where` scopes only the values ITS dropdown offers; it is not applied to " + 'any tile. That is why Service keeps broadcasting here: its variable is read by the ' + - 'Endpoint dropdown, not by a tile, so the value is never applied twice.', + 'Endpoint dropdown, not by a tile, so the value is never applied twice.\n\n' + + 'Example (data source variable: viewers pick which logs source every tile reads):\n' + + '[\n' + + ' { "type": "DATA_SOURCE", "name": "Logs source", "variableName": "logs",\n' + + ' "allowedSourceKinds": ["log"] }\n' + + ']\n' + + 'Tiles then set "sourceId": "$logs". A CONNECTION filter works the same way for ' + + 'raw SQL tiles ("connectionId": "$conn"); a DATA_SOURCE filter may set ' + + '"connectionId" to limit its sources to one connection. Alerts are not supported ' + + 'on tiles that use either.', ); const PROMQL_FILTERS_EXAMPLE = diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 987ab6ad2f..faa24fc2c4 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -5,6 +5,8 @@ import { getDashboardVariableDeclarations, getFilterVariableName, isFilterVariableEnabled, + isReferenceVariableFilterType, + pairWithVariableNames, } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, @@ -250,18 +252,24 @@ export function getFilterVariableWarnings( * Non-blocking check: a saved filter value of type 'variable' whose `name` * matches no variable-enabled filter is silently ignored when the dashboard * loads, so a typo'd variable name looks like a saved default but never - * applies. Warn so the agent can fix the name or add the filter. + * applies. Warn so the agent can fix the name or add the filter. Data source + * and connection variables count too, though they are not text variables. */ export function getSavedFilterValueWarnings( savedFilterValues: McpSavedFilterValues | undefined, filters: FilterForVariableDeclaration[] | undefined, ): string[] { if (!savedFilterValues?.length) return []; - const declaredNames = new Set( - getDashboardVariableDeclarations(filters).map( + const declaredNames = new Set([ + ...getDashboardVariableDeclarations(filters).map( declaration => declaration.name, ), - ); + ...pairWithVariableNames( + (filters ?? []).filter(filter => + isReferenceVariableFilterType(filter.type), + ), + ).map(({ name }) => name), + ]); const warnings: string[] = []; for (const value of savedFilterValues) { if (value.type !== 'variable') continue; diff --git a/packages/api/src/mcp/tools/dashboards/variables.ts b/packages/api/src/mcp/tools/dashboards/variables.ts index f1b98a6116..ec4f632f68 100644 --- a/packages/api/src/mcp/tools/dashboards/variables.ts +++ b/packages/api/src/mcp/tools/dashboards/variables.ts @@ -5,15 +5,44 @@ import { isFilterVariableEnabled, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; +import { + getConfigReferenceFields, + getReferenceVariableFilters, + getStringField, + parseVariableReference, +} from '@hyperdx/common-utils/dist/referenceVariables'; import type { ChartVariable } from '@hyperdx/common-utils/dist/types'; import { isFilterSingleSelect } from '@hyperdx/common-utils/dist/variables'; import { z } from 'zod'; +import type { ExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardFilter, ExternalDashboardFilterWithId, + ExternalDashboardTileWithId, } from '@/utils/zod'; +type ExternalFilters = + | (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] + | undefined; + +/** Names of the data source and connection variables a dashboard declares. */ +function getReferenceVariableNames(filters: ExternalFilters): Set { + return new Set(getReferenceVariableFilters(filters).map(({ name }) => name)); +} + +/** The external API name of a chart config field that can hold a reference. */ +function toExternalReferenceKey( + key: ReturnType[number]['key'], +): 'sourceId' | 'connectionId' { + switch (key) { + case 'source': + return 'sourceId'; + case 'connection': + return 'connectionId'; + } +} + /** Optional per-variable selection for a tile run. */ export const mcpVariableValuesParam = z .array( @@ -28,7 +57,9 @@ export const mcpVariableValuesParam = z .array(z.string()) .describe( 'Values to treat as selected. An empty array means nothing is selected for this variable. ' + - "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options.", + "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options. " + + 'For a DATA_SOURCE or CONNECTION variable, pass one source or connection ID; it ' + + "overrides the dashboard's saved selection.", ), }), ) @@ -65,9 +96,11 @@ export function resolveDashboardVariables( if (!variableValues?.length) return { variables }; const byName = new Map(variables.map(variable => [variable.name, variable])); + // Resolved separately, by resolveTileVariableReferences + const referenceVariableNames = getReferenceVariableNames(filters); const unknownVariableValues = variableValues .map(({ name }) => name) - .filter(name => !byName.has(name)); + .filter(name => !byName.has(name) && !referenceVariableNames.has(name)); if (unknownVariableValues.length > 0) { const available = variables.length > 0 @@ -121,12 +154,58 @@ export function resolveDashboardVariables( // Override the default empty selection with the given variableValues for (const { name, values } of variableValues) { - byName.get(name)!.values = values; + const variable = byName.get(name); + if (variable) variable.values = values; } return { variables }; } +/** + * Replace a tile's `$name` sourceId / connectionId with the source or + * connection its data source or connection variable selects: the caller's + * variableValues first, then the dashboard's saved selection. A tile with no + * selection cannot run, as on the dashboard, so that is an error naming the + * variable. + */ +export function resolveTileVariableReferences< + T extends ExternalDashboardTileWithId, +>( + tile: T, + filters: ExternalFilters, + variableValues: McpVariableValues | undefined, + savedFilterValues: ExternalDashboard['savedFilterValues'] = [], +): { tile: T } | { error: string } { + if (!('config' in tile) || tile.config == null) return { tile }; + const referenceVariableNames = getReferenceVariableNames(filters); + const selections = new Map(); + for (const value of savedFilterValues ?? []) { + if (value.type === 'variable') selections.set(value.name, value.values); + } + for (const { name, values } of variableValues ?? []) { + if (referenceVariableNames.has(name)) selections.set(name, values); + } + + let config: object = tile.config; + for (const { key } of getConfigReferenceFields(config)) { + const field = toExternalReferenceKey(key); + const name = parseVariableReference(getStringField(config, field)); + if (name == null) continue; + const selected = selections.get(name) ?? []; + if (selected.length !== 1) { + return { + error: + `Tile "${tile.id}" reads its ${field} from the $${name} variable, which ` + + (selected.length === 0 + ? 'has no selection. Pass one ID for it in variableValues, or save a selection on the dashboard.' + : 'accepts exactly one value.'), + }; + } + config = { ...config, [field]: selected[0] }; + } + return { tile: { ...tile, config } }; +} + /** Options of each static filter, keyed by the variable name it answers to. */ function buildStaticOptionsByName( filters: ExternalDashboardFilter[] | undefined, diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 25d3f0b9c7..309b18bbe3 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8439,6 +8439,102 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('reference variables', () => { + const connectionFilter = { + type: 'CONNECTION' as const, + name: 'Connection', + variableName: 'conn', + }; + const dataSourceFilter = { + type: 'DATA_SOURCE' as const, + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }; + const rawSqlTile = { + name: 'Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }; + + it('round-trips the variables, their values and the tiles that use them', async () => { + const savedFilterValues = [ + { type: 'variable', name: 'conn', values: [connection._id.toString()] }, + { + type: 'variable', + name: 'traces', + values: [traceSource._id.toString()], + }, + ]; + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' }), rawSqlTile], + tags: [], + filters: [connectionFilter, dataSourceFilter], + savedFilterValues, + }) + .expect(200); + + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + expect(fetched.body.data.filters).toEqual([ + expect.objectContaining(connectionFilter), + expect.objectContaining(dataSourceFilter), + ]); + expect(fetched.body.data.tiles.map(t => t.config)).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + expect(fetched.body.data.savedFilterValues).toEqual(savedFilterValues); + + const echoed = await authRequest( + 'put', + `${BASE_URL}/${created.body.data.id}`, + ) + .send(fetched.body.data) + .expect(200); + expect(echoed.body.data.filters).toEqual(fetched.body.data.filters); + expect(echoed.body.data.tiles.map(t => t.config)).toEqual( + fetched.body.data.tiles.map(t => t.config), + ); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it('rejects a connection reference to an undeclared connection variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [rawSqlTile], + tags: [], + filters: [{ ...dataSourceFilter, connectionId: undefined }], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a connection variable/); + }); + }); + describe('ad hoc filters', () => { const adhocFilter = () => ({ id: new ObjectId().toString(), diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 77d3b31361..fa3ad5b2f2 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -655,7 +655,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -742,7 +742,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -812,7 +812,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -888,7 +888,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "table" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -962,7 +962,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "number" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1021,7 +1021,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "pie" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1074,7 +1074,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1167,7 +1167,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "distribution" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1215,7 +1215,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "series" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1264,7 +1264,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "search" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1295,7 +1295,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "event_patterns" * sourceId: * type: string - * description: ID of the data source to mine patterns from. + * description: ID of the data source to mine patterns from, or `$name` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1345,7 +1345,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "sql" * connectionId: * type: string - * description: ID of the ClickHouse connection to execute the query against. + * description: | + * ID of the ClickHouse connection to execute the query against, or + * `$name` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" * sqlTemplate: * type: string @@ -1354,7 +1356,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR" * sourceId: * type: string - * description: Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters. + * description: Optional ID of the data source associated with this Raw SQL chart, or `$name` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters. * example: "65f5e4a3b9e77c001a567890" * numberFormat: * $ref: '#/components/schemas/NumberFormat' @@ -2368,12 +2370,90 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/QueryExpressionFilterInput' * - $ref: '#/components/schemas/StaticListFilterInput' * - $ref: '#/components/schemas/PrometheusLabelFilterInput' + * - $ref: '#/components/schemas/DataSourceFilterInput' + * - $ref: '#/components/schemas/ConnectionFilterInput' * discriminator: * propertyName: type * mapping: * QUERY_EXPRESSION: '#/components/schemas/QueryExpressionFilterInput' * STATIC_LIST: '#/components/schemas/StaticListFilterInput' * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' + * DATA_SOURCE: '#/components/schemas/DataSourceFilterInput' + * CONNECTION: '#/components/schemas/ConnectionFilterInput' + * + * DataSourceFilterInput: + * type: object + * description: | + * A variable whose single, required value is a source. Tiles reference it + * as `$variableName` in their sourceId, and other filters in sourceId or + * appliesToSourceIds, so viewers can switch the source those read. Tiles + * that use it do not load until a source is selected, and cannot have + * alerts. + * required: + * - type + * - name + * - allowedSourceKinds + * properties: + * type: + * type: string + * enum: [DATA_SOURCE] + * description: Filter type discriminator. Must be "DATA_SOURCE". + * example: "DATA_SOURCE" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Logs source" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token tiles and filters reference the selected source by, as `$variableName`. + * example: "logs" + * allowedSourceKinds: + * type: array + * minItems: 1 + * items: + * type: string + * enum: [log, trace, session, metric, promql] + * description: | + * Kinds of sources offered. A field can only reference the variable + * if it accepts every listed kind. + * example: ["log"] + * connectionId: + * type: string + * description: | + * Optional connection the offered sources are limited to: a + * connection ID, or `$name` of a CONNECTION filter on the dashboard. + * example: "$conn" + * + * ConnectionFilterInput: + * type: object + * description: | + * A variable whose single, required value is a connection. Raw SQL tiles + * reference it as `$variableName` in their connectionId, and data source + * variables in theirs. Tiles that use it do not load until a connection is + * selected, and cannot have alerts. + * required: + * - type + * - name + * properties: + * type: + * type: string + * enum: [CONNECTION] + * description: Filter type discriminator. Must be "CONNECTION". + * example: "CONNECTION" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Connection" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token raw SQL tiles reference the selected connection by, as `$variableName`. + * example: "conn" * * QueryExpressionFilterInput: * type: object @@ -2404,7 +2484,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "environment" * sourceId: * type: string - * description: Source ID this dashboard filter key applies to + * description: Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard * example: "65f5e4a3b9e77c001a111111" * sourceMetricType: * type: string @@ -3603,6 +3683,7 @@ router.put( tiles, filters, existingTiles: existingDashboard.tiles ?? [], + storedFilters: existingDashboard.filters, containers: effectiveContainers, }); if (validationError) { diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 69e143e950..8cc55c8f17 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -6,6 +6,7 @@ import { TIME_SERIES_DISPLAY_TYPE_BY_NAME, } from '@hyperdx/common-utils/dist/core/utils'; import { + collectRefinementIssues, validateDashboardContainersStructure, validateDashboardFilterFieldGating, validateDashboardFilterModes, @@ -13,16 +14,14 @@ import { validateDashboardFilterVariableNames, validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; -import { - isPrometheusLabelFilter, - isStaticListFilter, -} from '@hyperdx/common-utils/dist/filters'; +import { isPrometheusLabelFilter } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, isHeatmapCompatibleSource, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { AggregateFunctionSchema, BuilderSavedChartConfig, @@ -30,6 +29,7 @@ import { DASHBOARD_MAX_CONTAINERS, DashboardContainer, DashboardContainerSchema, + DashboardFilter, DisplayType, HeatmapMode, isLogSource, @@ -54,6 +54,7 @@ import { deleteDashboardAlerts } from '@/controllers/alerts'; import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; +import { getVariableReferenceError } from '@/utils/dashboardVariableReferences'; import { getExternalCompatibleFilters, getExternalCompatibleFilterValues, @@ -1307,19 +1308,21 @@ function getMissingSources( } } - if (filters?.length) { - for (const filter of filters) { - // Every variant but the static one names a source. - if (!isStaticListFilter(filter)) { - sourceIds.add(filter.sourceId); - } + for (const filter of filters ?? []) { + if ('sourceId' in filter) { + sourceIds.add(filter.sourceId); } } const existingSourceIds = new Set( sources.map(source => source._id.toString()), ); - return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); + // Variable references are checked by validateDashboardVariableReferences + return [...sourceIds].filter( + sourceId => + parseVariableReference(sourceId) == null && + !existingSourceIds.has(sourceId), + ); } /** @@ -1589,18 +1592,26 @@ async function getMissingOnClickDashboards( return [...dashboardIds].filter(id => !existingDashboardIds.has(id)); } -/** Returns connection IDs referenced in tiles that do not belong to the team */ +/** Returns connection IDs referenced in tiles and filters that do not belong to the team */ async function getMissingConnections( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] = [], ): Promise { const connectionIds = new Set(); + // Variable references are checked by validateDashboardVariableReferences + const addConnectionId = (id: string | undefined) => { + if (id && parseVariableReference(id) == null) connectionIds.add(id); + }; for (const tile of tiles) { if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { - connectionIds.add(tile.config.connectionId); + addConnectionId(tile.config.connectionId); } } + for (const filter of filters) { + if ('connectionId' in filter) addConnectionId(filter.connectionId); + } if (connectionIds.size === 0) return []; @@ -1731,6 +1742,11 @@ export type TileValidationContext = { filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; /** Existing internal tiles for scoping heatmap change detection (update paths). */ existingTiles?: DashboardDocument['tiles']; + /** + * Stored internal filters, which tile variable references are checked + * against when the request carries no `filters` (partial update paths). + */ + storedFilters?: DashboardFilter[]; /** Container set to validate tile containerId/tabId refs against. */ containers: DashboardContainer[]; }; @@ -1747,7 +1763,8 @@ export type TileValidationContext = { export async function validateDashboardTiles( ctx: TileValidationContext, ): Promise { - const { teamId, tiles, filters, existingTiles, containers } = ctx; + const { teamId, tiles, filters, existingTiles, storedFilters, containers } = + ctx; // Container/tab ref resolution. const tileRefIssues = collectTileContainerRefIssues(containers, tiles); @@ -1755,6 +1772,14 @@ export async function validateDashboardTiles( return tileRefIssues.join('; '); } + const variableReferenceError = getVariableReferenceError( + convertExternalTilesToInternal(tiles), + filters ? convertExternalFiltersToInternal(filters) : storedFilters, + ); + if (variableReferenceError) { + return variableReferenceError; + } + // Fetch sources/connections/onClick targets in parallel. const [ sources, @@ -1763,7 +1788,7 @@ export async function validateDashboardTiles( invalidOnClickSearchSources, ] = await Promise.all([ fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tiles), + getMissingConnections(teamId, tiles, filters), getMissingOnClickDashboards(teamId, tiles), getInvalidOnClickSearchSources(teamId, tiles), ]); @@ -1942,15 +1967,13 @@ export function collectTileContainerRefIssues( containers: DashboardContainer[], tiles: ExternalDashboardTileWithId[], ): string[] { - const schema = z.object({}).superRefine((_, ctx) => { - const containerById = new Map( - containers.map(c => [c.id, c]), - ); - validateDashboardTileContainerRefs(containerById, tiles, ctx); - }); - const result = schema.safeParse({}); - if (result.success) return []; - return result.error.issues.map(issue => { + const containerById = new Map( + containers.map(c => [c.id, c]), + ); + const issues = collectRefinementIssues(ctx => + validateDashboardTileContainerRefs(containerById, tiles, ctx), + ); + return issues.map(issue => { const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : ''; return `${path}${issue.message}`; }); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index cb63a42da8..3d61320540 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -5,7 +5,6 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, - isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -242,10 +241,7 @@ export function translateExternalChartToTileConfig( } /** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract< - DashboardFilter, - { type: 'ADHOC' | 'DATA_SOURCE' | 'CONNECTION' } ->; +type InternalOnlyDashboardFilter = Extract; export type ExternalCompatibleDashboardFilter = Exclude< DashboardFilter, InternalOnlyDashboardFilter @@ -257,7 +253,7 @@ export function getExternalCompatibleFilters( ): ExternalCompatibleDashboardFilter[] { return (filters ?? []).filter( (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter) && !isReferenceVariableFilter(filter), + !isAdhocFilter(filter), ); } @@ -287,6 +283,15 @@ export function translateFilterToExternalFilter( sourceId: filter.source.toString(), }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connection'), + ...(filter.connection ? { connectionId: filter.connection } : {}), + }; + + case 'CONNECTION': + return filter; + case 'QUERY_EXPRESSION': { // Ignore variableName and appliesToSourceIds if the filter is not in a mode that uses them const ignoredKeys = [ @@ -321,6 +326,15 @@ export function translateExternalFilterToFilter( case 'QUERY_EXPRESSION': return { ...omit(filter, 'sourceId'), source: filter.sourceId }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connectionId'), + ...(filter.connectionId ? { connection: filter.connectionId } : {}), + }; + + case 'CONNECTION': + return filter; + default: filter satisfies never; return filter; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d14d34849d..f0b5fac436 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,4 +1,5 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { addDuplicateTileIdIssues, AggregateFunctionSchema, @@ -9,8 +10,10 @@ import { AlertThresholdType, BackgroundChartSchema, ChartPaletteTokenSchema, + ConnectionDashboardFilterSchema, DASHBOARD_CONTAINER_ID_MAX, DASHBOARD_MAX_TILES, + DataSourceDashboardFilterSchema, HeatmapScaleTypeSchema, LineInterpolationSchema, MAX_LEGEND_TEMPLATE_LENGTH, @@ -50,6 +53,17 @@ export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); }, 'Invalid ObjectId'); +/** + * A source or connection field: an ObjectId, or a `$name` reference to the + * dashboard's data source or connection variable of that name. + */ +export const objectIdOrVariableReferenceSchema = z + .string() + .refine( + val => Types.ObjectId.isValid(val) || parseVariableReference(val) != null, + 'Invalid ObjectId or $variable reference', + ); + /** * A repeatable query param. Express's parser yields a bare string for a single * occurrence and an array for several, and `?p[]=x` lands on the same key as @@ -171,7 +185,7 @@ export { tagsSchema }; const externalQueryExpressionFilterShape = QueryExpressionDashboardFilterSchema.omit({ source: true }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, }); // A static filter's mode flags each accept exactly one value, so callers may @@ -184,17 +198,39 @@ const externalStaticListFilterShape = StaticListDashboardFilterSchema.extend({ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ source: true, }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); +// Reference variables are always single-select and required, so these +// fields, like the mode flags, accept one value each and may be omitted. +const referenceVariableFixedFields = { + minSelections: z.literal(1).default(1), + maxSelections: z.literal(1).default(1), + isBroadcastEnabled: z.literal(false).default(false), + isVariableEnabled: z.literal(true).default(true), +}; + +const externalDataSourceFilterShape = DataSourceDashboardFilterSchema.omit({ + connection: true, +}).extend({ + connectionId: objectIdOrVariableReferenceSchema.optional(), + ...referenceVariableFixedFields, +}); + +const externalConnectionFilterShape = ConnectionDashboardFilterSchema.extend( + referenceVariableFixedFields, +); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ externalQueryExpressionFilterShape.strict(), externalStaticListFilterShape.strict(), externalPromqlLabelFilterShape.strict(), + externalDataSourceFilterShape.strict(), + externalConnectionFilterShape.strict(), ], ); @@ -211,6 +247,8 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalQueryExpressionFilterShape.omit({ id: true }).strict(), externalStaticListFilterShape.omit({ id: true }).strict(), externalPromqlLabelFilterShape.omit({ id: true }).strict(), + externalDataSourceFilterShape.omit({ id: true }).strict(), + externalConnectionFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -326,9 +364,9 @@ export type ExternalDashboardSelectItem = z.infer< const externalDashboardRawSqlChartConfigBaseSchema = z.object({ configType: z.literal('sql'), - connectionId: objectIdSchema, + connectionId: objectIdOrVariableReferenceSchema, sqlTemplate: z.string().max(100000), - sourceId: objectIdSchema.optional(), + sourceId: objectIdOrVariableReferenceSchema.optional(), numberFormat: NumberFormatSchema.optional(), }); @@ -351,7 +389,7 @@ const externalFormulasSchema = z.array(MetricFormulaSchema).max(10).optional(); const externalShowOperandSeriesSchema = z.boolean().optional(); const externalDashboardTimeChartConfigSchema = z.object({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), asRatio: z.boolean().optional(), @@ -422,7 +460,7 @@ const externalDashboardStackedLineRawSqlChartConfigSchema = const externalDashboardTableChartConfigSchema = z.object({ displayType: z.literal('table'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), having: z.string().max(10000).optional(), @@ -462,7 +500,7 @@ const externalDashboardCategoricalBarRawSqlChartConfigSchema = const externalDashboardNumberChartConfigSchema = z.object({ displayType: z.literal('number'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Exactly one select item unless `formulas` is set: a formula number tile // displays the (single) formula value and its select entries are the // formula's operands (e.g. A / (A + B)). The one-item rule for @@ -502,7 +540,7 @@ const externalDashboardNumberChartConfigSchema = z.object({ const externalDashboardPieChartConfigSchema = z.object({ displayType: z.literal('pie'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -513,7 +551,7 @@ const externalDashboardPieChartConfigSchema = z.object({ const externalDashboardCategoricalBarChartConfigSchema = z.object({ displayType: z.literal('bar'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -550,7 +588,7 @@ export type ExternalDashboardHeatmapSelectItem = z.infer< const externalDashboardHeatmapChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('distribution').optional(), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardHeatmapSelectItemSchema).length(1), where: z.string().max(10000).optional().default(''), // `whereLanguageSchema` (an alias for `SearchConditionLanguageSchema`) @@ -564,7 +602,7 @@ const externalDashboardHeatmapChartConfigSchema = z.object({ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('series'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), numberFormat: NumberFormatSchema.optional(), @@ -572,7 +610,7 @@ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ const externalDashboardSearchChartConfigSchema = z.object({ displayType: z.literal('search'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -596,7 +634,7 @@ const externalDashboardMarkdownChartConfigSchema = z.object({ const externalDashboardEventPatternsChartConfigSchema = z.object({ displayType: z.literal('event_patterns'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000).optional().default(''), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -690,7 +728,7 @@ const externalPromqlLegendTemplateSchema = z const externalPromqlChartConfigBaseSchema = z.object({ configType: z.literal('promql'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Sets the Prometheus step and `$__interval` of range queries. granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), numberFormat: NumberFormatSchema.optional(),