diff --git a/.changeset/sql-query-dashboard-filter-source.md b/.changeset/sql-query-dashboard-filter-source.md new file mode 100644 index 000000000..5eef88e79 --- /dev/null +++ b/.changeset/sql-query-dashboard-filter-source.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +feat: let SQL query dashboard filters use an optional source + +A SQL query filter can now pick a source on its connection. With one, its SQL can use `$__sourceTable` and `$__filters`, which applies the dashboard's filter selections for that source, the same way raw SQL tiles do. The external API and MCP read and write it as `sourceId`. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 604e11c63..f092b53af 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4295,10 +4295,15 @@ "description": "ID of the ClickHouse connection the query runs on.", "example": "65f5e4a3b9e77c001a567890" }, + "sourceId": { + "type": "string", + "description": "Optional Log, Trace, Session, or Metric source on the same\nconnection. Lets `sqlTemplate` use `$__sourceTable` and\n`$__filters`.\n", + "example": "65f5e4a3b9e77c001a345678" + }, "sqlTemplate": { "type": "string", "minLength": 1, - "description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables, but not `$__sourceTable` or\n`$__filters`, since the filter has no source.\n", + "description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables. `$__sourceTable` and\n`$__filters` need a `sourceId`; `$__filters` expands to the\ndashboard's broadcast filter selections for that source.\n", "example": "SELECT DISTINCT env FROM default.envs ORDER BY env" }, "isBroadcastEnabled": { diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index bd6ffe6a6..cec4de9c4 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -936,9 +936,10 @@ describe('getFilterVariableWarnings', () => { }); describe('SQL query filters', () => { - const sqlFilter = (sqlTemplate: string) => ({ + const sqlFilter = (sqlTemplate: string, sourceId?: string) => ({ name: 'Environment', sqlTemplate, + sourceId, isVariableEnabled: true, variableName: 'env', }); @@ -968,10 +969,21 @@ describe('getFilterVariableWarnings', () => { sqlFilter('SELECT env FROM $__sourceTable'), ]); expect(warnings).toEqual([ - 'Filter "Environment": SQL query filters have no source, so $__sourceTable cannot be used', + 'Filter "Environment": SQL uses $__sourceTable but no source is selected', ]); }); + it('accepts source macros when the filter has a source', () => { + expect( + getFilterVariableWarnings([ + sqlFilter( + 'SELECT env FROM $__sourceTable WHERE $__filters', + '65f5e4a3b9e77c001a345678', + ), + ]), + ).toEqual([]); + }); + it.each([ ['a source macro', 'SELECT env FROM $__sourceTable(gauge'], ['a time macro', 'SELECT env FROM envs WHERE $__timeFilter(ts'], diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 74e2d4e03..982d5a1e4 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -196,7 +196,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Using $__filter or $__conditionalAll in a Lucene where (they are matched as literal text; switch to whereLanguage: "sql", or write ServiceName:$var). - Chaining a filter's where off a filter that is not variable-enabled. Only isVariableEnabled: true publishes the $variableName token; without it the reference resolves to nothing and the downstream dropdown is unscoped or empty. - Referencing a filter's OWN variable in its where. That narrows its dropdown to the values already picked, so the rest of the options vanish from a multi-select. Reference a sibling filter's variable instead. -- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate to SQL_QUERY filters. +- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate (and an optional sourceId) to SQL_QUERY filters. - Expecting a STATIC_LIST filter to broadcast. It cannot; a tile must reference its $variableName (via $__filter with the column passed explicitly) or nothing changes when the user picks a value.`; } @@ -1208,10 +1208,11 @@ Example (static list): ] SQL query filter shape: - { type: "SQL_QUERY", name, connectionId, sqlTemplate, variableName? } + { type: "SQL_QUERY", name, connectionId, sourceId?, sqlTemplate, variableName? } connectionId ClickHouse connection the query runs on (from clickstack_list_sources). - sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables; cannot use $__sourceTable or $__filters, because the filter has no source. + sourceId Optional log, trace, session, or metric source on that connection. Needed for $__sourceTable and $__filters; $__filters expands to the dashboard's broadcast filter selections for that source. + sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables, plus $__sourceTable and $__filters when sourceId is set. Use it when the values live in a lookup table, dictionary, or join that no single source column covers; otherwise prefer QUERY_EXPRESSION. Like a static filter it is variable-only, so tiles pass the column explicitly: $__filter(, $). @@ -1220,6 +1221,11 @@ Example (SQL query, narrowed by another filter's variable): { type: "SQL_QUERY", name: "Environment", connectionId: "", sqlTemplate: "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env", variableName: "env" } ] +Example (SQL query on a source, narrowed by the dashboard's broadcast filters): + filters: [ + { type: "SQL_QUERY", name: "Host", connectionId: "", sourceId: "", sqlTemplate: "SELECT DISTINCT ResourceAttributes['host.name'] FROM $__sourceTable WHERE $__timeFilter(TimestampTime) AND $__filters ORDER BY 1", variableName: "host" } + ] + Example (broadcast to every tile, the common case): filters: [ { type: "QUERY_EXPRESSION", name: "Service", expression: "ServiceName", sourceId: "" } diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index a195efa15..3fe42905b 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1777,13 +1777,21 @@ const mcpSqlQueryFilterSchema = z connectionId: objectIdSchema.describe( 'ClickHouse connection the query runs on. Get IDs from clickstack_list_sources.', ), + sourceId: objectIdSchema + .optional() + .describe( + 'Optional log, trace, session, or metric source on the same connection. ' + + 'Lets sqlTemplate use $__sourceTable and $__filters. Get IDs from ' + + 'clickstack_list_sources.', + ), sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + 'string; nulls and duplicates dropped; row order kept, so add ORDER BY). ' + 'May use the time macros ($__timeFilter(), $__fromTime, $__toTime) ' + "and other filters' variables, ideally as $__filter(, $). " + - 'Cannot use $__sourceTable or $__filters: the filter has no source, so name ' + - 'tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".', + '$__sourceTable and $__filters need a sourceId; $__filters expands to the ' + + "dashboard's broadcast filter selections for that source. Without a sourceId, " + + 'name tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".', ), }) .describe( diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 72f844579..a46eab0d5 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -162,7 +162,9 @@ export function getRawSqlTileMacroWarnings( */ export type FilterForVariableValidation = FilterForVariableDeclaration & Partial> & - Partial>; + Partial> & { + sourceId?: string; + }; /** * Non-blocking checks on the dashboard variables references in a filter's `where`. @@ -221,7 +223,11 @@ export function getFilterVariableWarnings( if (sqlTemplate) { // Also reports variable macro errors, which `result` already holds. - const detail = validateSqlQueryFilterTemplate(sqlTemplate, variables); + // Source tables aren't loaded here, so a source only allows its macros. + const detail = validateSqlQueryFilterTemplate(sqlTemplate, { + variables, + sourceTables: filter.sourceId ? 'unknown' : undefined, + }); if (detail && !issues.includes(detail)) issues.push(detail); } diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index e92d9bed1..3463d6200 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1966,6 +1966,98 @@ describe('dashboard router', () => { expect(updated.body.filters).toEqual([moved]); }); + + const createSqlFilterSource = (kind: SourceKind, teamId = team._id) => + Source.create({ + kind, + name: `Test ${kind} source`, + team: teamId, + connection: connectionId, + from: { databaseName: 'test_db', tableName: `${kind}_table` }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'body', + }); + + it('persists a SQL query filter with a source', async () => { + const source = await createSqlFilterSource(SourceKind.Log); + const filter = makeSqlFilter({ + source: source._id.toString(), + sqlTemplate: + 'SELECT DISTINCT ServiceName FROM $__sourceTable WHERE $__filters', + }); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + expect(created.body.filters).toEqual([filter]); + }); + + it('rejects a source that does not exist', async () => { + const source = new Types.ObjectId().toString(); + + const response = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [makeSqlFilter({ source })] }) + .expect(400); + + expect(response.body.message).toContain(source); + }); + + it("rejects another team's source", async () => { + const source = await createSqlFilterSource( + SourceKind.Log, + new Types.ObjectId(), + ); + + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ source: source._id.toString() })], + }) + .expect(400); + }); + + it('rejects a source on another connection', async () => { + const source = await createSqlFilterSource(SourceKind.Log); + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: 'http://localhost:8123', + username: 'default', + }); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeSqlFilter({ + source: source._id.toString(), + connection: otherConnection._id.toString(), + }), + ], + }) + .expect(400); + + expect(response.body.message).toContain("not on the filter's connection"); + }); + + it('rejects a PromQL source', async () => { + const source = await createSqlFilterSource(SourceKind.Promql); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ source: source._id.toString() })], + }) + .expect(400); + + expect(response.body.message).toContain(source._id.toString()); + }); }); describe('ad hoc filters', () => { diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 0fb5d54f8..c54496a38 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -8,6 +8,7 @@ import { isAdhocFilter, isPrometheusLabelFilter, isSqlQueryFilter, + SQL_QUERY_FILTER_SOURCE_KINDS, } from '@hyperdx/common-utils/dist/filters'; import { DashboardFilter, @@ -66,10 +67,15 @@ const addFilterIssues = ( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); }; +type FilterSourceRef = { + id: string; + kinds: SourceKind[]; + /** The connection the source must be on, when the filter has its own. */ + connection?: string; +}; + /** The sources a filter reads from, each with the kinds it must be. */ -function getFilterSourceRefs( - filter: DashboardFilter, -): { id: string; kinds: SourceKind[] }[] { +function getFilterSourceRefs(filter: DashboardFilter): FilterSourceRef[] { if (isPrometheusLabelFilter(filter)) { return [{ id: filter.source, kinds: [SourceKind.Promql] }]; } @@ -79,13 +85,23 @@ function getFilterSourceRefs( id => ({ id, kinds }), ); } + if (isSqlQueryFilter(filter) && filter.source) { + return [ + { + id: filter.source, + kinds: SQL_QUERY_FILTER_SOURCE_KINDS, + connection: filter.connection, + }, + ]; + } return []; } /** - * Rejects PROMETHEUS_LABEL and ADHOC filters that reference a source that is - * missing or of the wrong kind. Resolving these filters needs a live source of - * the right kind, unlike other filter types. References the stored filter with + * Rejects PROMETHEUS_LABEL, ADHOC and SQL_QUERY filters that reference a source + * that is missing or of the wrong kind, and SQL_QUERY filters whose source is + * on another connection. Resolving these filters needs a live source of the + * right kind, unlike other filter types. References the stored filter with * the same ID already made are skipped, so a since-deleted source doesn't block * unrelated saves. Returns an error message, or null when there is nothing to * reject. @@ -95,8 +111,8 @@ async function validateFilterSources( filters: DashboardFilter[] = [], existingFilters: DashboardFilter[] = [], ): Promise { - const refKey = ({ id, kinds }: { id: string; kinds: SourceKind[] }) => - `${id}:${kinds.join(',')}`; + const refKey = ({ id, kinds, connection }: FilterSourceRef) => + `${id}:${kinds.join(',')}:${connection ?? ''}`; const existingRefsById = new Map( existingFilters.map(filter => [ filter.id, @@ -111,17 +127,31 @@ async function validateFilterSources( if (newRefs.length === 0) return null; const sources = await getSources(teamId.toString()); - const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const sourceById = new Map(sources.map(s => [s._id.toString(), s])); const invalid = new Set( newRefs .filter(({ id, kinds }) => { - const kind = kindById.get(id); + const kind = sourceById.get(id)?.kind; return kind === undefined || !kinds.includes(kind); }) .map(({ id }) => id), ); - if (invalid.size === 0) return null; - return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${[...invalid].join(', ')}`; + if (invalid.size > 0) { + return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${[...invalid].join(', ')}`; + } + + // The query runs on the filter's connection, so its source must be there too. + const mismatched = new Set( + newRefs + .filter( + ({ id, connection }) => + connection !== undefined && + sourceById.get(id)?.connection.toString() !== connection, + ) + .map(({ id }) => id), + ); + if (mismatched.size === 0) return null; + return `The following filter source IDs are not on the filter's connection: ${[...mismatched].join(', ')}`; } /** diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 8772c5a2c..64ca54441 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6333,10 +6333,68 @@ describe('External API v2 Dashboards - new format', () => { ['broadcast enabled', { isBroadcastEnabled: true }], ['not variable-enabled', { isVariableEnabled: false }], ['no SQL', { sqlTemplate: '' }], - ['a source', { sourceId: new ObjectId().toString() }], + [ + 'a source that does not exist', + { sourceId: new ObjectId().toString() }, + ], ])('rejects a SQL query filter with %s', async (_label, overrides) => { await expectFilters([sqlFilterInput(overrides)], 400); }); + + it('accepts a SQL query filter with a source and round-trips it through GET', async () => { + const sqlTemplate = + 'SELECT DISTINCT ServiceName FROM $__sourceTable WHERE $__filters'; + const response = await sendFilters([ + sqlFilterInput({ sourceId: traceSource._id.toString(), sqlTemplate }), + ]); + expect(response.status).toBe(200); + + const [filter] = response.body.data.filters; + expect(filter).toMatchObject({ + type: 'SQL_QUERY', + connectionId: connection._id.toString(), + sourceId: traceSource._id.toString(), + sqlTemplate, + }); + expect(filter).not.toHaveProperty('source'); + + const getResponse = await authRequest( + 'get', + `${BASE_URL}/${response.body.data.id}`, + ).expect(200); + expect(getResponse.body.data.filters).toEqual( + response.body.data.filters, + ); + }); + + it('rejects a PromQL source', async () => { + const response = await sendFilters([ + sqlFilterInput({ sourceId: promqlSource._id.toString() }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain(promqlSource._id.toString()); + }); + + it('rejects a source on a different connection', async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + const response = await sendFilters([ + sqlFilterInput({ + connectionId: otherConnection._id.toString(), + sourceId: traceSource._id.toString(), + }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain(traceSource._id.toString()); + }); }); describe('static-list filters', () => { diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 3f47a774d..8b23888eb 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2712,13 +2712,21 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * description: ID of the ClickHouse connection the query runs on. * example: "65f5e4a3b9e77c001a567890" + * sourceId: + * type: string + * description: | + * Optional Log, Trace, Session, or Metric source on the same + * connection. Lets `sqlTemplate` use `$__sourceTable` and + * `$__filters`. + * example: "65f5e4a3b9e77c001a345678" * sqlTemplate: * type: string * minLength: 1 * description: | * Query whose first column populates the dropdown. May use the time - * macros and other filters' variables, but not `$__sourceTable` or - * `$__filters`, since the filter has no source. + * macros and other filters' variables. `$__sourceTable` and + * `$__filters` need a `sourceId`; `$__filters` expands to the + * dashboard's broadcast filter selections for that source. * example: "SELECT DISTINCT env FROM default.envs ORDER BY env" * isBroadcastEnabled: * type: boolean diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 27af9366e..fc8705a35 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -13,7 +13,10 @@ import { validateDashboardFilterVariableNames, validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; -import { isPrometheusLabelFilter } from '@hyperdx/common-utils/dist/filters'; +import { + isPrometheusLabelFilter, + SQL_QUERY_FILTER_SOURCE_KINDS, +} from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, isHeatmapCompatibleSource, @@ -1306,7 +1309,7 @@ function getMissingSources( if (filters?.length) { for (const filter of filters) { - if ('sourceId' in filter) { + if ('sourceId' in filter && filter.sourceId) { sourceIds.add(filter.sourceId); } } @@ -1350,6 +1353,28 @@ function getHeatmapTilesWithIncompatibleSources( return { distribution: [...distribution], series: [...series] }; } +/** + * Returns an error message if a SQL_QUERY filter's source is a kind its + * $__sourceTable macro can't expand (PromQL), or null. + */ +function getSqlQueryFilterSourceKindError( + sources: SourceForValidation[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], +): string | null { + const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const invalid = new Set( + filters.flatMap(filter => { + if (filter.type !== 'SQL_QUERY' || !filter.sourceId) return []; + const kind = kindById.get(filter.sourceId); + return kind === undefined || SQL_QUERY_FILTER_SOURCE_KINDS.includes(kind) + ? [] + : [filter.sourceId]; + }), + ); + if (invalid.size === 0) return null; + return `SQL_QUERY filters require a Log, Trace, Session, or Metric source. The following source IDs are not: ${[...invalid].join(', ')}`; +} + /** * Returns an error message string if any of the referenced source IDs is not * a valid PromQL source, or null if all of them are. `subject` names what @@ -1704,24 +1729,38 @@ export function convertExternalFiltersToInternal( function getSourceConnectionMismatches( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] = [], ): string[] { const sourceById = new Map(sources.map(s => [s._id.toString(), s])); - const mismatched: string[] = []; + const refs: { sourceId: string; connectionId: string }[] = []; for (const tile of tiles) { if ( isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config) && tile.config.sourceId ) { - const source = sourceById.get(tile.config.sourceId); - if (source && source.connection.toString() !== tile.config.connectionId) { - mismatched.push(tile.config.sourceId); - } + refs.push({ + sourceId: tile.config.sourceId, + connectionId: tile.config.connectionId, + }); + } + } + for (const filter of filters) { + if (filter.type === 'SQL_QUERY' && filter.sourceId) { + refs.push({ + sourceId: filter.sourceId, + connectionId: filter.connectionId, + }); } } - return mismatched; + return refs + .filter(({ sourceId, connectionId }) => { + const source = sourceById.get(sourceId); + return source && source.connection.toString() !== connectionId; + }) + .map(({ sourceId }) => sourceId); } // ── Shared tile validation ─────────────────────────────────────────────── @@ -1781,6 +1820,7 @@ export async function validateDashboardTiles( const sourceConnectionMismatches = getSourceConnectionMismatches( sources, tiles, + filters, ); if (sourceConnectionMismatches.length > 0) { return `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(', ')}`; @@ -1834,6 +1874,12 @@ export async function validateDashboardTiles( ); if (promqlLabelFilterError != null) return promqlLabelFilterError; + const sqlQueryFilterError = getSqlQueryFilterSourceKindError( + sources, + filters ?? [], + ); + if (sqlQueryFilterError != null) return sqlQueryFilterError; + if (missingOnClickDashboards.length > 0) { return `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`; } diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 4422bcf90..08acff7f0 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -285,8 +285,9 @@ export function translateFilterToExternalFilter( case 'SQL_QUERY': return { - ...omit(filter, 'connection', ...omittedKeys), + ...omit(filter, 'connection', 'source', ...omittedKeys), connectionId: filter.connection.toString(), + ...(filter.source ? { sourceId: filter.source.toString() } : {}), }; case 'QUERY_EXPRESSION': { @@ -322,8 +323,9 @@ export function translateExternalFilterToFilter( case 'SQL_QUERY': return { - ...omit(filter, 'connectionId'), + ...omit(filter, 'connectionId', 'sourceId'), connection: filter.connectionId, + ...(filter.sourceId ? { source: filter.sourceId } : {}), }; case 'QUERY_EXPRESSION': diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 2542a9d4a..6be031c55 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -190,11 +190,14 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); -// `connectionId` rather than `connection`, matching raw SQL tiles. +// `connectionId`/`sourceId` rather than `connection`/`source`, matching raw SQL +// tiles. const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ connection: true, + source: true, }).extend({ connectionId: objectIdSchema, + sourceId: objectIdSchema.optional(), isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 21c5936fd..a087a8baa 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -26,6 +26,7 @@ import { QUERY_EXPRESSION_FILTER_SOURCE_KINDS, type SavedFilterValueIssue, type SavedQueryIssue, + SQL_QUERY_FILTER_SOURCE_KINDS, validateDashboardFilterQueries, validateSavedFilterValues, validateSavedQuery, @@ -46,6 +47,7 @@ import { type QueryExpressionDashboardFilter, SavedChartConfig, SourceKind, + type SqlQueryDashboardFilter, TSource, } from '@hyperdx/common-utils/dist/types'; import { @@ -401,8 +403,14 @@ function resolveAppliesToSources( /** Whether the filter names a single source that import must map. */ const isSourceBackedTemplateFilter = ( filter: DashboardFilter, -): filter is QueryExpressionDashboardFilter | PromqlLabelDashboardFilter => - isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter); +): filter is + | QueryExpressionDashboardFilter + | PromqlLabelDashboardFilter + | SqlQueryDashboardFilter => + isQueryExpressionFilter(filter) || + isPrometheusLabelFilter(filter) || + // The source is optional for a SQL query filter + (isSqlQueryFilter(filter) && !!filter.source); /** Returns the filter if it names a source, or `undefined` if it doesn't. */ const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) => @@ -436,7 +444,9 @@ const filterSourceKinds = ( ? [SourceKind.Promql] : isQueryExpressionFilter(filter) ? QUERY_EXPRESSION_FILTER_SOURCE_KINDS - : undefined; + : isSqlQueryFilter(filter) + ? SQL_QUERY_FILTER_SOURCE_KINDS + : undefined; /** * One mapping select's value. Mantine's Select writes `null` when the user @@ -454,7 +464,7 @@ const MappingValueSchema = z */ export function buildMappingFormSchema( input: DashboardTemplate, - sources?: readonly Pick[], + sources?: readonly Pick[], ) { return z .object({ @@ -564,7 +574,6 @@ export function buildMappingFormSchema( message: 'Select a connection for this filter', }); } - return; } if (!isSourceBackedTemplateFilter(filter)) return; @@ -592,6 +601,22 @@ export function buildMappingFormSchema( ? 'Select a PromQL source for this filter' : 'Select a non-PromQL source for this filter', }); + return; + } + + // The filter's query runs on its connection, so its source must be there too. + const mappedConnectionId = data.filterConnectionMappings?.[idx]; + if ( + isSqlQueryFilter(filter) && + mappedSource && + mappedConnectionId && + mappedSource.connection !== mappedConnectionId + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['filterSourceMappings', idx], + message: "Select a source on this filter's connection", + }); } }); }); @@ -1134,7 +1159,14 @@ export function Mapping({ input }: { input: DashboardTemplate }) { const connection = findConnection( data.filterConnectionMappings?.[idx], ); - return { ...filter, connection: connection!.id }; + const source = filter.source + ? findSource(data.filterSourceMappings?.[idx]) + : undefined; + return { + ...filter, + connection: connection!.id, + source: source?.id, + }; } const source = findSource(data.filterSourceMappings?.[idx]); return { @@ -1310,6 +1342,10 @@ export function Mapping({ input }: { input: DashboardTemplate }) { {/** Map filter sources */} {input.filters?.map((filter, i) => { + // A SQL query filter's source must be on its mapped connection. + const sourceConnectionId = isSqlQueryFilter(filter) + ? filterConnectionMappings?.[i] + : undefined; const appliesToSourceIds = appliesToTemplateFilter(filter)?.appliesToSourceIds; return ( @@ -1324,8 +1360,13 @@ export function Mapping({ input }: { input: DashboardTemplate }) { control={control} name={`filterSourceMappings.${i}`} data={sources - ?.filter(source => - filterSourceKinds(filter)?.includes(source.kind), + ?.filter( + source => + filterSourceKinds(filter)?.includes( + source.kind, + ) && + (!sourceConnectionId || + source.connection === sourceConnectionId), ) .map(source => ({ value: source.id, diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index c3172241e..27d9a8aef 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -3332,6 +3332,7 @@ function DBDashboardPage({ dateRange={searchedTimeRange} variables={variables} unsatisfiedRequiredFilters={unsatisfiedRequiredFilters} + getDashboardFilters={getFilterQueriesForSource} /> )} {/* Selection indicator */} @@ -3511,6 +3512,7 @@ function DBDashboardPage({ showRequiredFilterOptions variables={variables} dateRange={searchedTimeRange} + getDashboardFilters={getFilterQueriesForSource} /> )} diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index ee1109412..5d3311053 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -14,6 +14,7 @@ import { } from '@hyperdx/common-utils/dist/filters'; import { AdhocFilterCondition, + BroadcastFilter, ChartVariable, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -175,6 +176,8 @@ interface DashboardFilterProps { variables?: ChartVariable[]; /** Required filters with nothing selected, which can block ad hoc filters. */ unsatisfiedRequiredFilters?: DashboardFilter[]; + /** The filter selections that broadcast to a source, for `$__filters` */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; } const DashboardFilters = ({ @@ -186,6 +189,7 @@ const DashboardFilters = ({ onSetAdhocConditions, variables, unsatisfiedRequiredFilters, + getDashboardFilters, }: DashboardFilterProps) => { // "Link" mode (opt-in, off by default): each dropdown's values are narrowed by // the others' selections. Off by default because contingent value lookups @@ -202,6 +206,7 @@ const DashboardFilters = ({ filters, dateRange, variables, + getDashboardFilters, // Only narrow by sibling selections when linked. selectionByFilterId: linked ? selectionByFilterId : EMPTY_SELECTIONS, }); diff --git a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx index 664064e76..4904e0802 100644 --- a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx +++ b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx @@ -11,14 +11,44 @@ import { buildMappingFormSchema, Mapping } from '@/DBDashboardImportPage'; // Sources/connections the import maps the template's source *names* onto. The // auto-mapping effect in `Mapping` matches by name (case-insensitive), so a // template tile with `source: 'Metrics'` resolves to `src-metrics`, etc. The -// mock objects only need the fields the import path touches (id/name/kind). +// mock objects only need the fields the import path touches +// (id/name/kind/connection). const mockSources = [ - { id: 'src-metrics', name: 'Metrics', kind: SourceKind.Metric }, - { id: 'src-traces', name: 'Traces', kind: SourceKind.Trace }, - { id: 'src-logs', name: 'Logs', kind: SourceKind.Log }, - { id: 'src-promql', name: 'Prom', kind: SourceKind.Promql }, + { + id: 'src-metrics', + name: 'Metrics', + kind: SourceKind.Metric, + connection: 'conn-default', + }, + { + id: 'src-traces', + name: 'Traces', + kind: SourceKind.Trace, + connection: 'conn-default', + }, + { + id: 'src-logs', + name: 'Logs', + kind: SourceKind.Log, + connection: 'conn-default', + }, + { + id: 'src-promql', + name: 'Prom', + kind: SourceKind.Promql, + connection: 'conn-default', + }, + { + id: 'src-other-logs', + name: 'Other logs', + kind: SourceKind.Log, + connection: 'conn-other', + }, +]; +const mockConnections = [ + { id: 'conn-default', name: 'Default' }, + { id: 'conn-other', name: 'Other' }, ]; -const mockConnections = [{ id: 'conn-default', name: 'Default' }]; const mockMutateAsync = jest.fn().mockResolvedValue({ id: 'dash-new' }); jest.mock('next/router', () => ({ @@ -265,7 +295,7 @@ const unresolvedConnectionTemplate = DashboardTemplateSchema.parse({ ], }); -const sqlQueryFilterTemplate = (connection: string) => +const sqlQueryFilterTemplate = (connection: string, source?: string) => DashboardTemplateSchema.parse({ version: '0.1.0', name: 'SQL Query Filter', @@ -276,6 +306,7 @@ const sqlQueryFilterTemplate = (connection: string) => type: 'SQL_QUERY', name: 'Environment', connection, + source, sqlTemplate: 'SELECT DISTINCT env FROM envs', isBroadcastEnabled: false, isVariableEnabled: true, @@ -425,6 +456,86 @@ describe('Dashboard import - SQL query filter connection', () => { type: 'SQL_QUERY', connection: 'conn-default', }); + expect(payload.filters[0].source).toBeUndefined(); + }); + + it('auto-maps an optional source by name and imports it as an id', async () => { + renderWithMantine( + , + ); + + const finish = await screen.findByRole('button', { + name: /finish import/i, + }); + expect(screen.getByDisplayValue('Logs')).toBeInTheDocument(); + await act(async () => { + fireEvent.click(finish); + }); + + await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1)); + const payload = mockMutateAsync.mock.calls[0][0]; + expect(DashboardWithoutIdSchema.safeParse(payload).success).toBe(true); + expect(payload.filters[0]).toMatchObject({ + type: 'SQL_QUERY', + connection: 'conn-default', + source: 'src-logs', + }); + }); + + it('rejects a PromQL source for a filter with a source', () => { + const result = buildMappingFormSchema( + sqlQueryFilterTemplate('default', 'Logs'), + mockSources, + ).safeParse({ + dashboardName: 'SQL Query Filter', + tags: [], + tileSourceMappings: ['src-metrics'], + connectionMappings: [''], + filterSourceMappings: ['src-promql'], + filterConnectionMappings: ['conn-default'], + }); + + expect( + result.error?.issues.map(issue => [issue.path.join('.'), issue.message]), + ).toEqual([ + ['filterSourceMappings.0', 'Select a non-PromQL source for this filter'], + ]); + }); + + it("rejects a source on another connection than the filter's", () => { + const result = buildMappingFormSchema( + sqlQueryFilterTemplate('default', 'Logs'), + mockSources, + ).safeParse({ + dashboardName: 'SQL Query Filter', + tags: [], + tileSourceMappings: ['src-metrics'], + connectionMappings: [''], + filterSourceMappings: ['src-other-logs'], + filterConnectionMappings: ['conn-default'], + }); + + expect( + result.error?.issues.map(issue => [issue.path.join('.'), issue.message]), + ).toEqual([ + ['filterSourceMappings.0', "Select a source on this filter's connection"], + ]); + }); + + it("only lists sources on the filter's mapped connection", async () => { + renderWithMantine( + , + ); + + await screen.findByRole('button', { name: /finish import/i }); + fireEvent.click(screen.getByDisplayValue('Logs')); + + expect( + await screen.findByRole('option', { name: 'Logs' }), + ).toBeInTheDocument(); + expect( + screen.queryByRole('option', { name: 'Other logs' }), + ).not.toBeInTheDocument(); }); it('blocks the import and flags the filter row when its connection is unmapped', async () => { diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index ea2304d03..2430fbeb2 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -48,6 +48,7 @@ import useResizable from '@/hooks/useResizable'; import { useSources } from '@/source'; import { getAllMetricTables, usePrevious } from '@/utils'; import { DEFAULT_TILE_ALERT } from '@/utils/alerts'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; import { SQL_PLACEHOLDERS } from './constants'; import { RawSqlChartInstructions } from './RawSqlChartInstructions'; @@ -56,63 +57,6 @@ import { buildRawSqlCompletions } from './utils'; import resizeStyles from '@/../styles/ResizablePanel.module.scss'; -type ConnectionSourceSyncResult = - | { field: 'connection'; value: string } - | { field: 'source'; value: '' } - | null; - -/** - * Decides how the `connection` and `source` form fields should be kept in sync - * in raw SQL mode. Extracted as a pure function so the behavior can be unit - * tested independently of the component's effect. - */ -export function resolveConnectionSourceSync({ - source, - connection, - prevSource, - prevConnection, - sources, -}: { - source: string | undefined; - connection: string | undefined; - prevSource: string | undefined; - prevConnection: string | undefined; - sources: { id: string; connection: string }[] | undefined; -}): ConnectionSourceSyncResult { - if (!sources) return null; - - // When the source changes, sync the connection to match. - if (source !== prevSource) { - const sourceConnection = sources.find(s => s.id === source)?.connection; - if (sourceConnection && sourceConnection !== connection) { - return { field: 'connection', value: sourceConnection }; - } - return null; - } - - // Set a default connection when none is selected. - if (!connection) { - const defaultConnection = sources[0]?.connection; - if (defaultConnection) { - return { field: 'connection', value: defaultConnection }; - } - return null; - } - - // When the connection changes, clear the source only if the currently - // selected source doesn't belong to the new connection. This avoids clearing - // a source that was just carried over (e.g. from builder mode), where the - // connection change above was itself triggered by the source. - if (connection !== prevConnection && prevConnection !== undefined) { - const sourceConnection = sources.find(s => s.id === source)?.connection; - if (source && sourceConnection !== connection) { - return { field: 'source', value: '' }; - } - } - - return null; -} - export default function RawSqlChartEditor({ control, setValue, diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx index e528d068c..a27c820cb 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx @@ -5,6 +5,7 @@ import { getFilterVariableName, } from '@hyperdx/common-utils/dist/filters'; import { + BroadcastFilter, ChartVariable, DashboardFilter, DashboardFilterType, @@ -83,6 +84,8 @@ interface DashboardFilterEditFormProps { variables?: ChartVariable[]; /** The dashboard's date range, which filter previews run over */ dateRange?: [Date, Date]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; onSave: (filter: DashboardFilter) => void; onClose: () => void; onCancel: () => void; @@ -102,6 +105,7 @@ export const DashboardFilterEditForm = ({ showRequiredFilterOptions, variables, dateRange, + getDashboardFilters, onSave, onClose, onCancel, @@ -258,8 +262,10 @@ export const DashboardFilterEditForm = ({ ) : formFilterType === 'PROMETHEUS_LABEL' ? ( diff --git a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx index fdfcae842..da54cbf94 100644 --- a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx @@ -1,12 +1,18 @@ -import { useMemo, useState } from 'react'; -import { useFormState, useWatch } from 'react-hook-form'; -import { validateSqlQueryFilterTemplate } from '@hyperdx/common-utils/dist/filters'; +import { useEffect, useMemo, useState } from 'react'; +import { UseFormSetValue, useFormState, useWatch } from 'react-hook-form'; +import { + getSqlQueryFilterSourceTables, + SQL_QUERY_FILTER_SOURCE_KINDS, + SqlQueryFilterSourceTables, + validateSqlQueryFilterTemplate, +} from '@hyperdx/common-utils/dist/filters'; import { INTERVAL_MACROS, MACRO_SUGGESTIONS, SOURCE_DEPENDENT_MACROS, } from '@hyperdx/common-utils/dist/macros'; import { + BroadcastFilter, DashboardFilter, SqlQueryDashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -15,6 +21,7 @@ import { IconAlertTriangle, IconPlayerPlay } from '@tabler/icons-react'; import { ConnectionSelectControlled } from '@/components/ConnectionSelect'; import EmptyState from '@/components/EmptyState'; +import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLEditorControlled } from '@/components/SQLEditor/SQLEditor'; import { buildSqlVariableCompletions, @@ -23,10 +30,12 @@ import { } from '@/components/SQLEditor/variableCompletions'; import { NOW } from '@/config'; import { useSqlQueryFilterValues } from '@/hooks/useSqlQueryFilterValues'; -import { truncateText } from '@/utils'; +import { useSources } from '@/source'; +import { truncateText, usePrevious } from '@/utils'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; import { CustomInputWrapper } from './CustomInputWrapper'; -import { FilterFormControl } from './filterFormState'; +import { FilterFormControl, FilterFormValues } from './filterFormState'; import { VariableNameInput } from './VariableNameInput'; /** Values the preview lists before summarizing the rest as a count. */ @@ -34,10 +43,10 @@ const PREVIEW_VALUES_SHOWN = 20; /** Characters of each value the preview shows before truncating it. */ const PREVIEW_VALUE_MAX_LENGTH = 20; -/** - * Macros a SQL query filter can't expand: it has no source, and no time buckets. - */ -const UNSUPPORTED_MACROS = new Set([ +/** Macros a SQL query filter can't expand: it has no time buckets. */ +const UNSUPPORTED_MACROS = new Set(INTERVAL_MACROS); +/** Without a source, the source macros can't expand either. */ +const UNSUPPORTED_MACROS_WITHOUT_SOURCE = new Set([ ...SOURCE_DEPENDENT_MACROS, ...INTERVAL_MACROS, ]); @@ -49,82 +58,167 @@ ORDER BY env`; interface SqlQueryFilterEditFormProps { control: FilterFormControl; + setValue: UseFormSetValue; /** Filters other than the one being edited, used to keep variable names unique. */ otherFilters: DashboardFilter[]; /** The range the preview runs over. Defaults to the last hour. */ dateRange?: [Date, Date]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; } /** Form describing a dashboard filter whose values come from a SQL query */ export const SqlQueryFilterEditForm = ({ control, + setValue, otherFilters, dateRange, + getDashboardFilters, }: SqlQueryFilterEditFormProps) => { const { errors } = useFormState({ control }); const variables = useChartVariables(); - const completions = useMemo( - () => [ + const { data: sources } = useSources(); + + const [connection, source, sqlTemplate] = useWatch({ + control, + name: ['connection', 'source', 'sqlTemplate'], + }); + const sourceObject = source ? sources?.find(s => s.id === source) : undefined; + + const prevSource = usePrevious(source); + const prevConnection = usePrevious(connection); + useEffect(() => { + // The source field is shared with the other filter types, so switching + // type can carry over one this filter can't use. + if ( + source && + sources && + (!sourceObject || + !SQL_QUERY_FILTER_SOURCE_KINDS.includes(sourceObject.kind)) + ) { + setValue('source', ''); + return; + } + const update = resolveConnectionSourceSync({ + source, + connection, + prevSource, + prevConnection, + sources, + }); + if (update?.field === 'connection') { + setValue('connection', update.value); + } else if (update?.field === 'source') { + setValue('source', update.value); + } + }, [ + connection, + prevConnection, + prevSource, + setValue, + source, + sourceObject, + sources, + ]); + + const completions = useMemo(() => { + const unsupported = source + ? UNSUPPORTED_MACROS + : UNSUPPORTED_MACROS_WITHOUT_SOURCE; + return [ ...MACRO_SUGGESTIONS.filter( // `sourceTable(gauge)` etc. are per-metric variants of `sourceTable`. - macro => !UNSUPPORTED_MACROS.has(macro.name.split('(')[0]), + macro => !unsupported.has(macro.name.split('(')[0]), ).map(toMacroCompletion), ...buildSqlVariableCompletions(variables), - ], - [variables], - ); + ]; + }, [variables, source]); - const [connection, sqlTemplate] = useWatch({ - control, - name: ['connection', 'sqlTemplate'], - }); // What the preview last ran, so editing doesn't re-query on every keystroke. const [previewed, setPreviewed] = useState<{ connection: string; + source?: string; sqlTemplate: string; }>(); const canPreview = !!connection && !!sqlTemplate?.trim(); const runPreview = () => { - if (canPreview) setPreviewed({ connection, sqlTemplate }); + if (canPreview) { + setPreviewed({ connection, source: source || undefined, sqlTemplate }); + } + }; + + let sourceTables: SqlQueryFilterSourceTables | 'unknown' | undefined; + if (sourceObject) { + sourceTables = getSqlQueryFilterSourceTables(sourceObject); + } else if (source) { + // Set, but the sources list hasn't loaded yet. + sourceTables = 'unknown'; + } + + const validateSqlTemplate = (value: unknown) => { + if (typeof value !== 'string' || !value.trim()) { + return 'This field is required'; + } + // `variables` leaves out this filter's own, so a reference to it is + // rejected as unknown. + const error = validateSqlQueryFilterTemplate(value, { + variables, + sourceTables, + }); + return error ?? true; }; return ( <> - -
- -
-
+ + +
+ +
+
+ + + {/* The source select's wrapper is inline-flex; a grid cell + stretches it to the full width. */} +
+ +
+
+
{ - if (typeof value !== 'string' || !value.trim()) { - return 'This field is required'; - } - // `variables` leaves out this filter's own, so a reference to - // it is rejected as unknown. - return ( - validateSqlQueryFilterTemplate(value, variables ?? []) ?? true - ); - }, - }} + rules={{ validate: validateSqlTemplate }} height="150px" enableLineWrapping placeholder={PLACEHOLDER_SQL} @@ -148,6 +242,7 @@ export const SqlQueryFilterEditForm = ({ {...previewed} dateRange={dateRange} variables={variables} + getDashboardFilters={getDashboardFilters} /> )} @@ -160,14 +255,18 @@ export const SqlQueryFilterEditForm = ({ const SqlQueryFilterPreview = ({ connection, + source, sqlTemplate, dateRange, variables, + getDashboardFilters, }: { connection: string; + source?: string; sqlTemplate: string; dateRange?: [Date, Date]; variables: ReturnType; + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }) => { const lastHour = useMemo( (): [Date, Date] => [new Date(NOW - 60 * 60 * 1000), new Date(NOW)], @@ -180,18 +279,20 @@ const SqlQueryFilterPreview = ({ type: 'SQL_QUERY', name: 'Preview', connection, + source, sqlTemplate, isBroadcastEnabled: false, isVariableEnabled: true, }, ], - [connection, sqlTemplate], + [connection, source, sqlTemplate], ); const { data, filterErrorMessages, isLoading, isFetching, isError } = useSqlQueryFilterValues({ filters, dateRange: dateRange ?? lastHour, variables, + getDashboardFilters, }); const values = data.get('preview')?.values ?? []; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx index 8c69569e5..96d373e73 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx @@ -1,5 +1,6 @@ import { RegisterOptions, useForm } from 'react-hook-form'; -import { screen } from '@testing-library/react'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { fireEvent, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { @@ -12,6 +13,31 @@ import { SqlVariablesProvider } from '@/components/SQLEditor/variableCompletions const mockUseSqlQueryFilterValues = jest.fn(); const mockCompletionLabels: string[] = []; +let mockSources: TSource[] = []; + +const logSource = (id: string, connection: string): TSource => ({ + id, + name: id, + kind: SourceKind.Log, + connection, + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', +}); + +jest.mock('@/source', () => ({ + ...jest.requireActual('@/source'), + useSources: () => ({ data: mockSources }), +})); +jest.mock('@/components/SourceSelect', () => ({ + SourceSelectControlled: ({ + control, + name, + }: { + control: FilterFormControl; + name: 'source'; + }) => , +})); jest.mock('@/hooks/useSqlQueryFilterValues', () => ({ useSqlQueryFilterValues: (...args: unknown[]) => @@ -69,13 +95,21 @@ const valuesResult = ({ const cloudVariable = { name: 'cloud', expression: 'cloud', values: [] }; -function Harness() { - const { control, handleSubmit } = useForm({ - defaultValues: { ...toFormValues(), type: 'SQL_QUERY' }, +function Harness({ + defaultValues, +}: { + defaultValues?: Partial; +}) { + const { control, handleSubmit, setValue } = useForm({ + defaultValues: { ...toFormValues(), type: 'SQL_QUERY', ...defaultValues }, }); return ( - + @@ -94,6 +128,7 @@ const renderForm = async (sql = 'SELECT env FROM envs') => { describe('SqlQueryFilterEditForm', () => { beforeEach(() => { + mockSources = []; mockUseSqlQueryFilterValues.mockReset(); mockUseSqlQueryFilterValues.mockReturnValue( valuesResult({ values: ['prod', 'dev'] }), @@ -133,10 +168,83 @@ describe('SqlQueryFilterEditForm', () => { await user.click(screen.getByRole('button', { name: 'Save' })); expect( - await screen.findByText(/unknown variable|cannot be used/), + await screen.findByText( + /unknown variable|cannot be used|no source is selected/, + ), ).toBeInTheDocument(); }); + describe('with a source', () => { + const renderWithSource = async (sql: string) => { + mockSources = [logSource('logs', 'conn1'), logSource('other', 'conn2')]; + renderWithMantine( + , + ); + const user = userEvent.setup(); + await user.type(screen.getByTestId('sql-input'), sql); + return user; + }; + + it('suggests and accepts the source macros', async () => { + const user = await renderWithSource( + 'SELECT ServiceName FROM $__sourceTable WHERE $__filters', + ); + + expect(mockCompletionLabels).toContain('$__sourceTable'); + await user.click(screen.getByRole('button', { name: 'Save' })); + + expect( + screen.queryByText(/no source is selected|cannot be used/), + ).not.toBeInTheDocument(); + }); + + it('previews with the source', async () => { + const user = await renderWithSource( + 'SELECT ServiceName FROM $__sourceTable', + ); + + await user.click(screen.getByTestId('filter-sql-preview-button')); + + expect(mockUseSqlQueryFilterValues).toHaveBeenLastCalledWith( + expect.objectContaining({ + filters: [expect.objectContaining({ source: 'logs' })], + }), + ); + }); + + it('clears the source when the connection changes to one it is not on', async () => { + await renderWithSource('SELECT 1'); + + fireEvent.change(screen.getByTestId('connection-input'), { + target: { value: 'conn2' }, + }); + + await waitFor(() => + expect(screen.getByTestId('source-input')).toHaveValue(''), + ); + }); + + it('clears a source of a kind the filter cannot use', async () => { + mockSources = [ + { + id: 'prom', + name: 'prom', + kind: SourceKind.Promql, + connection: 'conn1', + from: { databaseName: 'default', tableName: 'ts' }, + timestampValueExpression: 'Timestamp', + }, + ]; + renderWithMantine( + , + ); + + await waitFor(() => + expect(screen.getByTestId('source-input')).toHaveValue(''), + ); + }); + }); + it('shows the loader, not the last values, while a re-run fetches', async () => { mockUseSqlQueryFilterValues.mockReturnValue( valuesResult({ values: ['prod', 'dev'], isFetching: true }), diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index 08ea54b0b..601b6076a 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -324,9 +324,9 @@ describe('toSavedFilter', () => { name: 'Region', connection: 'conn1', sqlTemplate: ' SELECT region FROM regions ', + source: 'logs', // Left behind by the other editors before the type was switched. expression: 'Env', - source: 'logs', where: 'x = 1', options: ['prod'], }), @@ -337,6 +337,7 @@ describe('toSavedFilter', () => { type: 'SQL_QUERY', name: 'Region', connection: 'conn1', + source: 'logs', sqlTemplate: 'SELECT region FROM regions', isBroadcastEnabled: false, isVariableEnabled: true, @@ -344,10 +345,26 @@ describe('toSavedFilter', () => { }); expect(toFormValues(saved)).toMatchObject({ connection: 'conn1', + source: 'logs', sqlTemplate: 'SELECT region FROM regions', }); }); + it('saves a SQL query filter without a source when none is picked', () => { + const saved = toSavedFilter( + formValues({ + type: 'SQL_QUERY', + name: 'Region', + connection: 'conn1', + source: '', + sqlTemplate: 'SELECT region FROM regions', + }), + ); + + // The schema requires a non-empty source when one is set. + expect(saved).toMatchObject({ source: undefined }); + }); + it('rejects a SQL query filter without a connection', () => { expect(() => toSavedFilter( diff --git a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts index 7e7253d9e..8967624d5 100644 --- a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts +++ b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts @@ -82,7 +82,12 @@ export const toFormValues = ( // QUERY_EXPRESSION fields expression: queried?.expression ?? '', - source: queried?.source ?? promqlLabel?.source ?? presetSourceId ?? '', + source: + queried?.source ?? + promqlLabel?.source ?? + sqlQuery?.source ?? + presetSourceId ?? + '', sourceMetricType: queried?.sourceMetricType, where: queried?.where ?? adhoc?.where ?? '', whereLanguage: @@ -150,6 +155,7 @@ export const toSavedFilter = (values: FilterFormValues): DashboardFilter => { if (values.type === 'SQL_QUERY') { return DashboardFilterSchema.parse({ ...rest, + source: values.source || undefined, sqlTemplate: values.sqlTemplate.trim(), isBroadcastEnabled: false, isVariableEnabled: true, diff --git a/packages/app/src/components/DashboardFiltersModal/index.tsx b/packages/app/src/components/DashboardFiltersModal/index.tsx index 15ec455f0..96c27f9f9 100644 --- a/packages/app/src/components/DashboardFiltersModal/index.tsx +++ b/packages/app/src/components/DashboardFiltersModal/index.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useState } from 'react'; import { + BroadcastFilter, ChartVariable, DashboardFilter, TSource, @@ -22,6 +23,8 @@ interface DashboardFiltersEditModalProps { variables?: ChartVariable[]; /** The dashboard's date range, which filter previews run over */ dateRange?: [Date, Date]; + /** The filter selections that broadcast to a source, for filter previews */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; onClose: () => void; onSaveFilter: (filter: DashboardFilter) => void; onRemoveFilter: (id: string) => void; @@ -36,6 +39,7 @@ const DashboardFiltersModal = ({ showRequiredFilterOptions, variables, dateRange, + getDashboardFilters, onClose, onSaveFilter, onRemoveFilter, @@ -78,6 +82,7 @@ const DashboardFiltersModal = ({ showRequiredFilterOptions={showRequiredFilterOptions} variables={variables} dateRange={dateRange} + getDashboardFilters={getDashboardFilters} onSave={handleSaveFilter} onCancel={stopEditing} onClose={onClose} diff --git a/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx index 1f9691db6..6721b5918 100644 --- a/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx @@ -1,7 +1,10 @@ import React from 'react'; import { + BroadcastFilter, ChartVariable, + SourceKind, SqlQueryDashboardFilter, + TSource, } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { renderHook, waitFor } from '@testing-library/react'; @@ -12,10 +15,35 @@ import { } from '@/hooks/useSqlQueryFilterValues'; const mockQuery = jest.fn(); +const mockRenderSourceFiltersToSql = jest.fn(); +let mockSources: { data?: TSource[]; isLoading: boolean } = { + data: [], + isLoading: false, +}; jest.mock('@/clickhouse', () => ({ getClickhouseClient: () => ({ query: mockQuery }), })); +jest.mock('@/source', () => ({ + useSources: () => mockSources, +})); +jest.mock('@/hooks/useMetadata', () => ({ + useMetadataWithSettings: () => ({}), +})); +jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ + renderSourceFiltersToSql: (...args: unknown[]) => + mockRenderSourceFiltersToSql(...args), +})); + +const logs: TSource = { + id: 'logs', + name: 'Logs', + kind: SourceKind.Log, + connection: 'conn1', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', +}; const respondWith = (rows: { value: string | null }[]) => mockQuery.mockResolvedValue({ json: async () => ({ data: rows }) }); @@ -48,6 +76,7 @@ describe('useSqlQueryFilterValues', () => { const renderFilters = ( filters: SqlQueryDashboardFilter[], variables?: ChartVariable[], + getDashboardFilters?: (sourceId: string) => BroadcastFilter[], ) => renderHook( () => @@ -55,11 +84,14 @@ describe('useSqlQueryFilterValues', () => { filters, dateRange: DATE_RANGE, variables, + getDashboardFilters, }), { wrapper }, ); beforeEach(() => { + mockSources = { data: [logs], isLoading: false }; + mockRenderSourceFiltersToSql.mockReset(); mockQuery.mockReset(); respondWith([{ value: 'prod' }, { value: 'dev' }]); const queryClient = new QueryClient({ @@ -130,7 +162,7 @@ describe('useSqlQueryFilterValues', () => { expect(result.current.erroredFilterIds).toEqual(new Set(['sql1'])); expect(result.current.filterErrorMessages.get('sql1')).toMatch( - /have no source/, + /no source is selected/, ); expect(result.current.data.get('sql1')).toEqual({ values: [], @@ -139,6 +171,77 @@ describe('useSqlQueryFilterValues', () => { expect(mockQuery).not.toHaveBeenCalled(); }); + describe('with a source', () => { + it("expands $__sourceTable to the source's table", async () => { + const { result } = renderFilters( + [ + filter({ + source: 'logs', + sqlTemplate: 'SELECT 1 FROM $__sourceTable', + }), + ], + [], + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockQuery.mock.calls[0][0].query).toMatch(/default.*otel_logs/); + expect(mockRenderSourceFiltersToSql).not.toHaveBeenCalled(); + }); + + it("expands $__filters to the dashboard's selections for the source", async () => { + mockRenderSourceFiltersToSql.mockResolvedValue("(ServiceName = 'api')"); + const selections: BroadcastFilter[] = [ + { type: 'sql', condition: "ServiceName IN ('api')" }, + { type: 'promql', condition: 'job="api"' }, + ]; + const getDashboardFilters = jest.fn(() => selections); + + const { result } = renderFilters( + [ + filter({ + source: 'logs', + sqlTemplate: 'SELECT 1 FROM $__sourceTable WHERE $__filters', + }), + ], + [], + getDashboardFilters, + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(getDashboardFilters).toHaveBeenCalledWith('logs'); + expect(mockRenderSourceFiltersToSql).toHaveBeenCalledWith( + expect.objectContaining({ source: logs, filters: [selections[0]] }), + ); + expect(mockQuery.mock.calls[0][0].query).toContain( + "(ServiceName = 'api')", + ); + }); + + it('waits for the sources to load', () => { + mockSources = { data: undefined, isLoading: true }; + + const { result } = renderFilters([filter({ source: 'logs' })], []); + + expect(result.current.isLoading).toBe(true); + expect(result.current.data.get('sql1')).toEqual({ + values: [], + isLoading: true, + }); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('reports a source that no longer exists', () => { + const { result } = renderFilters([filter({ source: 'deleted' })], []); + + expect(result.current.filterErrorMessages.get('sql1')).toBe( + 'Source deleted not found', + ); + expect(mockQuery).not.toHaveBeenCalled(); + }); + }); + it('reports a failed query with its error message', async () => { mockQuery.mockRejectedValue(new Error('Unknown table default.envs')); diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index d55f19e80..db383efa7 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -7,6 +7,7 @@ import { isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; import { + BroadcastFilter, ChartVariable, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -34,6 +35,7 @@ export function useDashboardFilterValues({ dateRange, selectionByFilterId, variables, + getDashboardFilters, }: { filters: DashboardFilter[]; dateRange: [Date, Date]; @@ -41,6 +43,8 @@ export function useDashboardFilterValues({ selectionByFilterId?: ReadonlyMap; /** The dashboard's variables and their current selections, if any */ variables?: ChartVariable[]; + /** The filter selections that broadcast to a source, for `$__filters` */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }): DashboardFilterValuesResult { const [queriedFilters, staticFilters, promqlLabelFilters, sqlQueryFilters] = useMemo( @@ -74,6 +78,7 @@ export function useDashboardFilterValues({ filters: sqlQueryFilters, dateRange, variables, + getDashboardFilters, }); const data = useMemo( diff --git a/packages/app/src/hooks/useSqlQueryFilterValues.tsx b/packages/app/src/hooks/useSqlQueryFilterValues.tsx index 0a5948127..69132a123 100644 --- a/packages/app/src/hooks/useSqlQueryFilterValues.tsx +++ b/packages/app/src/hooks/useSqlQueryFilterValues.tsx @@ -1,9 +1,18 @@ import { useMemo } from 'react'; import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; -import { renderSqlQueryFilterValuesQuery } from '@hyperdx/common-utils/dist/filters'; +import { renderSourceFiltersToSql } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { + getSqlFilters, + getSqlQueryFilterSourceTables, + renderSqlQueryFilterValuesQuery, +} from '@hyperdx/common-utils/dist/filters'; +import { hasMacro } from '@hyperdx/common-utils/dist/macros'; +import { + BroadcastFilter, ChartVariable, + Filter, SqlQueryDashboardFilter, + TSource, } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, @@ -13,6 +22,10 @@ import { import { getClickhouseClient } from '@/clickhouse'; import { useMetadataQueryAttribution } from '@/queryAttribution'; +import { useSources } from '@/source'; +import { mapKeyBy } from '@/utils'; + +import { useMetadataWithSettings } from './useMetadata'; /** Seconds a filter's values query may run before ClickHouse stops it. */ const MAX_EXECUTION_SECONDS = 30; @@ -20,8 +33,12 @@ const MAX_EXECUTION_SECONDS = 30; type ValuesCall = { filterId: string; connection: string; + sqlTemplate: string; sql: string; params: Record; + source?: TSource; + /** The broadcast conditions `$__filters` expands to, when the SQL uses it. */ + broadcastFilters: Filter[]; }; /** Rows from the values query, without nulls or repeats, in query order. */ @@ -36,34 +53,67 @@ export function useSqlQueryFilterValues({ filters, dateRange, variables, + getDashboardFilters, }: { filters: SqlQueryDashboardFilter[]; dateRange: [Date, Date]; /** The dashboard's variables and their current selections, if any */ variables?: ChartVariable[]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }) { + const { data: sources, isLoading: isLoadingSources } = useSources(); + const sourcesById = useMemo(() => mapKeyBy(sources ?? [], 'id'), [sources]); + const metadata = useMetadataWithSettings(); + // Expanded here so react-query keys on the SQL that actually runs, which - // changes with other filters' selections. + // changes with other filters' selections. `$__filters` is expanded in the + // query function instead, since that needs the source's metadata. const { calls, templateErrors } = useMemo(() => { const resolved: ValuesCall[] = []; const errors = new Map(); for (const filter of filters) { + const source = filter.source ? sourcesById.get(filter.source) : undefined; + if (filter.source && !source) { + // Every source looks missing until the list arrives. + if (!isLoadingSources) { + errors.set(filter.id, `Source ${filter.source} not found`); + } + continue; + } try { + const rendered = renderSqlQueryFilterValuesQuery({ + sqlTemplate: filter.sqlTemplate, + variables, + dateRange, + sourceTables: source + ? getSqlQueryFilterSourceTables(source) + : undefined, + }); resolved.push({ filterId: filter.id, connection: filter.connection, - ...renderSqlQueryFilterValuesQuery({ - sqlTemplate: filter.sqlTemplate, - variables, - dateRange, - }), + sqlTemplate: filter.sqlTemplate, + ...rendered, + source, + broadcastFilters: + source && hasMacro(filter.sqlTemplate, 'filters') + ? getSqlFilters(getDashboardFilters?.(source.id) ?? []) + : [], }); } catch (e) { errors.set(filter.id, e instanceof Error ? e.message : String(e)); } } return { calls: resolved, templateErrors: errors }; - }, [filters, variables, dateRange]); + }, [ + filters, + variables, + dateRange, + sourcesById, + isLoadingSources, + getDashboardFilters, + ]); const attribution = useMetadataQueryAttribution(); @@ -74,15 +124,32 @@ export function useSqlQueryFilterValues({ call.connection, call.sql, call.params, + call.source?.id, + call.broadcastFilters, ], // Keep the last values while a new range or selection loads, rather than // blanking the dropdown. placeholderData: keepPreviousData, staleTime: 1000 * 60 * 5, queryFn: async ({ signal }) => { + let { sql } = call; + if (call.source && call.broadcastFilters.length > 0) { + const filtersSQL = await renderSourceFiltersToSql({ + source: call.source, + filters: call.broadcastFilters, + metadata, + }); + sql = renderSqlQueryFilterValuesQuery({ + sqlTemplate: call.sqlTemplate, + variables, + dateRange, + sourceTables: getSqlQueryFilterSourceTables(call.source), + filtersSQL, + }).sql; + } const result = (await getClickhouseClient({ attribution }) .query<'JSON'>({ - query: call.sql, + query: sql, query_params: call.params, format: 'JSON', abort_signal: signal, @@ -123,13 +190,22 @@ export function useSqlQueryFilterValues({ filterErrorMessages.set(filterId, message); } + // A filter waiting on its source has no call yet, but is still loading. + const isWaitingForSources = + isLoadingSources && filters.some(filter => filter.source); + for (const filter of filters) { + if (isWaitingForSources && !data.has(filter.id)) { + data.set(filter.id, { values: [], isLoading: true }); + } + } + return { data, erroredFilterIds, filterErrorMessages, - isLoading: results.some(r => r.isLoading), - isFetching: results.some(r => r.isFetching), + isLoading: isWaitingForSources || results.some(r => r.isLoading), + isFetching: isWaitingForSources || results.some(r => r.isFetching), isError: erroredFilterIds.size > 0, }; - }, [results, calls, templateErrors]); + }, [results, calls, templateErrors, filters, isLoadingSources]); } diff --git a/packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts b/packages/app/src/utils/__tests__/connectionSourceSync.test.ts similarity index 97% rename from packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts rename to packages/app/src/utils/__tests__/connectionSourceSync.test.ts index 7e979f0a1..b6207e6b5 100644 --- a/packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts +++ b/packages/app/src/utils/__tests__/connectionSourceSync.test.ts @@ -1,4 +1,4 @@ -import { resolveConnectionSourceSync } from '@/components/ChartEditor/RawSqlChartEditor'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; const sources = [ { id: 'src-a', connection: 'conn-1' }, diff --git a/packages/app/src/utils/connectionSourceSync.ts b/packages/app/src/utils/connectionSourceSync.ts new file mode 100644 index 000000000..8e8326f50 --- /dev/null +++ b/packages/app/src/utils/connectionSourceSync.ts @@ -0,0 +1,56 @@ +type ConnectionSourceSyncResult = + | { field: 'connection'; value: string } + | { field: 'source'; value: '' } + | null; + +/** + * Decides how a form's `connection` and `source` fields should be kept in + * sync. Extracted as a pure function so the behavior can be unit tested + * independently of the components' effects. + */ +export function resolveConnectionSourceSync({ + source, + connection, + prevSource, + prevConnection, + sources, +}: { + source: string | undefined; + connection: string | undefined; + prevSource: string | undefined; + prevConnection: string | undefined; + sources: { id: string; connection: string }[] | undefined; +}): ConnectionSourceSyncResult { + if (!sources) return null; + + // When the source changes, sync the connection to match. + if (source !== prevSource) { + const sourceConnection = sources.find(s => s.id === source)?.connection; + if (sourceConnection && sourceConnection !== connection) { + return { field: 'connection', value: sourceConnection }; + } + return null; + } + + // Set a default connection when none is selected. + if (!connection) { + const defaultConnection = sources[0]?.connection; + if (defaultConnection) { + return { field: 'connection', value: defaultConnection }; + } + return null; + } + + // When the connection changes, clear the source only if the currently + // selected source doesn't belong to the new connection. This avoids clearing + // a source that was just carried over (e.g. from builder mode), where the + // connection change above was itself triggered by the source. + if (connection !== prevConnection && prevConnection !== undefined) { + const sourceConnection = sources.find(s => s.id === source)?.connection; + if (source && sourceConnection !== connection) { + return { field: 'source', value: '' }; + } + } + + return null; +} diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index d121d555b..6f1390b9a 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -744,11 +744,15 @@ describe('filters', () => { }); describe('SQL query filters', () => { - const sqlFilter = (sqlTemplate: string): SqlQueryDashboardFilter => ({ + const sqlFilter = ( + sqlTemplate: string, + source?: string, + ): SqlQueryDashboardFilter => ({ id: 'sql', type: 'SQL_QUERY', name: 'Environment', connection: 'conn-1', + source, sqlTemplate, isBroadcastEnabled: false, isVariableEnabled: true, @@ -789,12 +793,30 @@ describe('filters', () => { filterName: 'Environment', language: 'sql', where: sql, - detail: `SQL query filters have no source, so ${macro} cannot be used`, + detail: `SQL uses ${macro} but no source is selected`, }, ]); }, ); + it.each([ + 'SELECT env FROM $__sourceTable WHERE $__filters', + 'SELECT env FROM $__sourceTable(gauge) UNION ALL SELECT env FROM $__sourceTable(sum)', + ])('accepts source macros when the filter has a source: %s', sql => { + expect( + validateDashboardFilterQueries([sqlFilter(sql, 'source-1')]), + ).toEqual([]); + }); + + // No source has both a table and metric tables. + it('flags a mix of bare and metric $__sourceTable calls', () => { + const sql = + 'SELECT env FROM $__sourceTable UNION ALL SELECT env FROM $__sourceTable(gauge)'; + expect( + validateDashboardFilterQueries([sqlFilter(sql, 'source-1')]), + ).toEqual([expect.objectContaining({ filterId: 'sql', where: sql })]); + }); + it.each([ ['an undeclared variable macro', 'WHERE $__filter(env, $missing)'], ['an unterminated macro', 'WHERE $__timeFilter(ts'], @@ -1289,14 +1311,71 @@ describe('filters', () => { expect(sql).not.toContain('$'); }); - it('throws when the template uses a source macro', () => { + it('throws when the template uses a source macro without a source', () => { expect(() => renderSqlQueryFilterValuesQuery({ sqlTemplate: 'SELECT env FROM $__sourceTable', variables: [], dateRange, }), - ).toThrow(/have no source/); + ).toThrow(/no source is selected/); + }); + + it("expands source macros against the filter's source", () => { + const { sql } = renderSqlQueryFilterValuesQuery({ + sqlTemplate: 'SELECT env FROM $__sourceTable WHERE $__filters', + variables: [], + sourceTables: { from: { databaseName: 'db', tableName: 'logs' } }, + filtersSQL: "(ServiceName = 'api')", + dateRange, + }); + expect(sql).toContain( + "SELECT env FROM `db`.`logs` WHERE (ServiceName = 'api')", + ); + }); + + it('expands $__filters to 1=1 when no filters apply', () => { + const { sql } = renderSqlQueryFilterValuesQuery({ + sqlTemplate: 'SELECT env FROM $__sourceTable WHERE $__filters', + variables: [], + sourceTables: { from: { databaseName: 'db', tableName: 'logs' } }, + dateRange, + }); + expect(sql).toContain('WHERE (1=1'); + }); + + it.each([ + ['a bare call on a metric source', 'SELECT 1 FROM $__sourceTable', true], + [ + 'a metric call on a non-metric source', + 'SELECT 1 FROM $__sourceTable(gauge)', + false, + ], + [ + 'a metric type the source has no table for', + 'SELECT 1 FROM $__sourceTable(sum)', + true, + ], + ])('throws on %s', (_label, sqlTemplate, isMetricSource) => { + expect(() => + renderSqlQueryFilterValuesQuery({ + sqlTemplate, + variables: [], + sourceTables: { + from: { databaseName: 'db', tableName: '' }, + metricTables: isMetricSource + ? { + gauge: 'metrics_gauge', + histogram: '', + sum: '', + summary: '', + 'exponential histogram': '', + } + : undefined, + }, + dateRange, + }), + ).toThrow(/\$__sourceTable/); }); it.each([ diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 0f6687e09..d3a649881 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -1639,6 +1639,65 @@ describe('utils', () => { ]); }); + it('should remap a SQL query filter source id to a name, dropping one that no longer exists', () => { + const sqlFilter = { + id: 'filter-sql', + type: 'SQL_QUERY' as const, + name: 'Service', + connection: 'conn1', + source: 'source1', + sqlTemplate: 'SELECT DISTINCT ServiceName FROM $__sourceTable', + isBroadcastEnabled: false as const, + isVariableEnabled: true as const, + }; + const dashboard: z.infer = { + id: 'dashboard1', + name: 'SQL Filter Dashboard', + tags: [], + tiles: [], + filters: [sqlFilter, { ...sqlFilter, id: 'gone', source: 'deleted' }], + }; + const sources: TSource[] = [ + { + id: 'source1', + name: 'Logs', + connection: 'conn1', + kind: SourceKind.Log, + from: { databaseName: 'db1', tableName: 'logs_table' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + ]; + const connections: Connection[] = [ + { + id: 'conn1', + name: 'Production DB', + host: 'http://localhost:8123', + username: 'default', + }, + ]; + + const template = convertToDashboardTemplate( + dashboard, + sources, + connections, + ); + + expect(template.filters?.[0]).toEqual({ + ...sqlFilter, + connection: 'Production DB', + source: 'Logs', + }); + // An empty name would fail the template schema's min(1) on import. + expect(template.filters?.[1]).toEqual({ + ...sqlFilter, + id: 'gone', + connection: 'Production DB', + source: undefined, + }); + expect(() => DashboardTemplateSchema.parse(template)).not.toThrow(); + }); + it('should convert a dashboard without filters to a dashboard template', () => { const dashboard: z.infer = { id: 'dashboard1', diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index a38d8e656..11bbbf2f4 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -51,6 +51,9 @@ import { CteChartConfig, DateRange, DisplayType, + Filter, + isLogSource, + isTraceSource, MetricsDataType, PromqlChartConfig, QuerySettings, @@ -61,6 +64,7 @@ import { SortSpecificationList, SqlAstFilter, SQLInterval, + TSource, } from '@/types'; import { substituteChartConfigVariables } from '@/variables'; @@ -2600,7 +2604,16 @@ async function translateMetricChartConfig( /** Renders the config's filters into a SQL condition string */ async function renderFiltersToSql( - chartConfig: RawSqlChartConfig, + chartConfig: Pick< + RawSqlChartConfig, + | 'filters' + | 'source' + | 'from' + | 'connection' + | 'implicitColumnExpression' + | 'bodyExpression' + | 'useTextIndexForImplicitColumn' + >, metadata: Metadata, ): Promise { if ( @@ -2650,6 +2663,38 @@ async function renderFiltersToSql( return conditions.length > 0 ? `(${conditions.join(' AND ')})` : undefined; } +/** + * The condition `$__filters` expands to for SQL that queries `source`, as for + * a raw SQL tile with that source. + */ +export function renderSourceFiltersToSql({ + source, + filters, + metadata, +}: { + source: TSource; + filters: Filter[]; + metadata: Metadata; +}): Promise { + const isLogOrTrace = isLogSource(source) || isTraceSource(source); + return renderFiltersToSql( + { + filters, + source: source.id, + from: source.from, + connection: source.connection, + implicitColumnExpression: isLogOrTrace + ? source.implicitColumnExpression + : undefined, + bodyExpression: isLogSource(source) ? source.bodyExpression : undefined, + useTextIndexForImplicitColumn: isLogOrTrace + ? source.useTextIndexForImplicitColumn + : undefined, + }, + metadata, + ); +} + export async function renderRawSqlChartConfig( chartConfig: RawSqlChartConfig & Partial, metadata: Metadata, diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index b3e4546e3..312a333b7 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -952,15 +952,20 @@ export function convertToDashboardTemplate( // A static filter references nothing in the workspace if (isStaticListFilter(filter)) return filter; + const getSourceName = (id: string) => + sources.find(source => source.id === id)?.name ?? ''; + if (isSqlQueryFilter(filter)) { filter.connection = connections.find(conn => conn.id === filter.connection)?.name ?? ''; + // The source is optional, so drop it rather than export an empty name + // the template schema would reject. + if (filter.source) { + filter.source = getSourceName(filter.source) || undefined; + } return filter; } - const getSourceName = (id: string) => - sources.find(source => source.id === id)?.name ?? ''; - if (isAdhocFilter(filter)) { filter.sources = filter.sources.map(getSourceName); } else { diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 8beb784a9..e1872fc89 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -3,6 +3,7 @@ import * as SQLParser from 'node-sql-parser'; import { escapeSqlString, replaceJsonExpressions } from '@/core/utils'; import { getSourceDependentMacrosUsed, + getSourceTableMacroArgCounts, hasMacro, INTERVAL_MACROS, replaceMacros, @@ -23,10 +24,13 @@ import { DashboardFilter, DashboardFilterValue, Filter, + MetricsDataType, PROMQL_ADHOC_FILTER_OPERATORS, PromqlConditionFilter, + RawSqlChartConfig, SourceKind, SQL_ADHOC_FILTER_OPERATORS, + TSource, } from '@/types'; import { escapePromqlStringValue, @@ -811,10 +815,11 @@ export function validateDashboardFilterQueries( for (const filter of filters) { if (isSqlQueryFilter(filter)) { - const detail = validateSqlQueryFilterTemplate( - filter.sqlTemplate, + // Source IDs here may not be mapped to this team's sources yet. + const detail = validateSqlQueryFilterTemplate(filter.sqlTemplate, { variables, - ); + sourceTables: filter.source ? 'unknown' : undefined, + }); if (detail) { issues.push({ filterId: filter.id, @@ -856,16 +861,70 @@ export function validateDashboardFilterQueries( return issues; } +/** The tables a SQL query filter's source resolves `$__sourceTable` to. */ +export type SqlQueryFilterSourceTables = Pick< + RawSqlChartConfig, + 'from' | 'metricTables' +>; + +/** The `$__sourceTable` tables of a source, as a raw SQL tile resolves them. */ +export function getSqlQueryFilterSourceTables( + source: TSource, +): SqlQueryFilterSourceTables { + return { + from: source.from, + metricTables: + source.kind === SourceKind.Metric ? source.metricTables : undefined, + }; +} + +/** + * Stand-in tables for a source that is set but not loaded. Every metric table + * is configured when the SQL calls `$__sourceTable()`, so only a + * mix of bare and metric calls, which no source satisfies, is rejected. + */ +function getUnknownSourceTables( + sqlTemplate: string, +): SqlQueryFilterSourceTables { + const hasMetricCall = getSourceTableMacroArgCounts(sqlTemplate).some( + count => count > 0, + ); + return { + from: { databaseName: 'source', tableName: 'table' }, + metricTables: hasMetricCall + ? { + [MetricsDataType.Gauge]: 'table', + [MetricsDataType.Histogram]: 'table', + [MetricsDataType.Sum]: 'table', + [MetricsDataType.Summary]: 'table', + [MetricsDataType.ExponentialHistogram]: 'table', + } + : undefined, + }; +} + +type SqlQueryFilterTemplateContext = { + /** The dashboard's variables and their current selections. */ + variables: ChartVariable[] | undefined; + /** + * The filter's source tables. `'unknown'` when the filter has a source whose + * tables aren't loaded; undefined when it has no source. + */ + sourceTables?: SqlQueryFilterSourceTables | 'unknown'; + /** The broadcast filter conditions `$__filters` expands to. */ + filtersSQL?: string; +}; + /** * Why a SQL query filter's template can't expand, or undefined when it can. * The SQL itself is not parsed: ClickHouse syntax is only checked when it runs. */ export function validateSqlQueryFilterTemplate( sqlTemplate: string, - variables: ChartVariable[], + context: SqlQueryFilterTemplateContext, ): string | undefined { try { - expandSqlQueryFilterTemplate(sqlTemplate, variables); + expandSqlQueryFilterTemplate(sqlTemplate, context); return undefined; } catch (e) { return e instanceof Error ? e.message : String(e); @@ -875,15 +934,17 @@ export function validateSqlQueryFilterTemplate( /** Expand a SQL query filter's macros and variables. Throws when it can't. */ function expandSqlQueryFilterTemplate( sqlTemplate: string, - variables: ChartVariable[] | undefined, + { variables, sourceTables, filtersSQL }: SqlQueryFilterTemplateContext, ): string { - const sourceMacros = getSourceDependentMacrosUsed(sqlTemplate); - if (sourceMacros.length > 0) { - throw new Error( - `SQL query filters have no source, so ${sourceMacros - .map(macro => `$__${macro}`) - .join(' and ')} cannot be used`, - ); + if (!sourceTables) { + const sourceMacros = getSourceDependentMacrosUsed(sqlTemplate); + if (sourceMacros.length > 0) { + throw new Error( + `SQL uses ${sourceMacros + .map(macro => `$__${macro}`) + .join(' and ')} but no source is selected`, + ); + } } const intervalMacros = INTERVAL_MACROS.filter(macro => hasMacro(sqlTemplate, macro), @@ -895,7 +956,11 @@ function expandSqlQueryFilterTemplate( .join(' and ')} cannot be used`, ); } - return replaceMacros({ sqlTemplate, variables }); + const tables = + sourceTables === 'unknown' + ? getUnknownSourceTables(sqlTemplate) + : sourceTables; + return replaceMacros({ sqlTemplate, variables, ...tables }, filtersSQL); } /** Most dropdown values a SQL query filter reads from its query. */ @@ -910,14 +975,13 @@ export const SQL_QUERY_FILTER_VALUES_LIMIT = 10_000; */ export function renderSqlQueryFilterValuesQuery({ sqlTemplate, - variables, dateRange, -}: { + ...context +}: SqlQueryFilterTemplateContext & { sqlTemplate: string; - variables: ChartVariable[] | undefined; dateRange: [Date, Date]; }): { sql: string; params: Record } { - const sql = expandSqlQueryFilterTemplate(sqlTemplate, variables) + const sql = expandSqlQueryFilterTemplate(sqlTemplate, context) .trim() .replace(/;+$/, ''); // The SQL goes on its own lines so a trailing `--` comment can't swallow the @@ -999,6 +1063,14 @@ export const QUERY_EXPRESSION_FILTER_SOURCE_KINDS: SourceKind[] = [ SourceKind.Metric, ]; +/** The source kinds a SQL_QUERY filter can use, the same as raw SQL tiles. */ +export const SQL_QUERY_FILTER_SOURCE_KINDS: SourceKind[] = [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, +]; + /** * The source kinds an ADHOC filter of the given source type can read keys * from. Metric sources need a metric type to pick a table, so they are left out. diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 75b4dda70..11bc4c98a 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2419,6 +2419,11 @@ export const SqlQueryDashboardFilterSchema = dashboardFilterBaseSchema.extend({ type: z.literal(DashboardFilterType.enum.SQL_QUERY), /** ID of the ClickHouse connection the query runs on. */ connection: z.string().min(1), + /** + * ID of the source `$__sourceTable` and `$__filters` resolve against, on the + * same connection. Optional, as for raw SQL tiles. + */ + source: z.string().min(1).optional(), sqlTemplate: z.string().min(1), // Variable-only: there is no column to broadcast isBroadcastEnabled: z.literal(false),