Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/source-vars-1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/common-utils': minor
'@hyperdx/api': minor
'@hyperdx/app': patch
---

feat: add data source and connection variable types to dashboards
6 changes: 6 additions & 0 deletions packages/api/src/controllers/alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
isPromqlSavedChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import { configUsesReferenceVariable } from '@hyperdx/common-utils/dist/referenceVariables';
import { isRangeThresholdType } from '@hyperdx/common-utils/dist/types';
import { groupBy } from 'lodash';
import { type FilterQuery, Types } from 'mongoose';
Expand Down Expand Up @@ -97,6 +98,11 @@ export const validateAlertInput = async (

// The alert checker cannot evaluate these tiles, so the alert would never run.
if (tile.config != null && !savedChartConfigSupportsAlerts(tile.config)) {
if (configUsesReferenceVariable(tile.config)) {
throw new Api400Error(
'Alerts are not supported on tiles that use a data source or connection variable',
);
}
if (isPromqlSavedChartConfig(tile.config)) {
throw new Api400Error('Alerts are not supported on PromQL tiles');
}
Expand Down
124 changes: 124 additions & 0 deletions packages/api/src/routers/api/__tests__/dashboard.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1518,6 +1518,130 @@ describe('dashboard router', () => {
);
});

describe('data source variables', () => {
const dataSourceFilter = {
id: new Types.ObjectId().toString(),
type: 'DATA_SOURCE' as const,
name: 'Logs source',
variableName: 'logs',
allowedSourceKinds: [SourceKind.Log],
minSelections: 1,
maxSelections: 1,
isBroadcastEnabled: false,
isVariableEnabled: true,
};

it('persists a tile and a filter that reference the variable', async () => {
const tile = makeTile({ sourceId: '$logs' });
const queriedFilter = {
id: new Types.ObjectId().toString(),
type: 'QUERY_EXPRESSION' as const,
name: 'Service',
expression: 'ServiceName',
source: '$logs',
};

const created = await agent
.post('/dashboards')
.send({
...MOCK_DASHBOARD,
tiles: [tile],
filters: [dataSourceFilter, queriedFilter],
})
.expect(200);

expect(created.body.tiles[0].config.source).toBe('$logs');
expect(created.body.filters).toEqual([dataSourceFilter, queriedFilter]);
});

it('rejects a reference to an unknown variable', async () => {
const res = await agent
.post('/dashboards')
.send({ ...MOCK_DASHBOARD, tiles: [makeTile({ sourceId: '$nope' })] })
.expect(400);

expect(JSON.stringify(res.body)).toMatch(
/does not name a data source variable/,
);
});

it('rejects a variable offering kinds the reference cannot use', async () => {
const res = await agent
.post('/dashboards')
.send({
...MOCK_DASHBOARD,
tiles: [makeTile({ sourceId: '$logs' })],
filters: [
{
...dataSourceFilter,
allowedSourceKinds: [SourceKind.Log, SourceKind.Promql],
},
],
})
.expect(400);

expect(JSON.stringify(res.body)).toMatch(/promql/);
});

it('validates a PATCH of tiles against the stored filters', async () => {
const created = await agent
.post('/dashboards')
.send({ ...MOCK_DASHBOARD, filters: [dataSourceFilter] })
.expect(200);

await agent
.patch(`/dashboards/${created.body.id}`)
.send({ tiles: [makeTile({ sourceId: '$logs' })] })
.expect(200);
await agent
.patch(`/dashboards/${created.body.id}`)
.send({ filters: [] })
.expect(400);
});

it('drops an alert on a tile that uses the variable', async () => {
await agent
.post('/dashboards')
.send({
...MOCK_DASHBOARD,
tiles: [
makeTile({
sourceId: '$logs',
alert: makeMockAlert(webhook._id.toString()),
}),
],
filters: [dataSourceFilter],
})
.expect(200);

expect(await Alert.countDocuments({ team: team._id })).toBe(0);
});

it('rejects a tile alert on a tile that uses the variable', async () => {
const created = await agent
.post('/dashboards')
.send({
...MOCK_DASHBOARD,
tiles: [makeTile({ sourceId: '$logs' })],
filters: [dataSourceFilter],
})
.expect(200);

const res = await agent
.post('/alerts')
.send(
makeAlertInput({
dashboardId: created.body.id,
tileId: created.body.tiles[0].id,
webhookId: webhook._id.toString(),
}),
)
.expect(400);

expect(res.body.message).toMatch(/data source or connection variable/);
});
});

describe('promql-label filters', () => {
const createPromqlSource = () =>
Source.create({
Expand Down
26 changes: 19 additions & 7 deletions packages/api/src/routers/api/dashboards.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
getVariableReferenceError,
validateDashboardFilterModes,
validateDashboardFilterOptionUniqueness,
validateDashboardFilterVariableNames,
Expand All @@ -8,6 +9,7 @@ import {
isAdhocFilter,
isPrometheusLabelFilter,
} from '@hyperdx/common-utils/dist/filters';
import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables';
import {
DashboardFilter,
DashboardSchema,
Expand Down Expand Up @@ -103,7 +105,10 @@ async function validateFilterSources(
);
const newRefs = filters.flatMap(filter =>
getFilterSourceRefs(filter).filter(
ref => !existingRefsById.get(filter.id)?.has(refKey(ref)),
ref =>
// Variable references are checked by validateDashboardVariableReferences
parseVariableReference(ref.id) == null &&
!existingRefsById.get(filter.id)?.has(refKey(ref)),
),
);
if (newRefs.length === 0) return null;
Expand Down Expand Up @@ -178,7 +183,9 @@ router.post(
// dashboard to the provisioner.
const dashboard = _.omit(req.body, 'provisioned');

const sourceError = await validateFilterSources(teamId, req.body.filters);
const sourceError =
getVariableReferenceError(req.body.tiles, req.body.filters) ??
(await validateFilterSources(teamId, req.body.filters));
if (sourceError != null) {
return res.status(400).json({ message: sourceError });
}
Expand Down Expand Up @@ -223,11 +230,16 @@ router.patch(
// null still signals field removal downstream.
const updates = DashboardWithoutIdSchema.partial().parse(req.body);

const sourceError = await validateFilterSources(
teamId,
req.body.filters,
dashboard.filters,
);
const sourceError =
getVariableReferenceError(
req.body.tiles ?? dashboard.tiles,
req.body.filters ?? dashboard.filters,
) ??
(await validateFilterSources(
teamId,
req.body.filters,
dashboard.filters,
));
if (sourceError != null) {
return res.status(400).json({ message: sourceError });
}
Expand Down
17 changes: 8 additions & 9 deletions packages/api/src/routers/external-api/v2/utils/dashboards.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
TIME_SERIES_DISPLAY_TYPE_BY_NAME,
} from '@hyperdx/common-utils/dist/core/utils';
import {
collectRefinementIssues,
validateDashboardContainersStructure,
validateDashboardFilterFieldGating,
validateDashboardFilterModes,
Expand Down Expand Up @@ -1942,15 +1943,13 @@ export function collectTileContainerRefIssues(
containers: DashboardContainer[],
tiles: ExternalDashboardTileWithId[],
): string[] {
const schema = z.object({}).superRefine((_, ctx) => {
const containerById = new Map<string, DashboardContainer>(
containers.map(c => [c.id, c]),
);
validateDashboardTileContainerRefs(containerById, tiles, ctx);
});
const result = schema.safeParse({});
if (result.success) return [];
return result.error.issues.map(issue => {
const containerById = new Map<string, DashboardContainer>(
containers.map(c => [c.id, c]),
);
const issues = collectRefinementIssues(ctx =>
validateDashboardTileContainerRefs(containerById, tiles, ctx),
);
return issues.map(issue => {
const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : '';
return `${path}${issue.message}`;
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,41 @@ describe('provisionDashboards', () => {
// rejects the legacy enum and `readDashboardFiles` skips the file
// outright, so a provisioned dashboard authored against #2265
// wouldn't ship at all.
it('parses a tile that references a declared data source variable', () => {
fs.writeFileSync(
path.join(tmpDir, 'source-variable.json'),
JSON.stringify({
name: 'Source Variable',
tiles: [makeTile({ sourceId: '$logs' })],
filters: [
{
id: 'ds',
type: 'DATA_SOURCE',
name: 'Logs source',
variableName: 'logs',
allowedSourceKinds: ['log'],
minSelections: 1,
maxSelections: 1,
isBroadcastEnabled: false,
isVariableEnabled: true,
},
],
}),
);
expect(readDashboardFiles(tmpDir)).toHaveLength(1);
});

it('skips a tile that references an undeclared data source variable', () => {
fs.writeFileSync(
path.join(tmpDir, 'unknown-variable.json'),
JSON.stringify({
name: 'Unknown Variable',
tiles: [makeTile({ sourceId: '$logs' })],
}),
);
expect(readDashboardFiles(tmpDir)).toEqual([]);
});

it('migrates legacy chart-N tile colors before schema validation', () => {
const tile = makeTile();
(tile.config as any).color = 'chart-1';
Expand Down
11 changes: 8 additions & 3 deletions packages/api/src/tasks/provisionDashboards/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { validateDashboardFilterOptionUniqueness } from '@hyperdx/common-utils/dist/dashboardValidation';
import {
validateDashboardFilterOptionUniqueness,
validateDashboardVariableReferences,
} from '@hyperdx/common-utils/dist/dashboardValidation';
import {
DashboardWithoutId,
DashboardWithoutIdSchema,
Expand Down Expand Up @@ -30,8 +33,10 @@ function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown {
}

const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine(
(data, ctx) =>
validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx),
(data, ctx) => {
validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx);
validateDashboardVariableReferences(data.tiles, data.filters ?? [], ctx);
},
);

export function readDashboardFiles(dir: string): DashboardWithoutId[] {
Expand Down
8 changes: 6 additions & 2 deletions packages/api/src/utils/externalApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
isFilterBroadcastEnabled,
isFilterRequired,
isFilterVariableEnabled,
isReferenceVariableFilter,
} from '@hyperdx/common-utils/dist/filters';
import {
AlertErrorType,
Expand Down Expand Up @@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig(
}

/** Filter types the external API and MCP don't expose yet. */
type InternalOnlyDashboardFilter = Extract<DashboardFilter, { type: 'ADHOC' }>;
type InternalOnlyDashboardFilter = Extract<
DashboardFilter,
{ type: 'ADHOC' | 'DATA_SOURCE' | 'CONNECTION' }
>;
export type ExternalCompatibleDashboardFilter = Exclude<
DashboardFilter,
InternalOnlyDashboardFilter
Expand All @@ -253,7 +257,7 @@ export function getExternalCompatibleFilters(
): ExternalCompatibleDashboardFilter[] {
return (filters ?? []).filter(
(filter): filter is ExternalCompatibleDashboardFilter =>
!isAdhocFilter(filter),
!isAdhocFilter(filter) && !isReferenceVariableFilter(filter),
);
}

Expand Down
7 changes: 2 additions & 5 deletions packages/app/src/components/ChartEditor/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
isMetricSource,
isRangeThresholdType,
isTraceSource,
NON_PROMQL_SOURCE_KINDS,
PromqlChartConfig,
PromqlHeatmapMode,
PromqlSavedChartConfig,
Expand Down Expand Up @@ -208,10 +209,6 @@ export const isPromqlDisplayType = (
displayType === DisplayType.Number ||
displayType === DisplayType.Heatmap;

const NON_PROMQL_SOURCE_KINDS = Object.values(SourceKind).filter(
kind => kind !== SourceKind.Promql,
);

/**
* Search and event patterns list raw rows out of the source's `from` table. A
* metric source keeps its rows in `metricTables` and leaves `from.tableName`
Expand Down Expand Up @@ -240,7 +237,7 @@ export function getAllowedSourceKinds({
if (isStringSelectDisplayType(displayType)) {
return ROW_LISTING_SOURCE_KINDS;
}
return NON_PROMQL_SOURCE_KINDS;
return [...NON_PROMQL_SOURCE_KINDS];
}

const isCustomOrderByDisplayType = (
Expand Down
Loading
Loading