diff --git a/packages/app/src/utils/__tests__/multiSourceMerge.test.ts b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts new file mode 100644 index 0000000000..ae77b18f9e --- /dev/null +++ b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts @@ -0,0 +1,379 @@ +import { + computeFrontier, + coveredUntil, + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; + +const TS_KEY = '__hdx_timestamp'; + +const T = (iso: string) => new Date(iso); +const ms = (iso: string) => new Date(iso).getTime(); + +// Search range: 10:00 - 12:00 UTC +const DATE_RANGE: [Date, Date] = [ + T('2026-08-07T10:00:00Z'), + T('2026-08-07T12:00:00Z'), +]; + +const row = (iso: string, extra: Record = {}) => ({ + [TS_KEY]: iso, + ...extra, +}); + +const makeStream = ( + overrides: Partial & { sourceId: string }, +): StreamSnapshot => ({ + sourceName: overrides.sourceId, + rows: [], + window: null, + lastPageRowCount: null, + hasNextPage: true, + isActive: true, + dateRange: DATE_RANGE, + ...overrides, +}); + +const parseTs = (r: Record) => new Date(r[TS_KEY]).getTime(); + +describe('coveredUntil (DESC)', () => { + it('covers the whole range when the stream is fully drained', () => { + const stream = makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[0].getTime()); + }); + + it('covers nothing during the initial fetch even though hasNextPage is still false', () => { + // useInfiniteQuery reports hasNextPage=false before the first page lands; + // that must not be mistaken for a drained stream. + const stream = makeStream({ sourceId: 'a', hasNextPage: false }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers nothing before the first page arrives', () => { + const stream = makeStream({ sourceId: 'a' }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers through the window start when the last page was empty', () => { + const stream = makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:45:00Z'), + ); + }); + + it('covers only through the oldest fetched row when stopped mid-window at LIMIT', () => { + const stream = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 2, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); +}); + +describe('coveredUntil (ASC)', () => { + it('mirrors the DESC semantics from the start of the range', () => { + expect(coveredUntil(makeStream({ sourceId: 'a' }), 'ASC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(DATE_RANGE[1].getTime()); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(ms('2026-08-07T10:15:00Z')); + }); +}); + +const drainedStream = (sourceId: string) => + makeStream({ + sourceId, + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + +describe('computeFrontier', () => { + it('is the least-covered active stream (max for DESC)', () => { + const drained = drainedStream('a'); + const midWindow = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 1, + }); + expect(computeFrontier([drained, midWindow], 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); + + it('ignores inactive (errored/excluded) streams so they cannot stall the merge', () => { + const drained = drainedStream('a'); + const errored = makeStream({ sourceId: 'b', isActive: false }); + expect(computeFrontier([drained, errored], 'DESC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + }); + + it('is null when no stream is active', () => { + const errored = makeStream({ sourceId: 'a', isActive: false }); + expect(computeFrontier([errored], 'DESC', parseTs)).toBeNull(); + }); +}); + +describe('mergeStreams', () => { + const window0 = { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }; + + it('interleaves rows across streams newest-first and tags their source', () => { + const a = makeStream({ + sourceId: 'a', + sourceName: 'app logs', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:57:00Z')], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + sourceName: 'traces', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + '2026-08-07T11:57:00Z', + ]); + expect(rows.map(r => r[MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME])).toEqual([ + 'app logs', + 'traces', + 'app logs', + ]); + expect(rows[0][MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]).toBe('a'); + }); + + it('holds back rows older than the frontier until lagging streams catch up', () => { + // Stream a is fully drained down to 10:00; stream b stopped at LIMIT with + // its oldest row at 11:50 — anything older than 11:50 from a must wait. + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T11:55:00Z'), + row('2026-08-07T11:49:00Z'), // older than b's coverage — held back + ], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'DESC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T11:50:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:55:00Z', + '2026-08-07T11:50:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('shows everything when all streams are drained', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T10:05:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:03:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows, laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows).toHaveLength(2); + expect(laggingSourceIds).toEqual([]); + }); + + it('holds everything back while a stream has no page yet, without marking it lagging', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const pending = makeStream({ sourceId: 'b' }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, pending], + 'DESC', + TS_KEY, + ); + + // Frontier sits at the range end until b's first page lands. + expect(rows).toEqual([]); + // b's initial fetch is already in flight — nothing to advance. + expect(laggingSourceIds).toEqual([]); + }); + + it('still shows rows from errored streams but never waits on them', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const errored = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + isActive: false, + }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, errored], + 'DESC', + TS_KEY, + ); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + ]); + expect(laggingSourceIds).toEqual([]); + }); + + it('merges oldest-first with a mirrored frontier for ASC', () => { + const window0Asc = { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }; + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T10:01:00Z'), + row('2026-08-07T10:20:00Z'), // beyond b's coverage — held back + ], + window: window0Asc, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:05:00Z')], + window: window0Asc, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'ASC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T10:05:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T10:01:00Z', + '2026-08-07T10:05:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('advances every stream tied at the frontier', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(laggingSourceIds).toEqual(['a', 'b']); + }); +}); diff --git a/packages/app/src/utils/__tests__/sourceParams.test.ts b/packages/app/src/utils/__tests__/sourceParams.test.ts index d26d25c538..c2a86598c7 100644 --- a/packages/app/src/utils/__tests__/sourceParams.test.ts +++ b/packages/app/src/utils/__tests__/sourceParams.test.ts @@ -2,6 +2,7 @@ import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { resolveSourceParam, + resolveSourcesParam, SourceForParamResolution, } from '@/utils/sourceParams'; @@ -174,3 +175,65 @@ describe('resolveSourceParam', () => { }); }); }); + +describe('resolveSourcesParam', () => { + it('resolves a list by ID and by name', () => { + expect(resolveSourcesParam(['log-1', 'Traces'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: [], + }); + }); + + it('reports pending while sources are loading', () => { + expect(resolveSourcesParam(['log-1'], undefined)).toEqual({ + status: 'pending', + }); + }); + + it('resolves an empty selection without waiting on the source list', () => { + expect(resolveSourcesParam([], undefined)).toEqual({ + status: 'resolved', + sources: [], + unresolved: [], + }); + }); + + it('keeps what resolves and reports the rest, so one bad entry does not sink the selection', () => { + expect(resolveSourcesParam(['log-1', 'Nope', 'Traces'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: ['Nope'], + }); + }); + + it('reports an entry naming a source of the wrong kind', () => { + expect( + resolveSourcesParam(['log-1', 'Traces'], SOURCES, { + kinds: [SourceKind.Log], + }), + ).toEqual({ + status: 'resolved', + sources: [LOGS_1], + unresolved: ['Traces'], + }); + }); + + it('dedupes entries that resolve to the same source', () => { + expect(resolveSourcesParam(['log-1', 'log-1'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1], + unresolved: [], + }); + }); + + it('caps the selection at `max`, keeping the first entries', () => { + expect( + resolveSourcesParam(['log-1', 'Traces', 'Old Logs'], SOURCES, { max: 2 }), + ).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: [], + }); + }); +}); diff --git a/packages/app/src/utils/multiSourceMerge.ts b/packages/app/src/utils/multiSourceMerge.ts new file mode 100644 index 0000000000..ae53b730c4 --- /dev/null +++ b/packages/app/src/utils/multiSourceMerge.ts @@ -0,0 +1,239 @@ +/** + * Pure merge logic for multi-source search: k-way merges per-source result + * streams by timestamp, bounded by a "safe frontier" so the interleaved + * timeline never shows a gap another source could still fill. + * + * Every source stream paginates through the same progressive time windows + * (see utils/searchWindows.ts — windows are a pure function of the date + * range), but streams advance at different speeds: one source may be three + * windows deep while another is still mid-window at its row LIMIT. A merged + * DESC timeline is only correct down to the timestamp every stream has + * covered; rows older than that are held back until the lagging streams catch + * up. + */ + +/** Client-side fields tagged onto every merged row. Never sent to ClickHouse. */ +export const MULTI_SOURCE_ROW_FIELDS = { + SOURCE_ID: '__hdx_source_id', + SOURCE_NAME: '__hdx_source_name', + SOURCE_COLOR: '__hdx_source_color', +} as const; + +export type MergeDirection = 'ASC' | 'DESC'; + +export type StreamSnapshot = { + sourceId: string; + sourceName: string; + /** Badge/series color for this source; tagged onto rows for the table cell. */ + sourceColor?: string; + /** + * Rows fetched so far, in stream order (newest-first for DESC, + * oldest-first for ASC) — the order the windowed query produces. + */ + rows: Record[]; + /** The last fetched page's time window; null when no page has completed. */ + window: { startTime: Date; endTime: Date } | null; + /** + * Row count of the last fetched page; 0 means the window was drained, + * >0 means the stream may have stopped mid-window at its LIMIT. + * Null when no page has completed. + */ + lastPageRowCount: number | null; + hasNextPage: boolean; + /** + * Errored/excluded streams don't bound the frontier (they'd stall the merge + * forever); their already-fetched rows are still shown. + */ + isActive: boolean; + /** The full searched range, used when a stream is fully drained. */ + dateRange: [Date, Date]; +}; + +/** + * Epoch-ms timestamp T such that this stream is guaranteed to have produced + * every row it has on the already-covered side of T: + * DESC — all of the stream's rows with ts >= T are fetched; + * ASC — all of the stream's rows with ts <= T are fetched. + * + * Conservative by construction: when the stream stopped mid-window at its + * LIMIT, coverage only extends to the last row it returned, not the window + * boundary. + */ +export function coveredUntil( + stream: StreamSnapshot, + direction: MergeDirection, + parseTs: (row: Record) => number, +): number { + const [start, end] = stream.dateRange; + + if (stream.window == null || stream.lastPageRowCount == null) { + // Nothing fetched yet: no coverage at all. Checked before hasNextPage — + // useInfiniteQuery reports hasNextPage=false during the initial fetch, + // which must not read as "fully drained". + return direction === 'DESC' ? end.getTime() : start.getTime(); + } + + if (!stream.hasNextPage) { + // Fully drained: the stream covered the entire searched range. + return direction === 'DESC' ? start.getTime() : end.getTime(); + } + + if (stream.lastPageRowCount === 0) { + // The last window came back empty, so it is fully covered. + return direction === 'DESC' + ? stream.window.startTime.getTime() + : stream.window.endTime.getTime(); + } + + // Mid-window at LIMIT: covered only through the last row returned. Rows are + // in stream order, so the last row is the furthest-along one. + const lastRow = stream.rows[stream.rows.length - 1]; + if (lastRow == null) { + // Defensive: a non-zero lastPageRowCount implies rows exist. + return direction === 'DESC' + ? stream.window.endTime.getTime() + : stream.window.startTime.getTime(); + } + return parseTs(lastRow); +} + +/** + * The merge frontier: the timestamp every active stream has covered. + * DESC — rows with ts >= frontier are safe to show; ASC — ts <= frontier. + * Null when there are no active streams (nothing bounds the merge). + */ +export function computeFrontier( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): number | null { + let frontier: number | null = null; + for (const stream of streams) { + if (!stream.isActive) continue; + const covered = coveredUntil(stream, direction, parseTs); + if (frontier == null) { + frontier = covered; + } else { + frontier = + direction === 'DESC' + ? Math.max(frontier, covered) + : Math.min(frontier, covered); + } + } + return frontier; +} + +/** + * The active streams holding the frontier back that can be advanced with + * another page fetch. Streams whose initial fetch hasn't completed are not + * included — their in-flight request IS their advancement. + */ +function laggingStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): StreamSnapshot[] { + const frontier = computeFrontier(streams, direction, parseTs); + if (frontier == null) return []; + return streams.filter( + stream => + stream.isActive && + stream.hasNextPage && + stream.window != null && + coveredUntil(stream, direction, parseTs) === frontier, + ); +} + +export type MergedRow = Record; + +/** + * Merge all fetched rows across streams into one timestamp-ordered list, + * tagged with their origin source, held back at the frontier. + * + * Rows from inactive (errored/excluded) streams are still included — they are + * valid data — but only active streams bound the frontier, so a dead source + * can't freeze the timeline. + */ +function mergeStreamRows( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): MergedRow[] { + // Timestamps repeat heavily at second precision; cache the Date parse per + // distinct raw value (same trick as ChartUtils' time-chart transform). + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + const frontier = computeFrontier(streams, direction, parseTs); + + const tagged: { row: MergedRow; ts: number }[] = []; + for (const stream of streams) { + for (const row of stream.rows) { + const ts = parseTs(row); + if ( + frontier != null && + (direction === 'DESC' ? ts < frontier : ts > frontier) + ) { + continue; + } + tagged.push({ + row: { + ...row, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: stream.sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: stream.sourceName, + ...(stream.sourceColor != null + ? { [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: stream.sourceColor } + : {}), + }, + ts, + }); + } + } + + // Array.prototype.sort is stable, so ties keep (stream order, row order). + tagged.sort((a, b) => (direction === 'DESC' ? b.ts - a.ts : a.ts - b.ts)); + + return tagged.map(t => t.row); +} + +/** + * Convenience wrapper used by the table component: one pass producing the + * merged rows, the frontier (for the "loading up to" indicator), and which + * streams to advance on the next fetch. + */ +export function mergeStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): { + rows: MergedRow[]; + frontier: number | null; + laggingSourceIds: string[]; +} { + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + return { + rows: mergeStreamRows(streams, direction, timestampKey), + frontier: computeFrontier(streams, direction, parseTs), + laggingSourceIds: laggingStreams(streams, direction, parseTs).map( + s => s.sourceId, + ), + }; +} diff --git a/packages/app/src/utils/sourceParams.ts b/packages/app/src/utils/sourceParams.ts index 5874111c99..dc759c11b7 100644 --- a/packages/app/src/utils/sourceParams.ts +++ b/packages/app/src/utils/sourceParams.ts @@ -55,6 +55,51 @@ export type SourceParamResolution = * lowest ID, so the same link always resolves to the same source no matter what * order the API returns them in. */ +/** + * Resolve a list of source params (IDs or names) for multi-source search. + * Each element resolves with the same rules as `resolveSourceParam`; results + * are deduped by ID and capped at `max`. Elements that can't be resolved (or + * resolve to a source of the wrong kind) are reported in `unresolved` so the + * caller can warn without failing the rest of the selection. + */ +export function resolveSourcesParam( + paramValues: string[] | null | undefined, + sources: T[] | undefined, + { kinds, max }: { kinds?: SourceKind[]; max?: number } = {}, +): + | { status: 'pending' } + | { status: 'resolved'; sources: T[]; unresolved: string[] } { + if (paramValues == null || paramValues.length === 0) { + return { status: 'resolved', sources: [], unresolved: [] }; + } + if (sources == null) return { status: 'pending' }; + + const resolved: T[] = []; + const seenIds = new Set(); + const unresolved: string[] = []; + + for (const value of paramValues) { + const resolution = resolveSourceParam(value, sources, { kinds }); + if (resolution.status === 'resolved') { + if (!seenIds.has(resolution.source.id)) { + seenIds.add(resolution.source.id); + resolved.push(resolution.source); + } + } else if ( + resolution.status === 'not-found' || + resolution.status === 'wrong-kind' + ) { + unresolved.push(value); + } + } + + return { + status: 'resolved', + sources: max != null ? resolved.slice(0, max) : resolved, + unresolved, + }; +} + export function resolveSourceParam( paramValue: string | null | undefined, sources: T[] | undefined,