From e3113ab9e7a093fcb2854e06cd97eac72a29dbf3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 09:27:24 -0800 Subject: [PATCH 1/6] docs(proposals): 0012 bounded trace ranking for drill-down Drill-down ranks every retained trace in its window, so it times out once the server holds data. Propose exactly maintained per-bucket rank lists in the read cache, and add the filebench scale benchmark that measures the current cost. --- docs/proposals/0012-bounded-trace-ranking.md | 230 ++++++++++++++++++ docs/proposals/README.md | 1 + .../trace_candidate_scale_bench_test.go | 106 ++++++++ 3 files changed, 337 insertions(+) create mode 100644 docs/proposals/0012-bounded-trace-ranking.md create mode 100644 internal/engine/trace_candidate_scale_bench_test.go diff --git a/docs/proposals/0012-bounded-trace-ranking.md b/docs/proposals/0012-bounded-trace-ranking.md new file mode 100644 index 00000000..fac5fb36 --- /dev/null +++ b/docs/proposals/0012-bounded-trace-ranking.md @@ -0,0 +1,230 @@ +# Bounded trace ranking for drill-down + +Status: Draft +Date: 2026-11-18 +Pull request: TBD + +## Summary + +Drill-down ranks every retained trace in its window to return the 21 slowest +or failing ones, so its cost grows with retained data and it times out on a +server that has run for a while. This proposal keeps a small, exactly +maintained ranked list of traces per time bucket and scope in the read cache. +A query merges the lists of the buckets inside its window and scans exactly +only what the lists cannot answer. Cost then depends on window length and list +size, not on how many traces are retained. + +## Motivation + +`queryrows.CompletedTraceQuery` serves service-map drill-down +(`indexedExemplars` in `internal/query/exemplars.go`, limit 21) and the +recent-trace lookup (`observability.Service.Trace`, limit 1). It orders every +in-window trace candidate by `has_error DESC, max_end-min_start DESC`. + +Evidence: + +- pitot-bench on a Hetzner ccx33 pair, `v2026.11.9-alpha`, 48k items/s mixed + ingest with 20 UI reads/s. A fresh server (about 15M rows) passes the read + gate at 20/20 ops/s on 3.25 cores. After a 10-minute prefill (about 180M + rows) the same load completes 2.6/20 ops/s on 7.3 cores. Every one of 1,716 + failures is a 504 on `/api/queries/exemplars`, and no drill-down or trace + detail succeeds. A local run on an M-series Mac reproduces it (4.8/20 ops/s). +- `internal/engine/trace_candidate_scale_bench_test.go` (build tag + `filebench`) times the query against a fully cached index with a two-batch + uncached tail: + + | Cached traces | Single-service traces | Two-service traces | + |---|---|---| + | 400k | 21 ms | 93 ms | + | 1.6M | 197 ms | 633 ms | + + Four times the traces costs seven to nine times the time. The Hetzner run + held roughly 30M traces; a day at 48k rows/s holds billions of rows, and + dashboard windows reach 24h. +- `EXPLAIN ANALYZE` at 1.6M two-service traces (0.59 s): every candidate has + `service=NULL`, so a service scope takes the repair path and aggregates every + retained `read_trace_parts` row (`HASH_JOIN RIGHT_SEMI` 207 ms with a 64,000 + row estimate against 1.6M actual, `HASH_GROUP_BY` 258 ms, a second 206 ms + group-by for the boundary subquery). DuckDB factors `trace_candidates` into a + materialized common subplan despite the `NOT MATERIALIZED` hint and spills + 275 MB. The top-N steps themselves cost about 4 ms. + +Rewriting the joins does not help once the index is fully cached (603 ms +against 624 ms). The work is proportional to retained traces by construction. + +## Goals and non-goals + +Goals: + +- Drill-down and recent-trace cost independent of retained traces, for every + scope (namespace and service set or empty) and window up to the retention + period. +- Exactly the current results, including ties (`trace_id`, then + `namespace`), late publication, compaction and retention. +- Bounded maintenance cost per cached batch. + +Non-goals: + +- Changing the ranking key or the definition of a trace's duration + (`max_end - min_start` over in-window, in-scope spans). +- Approximate ranking. Peers rank by root-span duration (SigNoz, Honeycomb, + Uptrace), return the first N matches (Tempo, Jaeger), or accept per-block + durations that understate split traces (Tempo). Pitot keeps its exact + definition. +- Trace search with arbitrary attribute filters, which keeps the generic path. + +## Design + +### Rank lists + +A new private read-cache table, `read_trace_rank`: + +| Column | Meaning | +|---|---| +| `tier` | minute, hour or day | +| `bucket` | bucket start, Unix nanoseconds | +| `namespace`, `service` | scope; `service` is `''` for the all-services list | +| `trace_id`, `min_start`, `max_start`, `max_end`, `has_error` | the trace's score in that scope | + +Each (tier, bucket, namespace, service) group holds the top +`queryrows.TraceRankDepth` (21) traces by the production ordering. A trace +belongs to the minute bucket of its `min_start`. Only short traces +(`max_start - min_start` under one minute) enter lists. + +A trace whose spans cover several services gets one row per service. Each +row's score is computed from that service's parts, because a service-scoped +query ranks only that service's spans. Rank rows store results only. The +"one incremental candidate per trace" rule still holds for +`read_trace_candidates`. + +### Why it is exact + +Each short trace lies in exactly one minute bucket per scope, so buckets +partition the traces. If a trace is in the global top K of a window, at most +K-1 traces outrank it anywhere, so at most K-1 outrank it in its own bucket. +It is therefore in its bucket's list, and the union of the lists of fully +covered buckets contains the global top K over those buckets. Hour and day +lists are the top K of their child lists, by the same argument. + +Scores come from merged candidates, never from per-batch parts. A newly +published span only lowers `min_start`, raises `max_end` or sets `has_error`, +so a trace's score never falls on publication. Updating a list is then an +insert with eviction of the lowest entry. Two events can lower a score or move +a trace between buckets: retention retiring part of a trace, and `min_start` +moving into an earlier bucket. Both rebuild the affected bucket from +`read_trace_candidates`, at a cost bounded by that bucket's traces. + +### Maintenance + +In the same `cacheBatches` transaction that inserts parts, merges candidates +and writes markers (`internal/engine/batch_cache.go`): + +1. Before the candidate MERGE, record the bucket keys of the candidates it + touches. After the MERGE, record their new keys. +2. Recompute those minute groups from `read_trace_candidates` and, for + per-service scores, from that bucket's `read_trace_parts`, using + `QUALIFY row_number() OVER (PARTITION BY namespace, service ORDER BY ...) + <= 21`. +3. Recompute affected hour groups from their minute lists and affected day + groups from their hour lists. + +Retention repair runs the same steps for the keys in `cache_retired_traces`. +A compaction covered by a cached replacement chain changes no candidate and +does no rank work. An uncovered one is treated as retention. Contributions, +rank rows and markers commit together, so a reader in one read transaction +under the pinned snapshot sees them consistently. + +### Query + +`CompletedTraceQuery` and its tests stay unchanged. `aggregateSources` and +`traceCandidateSource` bind `trace_candidates` as the union of: + +- rank rows of the day, hour and minute buckets covered by the window, + splitting it greedily into the largest covered buckets. A bucket is covered + when it starts at or after the window start and ends at least one minute + before the window end, because a short trace can extend up to a minute past + its bucket. Every short trace in a covered bucket is then fully inside the + window, as the index path requires (`min_start >= S AND max_start < E`); +- an exact candidate scan of the uncovered minutes at each edge; +- an exact scan of long traces (`max_start - min_start` of one minute or more) + inside the window, a set bounded by how many long traces exist; +- the existing partial-trace and uncached-tail sources. + +A limit above `TraceRankDepth`, or a bucket that query-time repair has marked +dirty, falls back to the exact candidate scan for those buckets. + +Rows read per request are about (day + hour + minute buckets) × groups × 21, +plus the edge minutes, the long traces and the tail. For a 24h window that is +at most about 2 + 46 + 118 buckets, independent of retention. + +### Remaining scan + +The `touched` stage still needs the cached rows of traces that also appear in +the uncached tail, and the boundary `partial` filter still reads the index. +Neither has an index today: a per-trace index ran out of memory, and +`views_test.go` asserts that the read tables have none. The first build +bounds both by pruning `read_trace_candidates` on `min_start` within the tail +and edge time ranges. Phase 2 measures whether that is enough. + +### Versioning + +The rank table changes read-cache schema and semantics, so `readCacheVersion` +goes from 19 to 20, and a mismatch rebuilds every private read table. + +## Phases + +1. **Harness, done.** The scale benchmark above, with full cache refresh and + a two-service toggle. The pitot-bench `--prefill` phase + (labstack/pitot-bench#14) runs measured reads on a populated server. +2. **Minute tier.** `read_trace_rank` with minute buckets only, maintenance + in `cacheBatches` and retention repair, and query binding. Gate: the + existing exactness suite (`trace_candidate_window_test.go`) passes + unchanged, and an added oracle test compares random windows, scopes and + limits against raw spans after late publication, compaction and retention. + The scale benchmark must show drill-down time growing no more than the + number of minutes in the window between 1.6M, 16M and 30M traces. +3. **Hour and day tiers.** Gate: a 24h window costs under 50 ms at 30M + traces in the scale benchmark, and refresh cost per batch rises by no more + than 50%. +4. **End-to-end gate.** pitot-bench on Hetzner with a full-day prefill (about + 4B rows; about 4 hours of prefill on a ccx33 pair), then 48k items/s with + 20 reads/s. The read gate must pass. A 10-minute prefill is not enough, + because it holds about 1/20 of a day's data. + +## Alternatives + +- **Rewriting the joins.** No gain once the index is fully cached. +- **Per-batch top-K.** Inexact, because a trace split across batches has a + merged duration that no batch's partial shows. Its cost also grows with the + number of batches. +- **Threshold algorithms (Fagin TA/NRA) over per-batch partials.** Unseen + batches can extend any trace, so the only available upper bounds are + window-wide and too loose to stop early. +- **Ordered index on the score.** Exact, but DuckDB indexes do not serve + ordered limit scans, and a per-trace index already ran out of memory. +- **Sampling, or Jaeger-style per-bucket `LIMIT N*k`.** Approximate, with no + bound on missing the true top K. + +## Risks + +- Maintenance cost. Measured on a simulated table: 7–12 ms per 40k-part + batch, against about 25 ms per batch for today's refresh. Recomputing only + touched buckets should be cheaper; phase 2 measures it. +- Long traces bypass the lists. A workload of mostly long traces degrades + toward today's cost. Phase 2 reports their share in the benchmark and on + Hetzner. +- Planner estimates. Bad cardinality estimates already reversed a hash build + in the exemplar path. Bound the new sources with explicit predicates and + check the plans in tests. +- Materialization. DuckDB 2 may factor repeated sources into materialized + subplans regardless of hints. Keep the merged rank source small enough that + materializing it is cheap. + +## Open questions + +- Should the one-minute short-trace threshold follow the bucket width or be + measured from real span-duration distributions? +- Is zonemap pruning on `min_start` enough for the tail join, or does the + tail need its own bounded structure? +- Rank depth: 21 fits both callers. Should the trace search page reuse the + lists at a larger depth? diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 385fac3f..5acc6020 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -38,5 +38,6 @@ Earlier proposals predate the template and keep their original structure. | 0004 | [Full UI without an LLM key, and PitotQL](0004-pitotql.md) | Accepted | | 0005 | [Dedicated attribute columns and batch format 5](0005-dedicated-attribute-columns.md) | Accepted | | 0006 | [Read performance under sustained ingest](0006-read-performance.md) | Draft | +| 0012 | [Bounded trace ranking for drill-down](0012-bounded-trace-ranking.md) | Draft | Settled decisions that a proposal leads to are recorded in [../adr](../adr/README.md). diff --git a/internal/engine/trace_candidate_scale_bench_test.go b/internal/engine/trace_candidate_scale_bench_test.go new file mode 100644 index 00000000..5c8b7276 --- /dev/null +++ b/internal/engine/trace_candidate_scale_bench_test.go @@ -0,0 +1,106 @@ +//go:build filebench + +package engine + +import ( + "fmt" + "os" + "sort" + "strconv" + "testing" + "time" + + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +// Time the completed-trace kernel as retained traces grow, with a raw tail of +// two uncached recent batches like live ingest. PITOT_SCALE_BATCHES sets the +// fully cached batches of 20,000 two-span traces each; PITOT_SCALE_MIXED=1 puts +// each trace's spans in two services, as real request traces do. +func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { + mixed := os.Getenv("PITOT_SCALE_MIXED") == "1" + batches := 20 + if v, err := strconv.Atoi(os.Getenv("PITOT_SCALE_BATCHES")); err == nil && v > 0 { + batches = v + } + const traces = 20000 + d, repo := newBatchCacheTest(t) + base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + commit := func(part int) { + batch := telemetrystore.Batch{ID: fmt.Sprintf("part_%d", part)} + for i := range traces { + id := uint64(part*traces+i+1) * 0x9e3779b97f4a7c15 + start := base.Add(time.Duration(part)*time.Second + time.Duration(i%1000)*time.Millisecond) + status := "STATUS_CODE_OK" + if i%97 == 0 { + status = "STATUS_CODE_ERROR" + } + for span := range 2 { + service := "api" + if mixed && span == 1 { + service = "db" + } + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "prod", ServiceName: service, + TraceID: fmt.Sprintf("%032x", id), SpanID: fmt.Sprintf("%016x", span+1), + StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(time.Duration(1+i%50) * time.Millisecond).UnixNano(), + DurationMS: float64(1 + i%50), StatusCode: status, IngestedAt: start.UnixNano()}) + } + } + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + } + for part := range batches { + commit(part) + } + // One pass stops at the row budget; repeat until every batch is cached. + for { + rows, err := d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) + } + if rows == 0 { + break + } + } + // Recent batches the read cache has not reached yet. + for part := batches; part < batches+2; part++ { + commit(part) + } + w := queryrows.Window{Start: base.Add(-time.Hour), End: base.Add(2 * time.Hour), Namespace: "prod", Service: "api", Kind: queryrows.TraceCandidateRead} + args := []any{w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, 21} + run := func(query string) []string { + rows, err := d.QueryContext(queryrows.WithWindow(t.Context(), w), query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []string + for rows.Next() { + var namespace, trace string + var hasError int + if err := rows.Scan(&namespace, &trace, &hasError); err != nil { + t.Fatal(err) + } + out = append(out, fmt.Sprint(namespace, trace, hasError)) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out + } + if got := run(queryrows.CompletedTraceQuery); len(got) != 21 { + t.Fatalf("completed rows=%d", len(got)) + } + const samples = 15 + var ms []float64 + for range samples { + started := time.Now() + run(queryrows.CompletedTraceQuery) + ms = append(ms, float64(time.Since(started))/float64(time.Millisecond)) + } + sort.Float64s(ms) + t.Logf("cached_traces=%d tail_traces=%d mixed_services=%v median_ms=%.1f max_ms=%.1f", batches*traces, 2*traces, mixed, ms[samples/2], ms[samples-1]) +} From 5c5cd41561ad8f45b10d80f08746db5851f8b23c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 09:27:41 -0800 Subject: [PATCH 2/6] docs(proposals): link 0012 to its pull request --- docs/proposals/0012-bounded-trace-ranking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposals/0012-bounded-trace-ranking.md b/docs/proposals/0012-bounded-trace-ranking.md index fac5fb36..af804cb7 100644 --- a/docs/proposals/0012-bounded-trace-ranking.md +++ b/docs/proposals/0012-bounded-trace-ranking.md @@ -2,7 +2,7 @@ Status: Draft Date: 2026-11-18 -Pull request: TBD +Pull request: #504 ## Summary From 0189a885245e93efe7b272092f9f2d03460064ab Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 11:14:50 -0800 Subject: [PATCH 3/6] docs: accept 0012 with best-span exemplar lists Rank drill-down traces by their best in-scope span and cache bounded per-batch, scope and minute exemplar lists with no cross-batch state. Update the read-cache rules in AGENTS.md to match. --- AGENTS.md | 11 +- docs/proposals/0012-bounded-trace-ranking.md | 331 +++++++++---------- docs/proposals/README.md | 2 +- 3 files changed, 158 insertions(+), 186 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bdc3f215..400dd63a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,11 +89,12 @@ remain immediately visible. Compaction and retention must invalidate retired contributions without double counting or losing late publications. - Version disposable read-cache schema and semantics together; mismatches rebuild - all private read tables. Cache per-batch trace bounds, - and one incremental candidate per trace, never individual event copies or - four globally rewritten scope indexes. Mixed trace scopes use batch parts. Exact - scoped boundaries read footer-pruned Parquet, and compaction derives complete - output contributions from cached inputs. Body search matches redacted text. + all private read tables. Notable traces rank by their best in-scope span + (error first, then longest). Cache bounded per-(batch, scope, minute) span + exemplar lists with no cross-batch trace state, never individual event copies + or globally rewritten scope indexes; uncached batches union in ranked the same + way. Windows within one minute read footer-pruned Parquet, and compaction + derives output lists from cached inputs. Body search matches redacted text. Analytical service/edge watermark lag is separate. Completed-batch writes have their own gate and write-pool slot; analytical writes use disjoint tables. Maintenance must hold both gates for checkpointing. diff --git a/docs/proposals/0012-bounded-trace-ranking.md b/docs/proposals/0012-bounded-trace-ranking.md index af804cb7..5fdaf685 100644 --- a/docs/proposals/0012-bounded-trace-ranking.md +++ b/docs/proposals/0012-bounded-trace-ranking.md @@ -1,230 +1,201 @@ # Bounded trace ranking for drill-down -Status: Draft +Status: Accepted Date: 2026-11-18 Pull request: #504 ## Summary Drill-down ranks every retained trace in its window to return the 21 slowest -or failing ones, so its cost grows with retained data and it times out on a -server that has run for a while. This proposal keeps a small, exactly -maintained ranked list of traces per time bucket and scope in the read cache. -A query merges the lists of the buckets inside its window and scans exactly -only what the lists cannot answer. Cost then depends on window length and list -size, not on how many traces are retained. +or failing ones. Its cost, and the read-cache maintenance behind it, grow with +retained data, so it times out on a server that has run for a while. This +proposal ranks each trace by its best in-scope span and caches a small, +append-only list of span exemplars per batch, scope and minute. Nothing in the +cache spans batches. A query reads the lists of the minutes in its window, +unions the uncached batches ranked the same way, and takes the top K. Cost +depends on window length and the number of services, not on retained traces. ## Motivation `queryrows.CompletedTraceQuery` serves service-map drill-down (`indexedExemplars` in `internal/query/exemplars.go`, limit 21) and the recent-trace lookup (`observability.Service.Trace`, limit 1). It orders every -in-window trace candidate by `has_error DESC, max_end-min_start DESC`. +in-window trace candidate by `has_error DESC, max_end-min_start DESC` from +`read_trace_candidates` and `read_trace_parts`. Evidence: - pitot-bench on a Hetzner ccx33 pair, `v2026.11.9-alpha`, 48k items/s mixed ingest with 20 UI reads/s. A fresh server (about 15M rows) passes the read gate at 20/20 ops/s on 3.25 cores. After a 10-minute prefill (about 180M - rows) the same load completes 2.6/20 ops/s on 7.3 cores. Every one of 1,716 - failures is a 504 on `/api/queries/exemplars`, and no drill-down or trace - detail succeeds. A local run on an M-series Mac reproduces it (4.8/20 ops/s). + rows) the same load completes 2.6/20 ops/s on 7.3 cores. All 1,716 failures + are 504s on `/api/queries/exemplars`, and no drill-down or trace detail + succeeds. A local run reproduces it (4.8/20 ops/s). - `internal/engine/trace_candidate_scale_bench_test.go` (build tag - `filebench`) times the query against a fully cached index with a two-batch - uncached tail: + `filebench`), fully cached: | Cached traces | Single-service traces | Two-service traces | |---|---|---| | 400k | 21 ms | 93 ms | | 1.6M | 197 ms | 633 ms | + | 16M (4 GB engine) | 831 ms | 2,426 ms | - Four times the traces costs seven to nine times the time. The Hetzner run - held roughly 30M traces; a day at 48k rows/s holds billions of rows, and - dashboard windows reach 24h. -- `EXPLAIN ANALYZE` at 1.6M two-service traces (0.59 s): every candidate has - `service=NULL`, so a service scope takes the repair path and aggregates every - retained `read_trace_parts` row (`HASH_JOIN RIGHT_SEMI` 207 ms with a 64,000 - row estimate against 1.6M actual, `HASH_GROUP_BY` 258 ms, a second 206 ms - group-by for the boundary subquery). DuckDB factors `trace_candidates` into a - materialized common subplan despite the `NOT MATERIALIZED` hint and spills - 275 MB. The top-N steps themselves cost about 4 ms. + With the test engine's 256 MB limit, the 16M two-service case runs out of + memory. +- `EXPLAIN ANALYZE` at 1.6M two-service traces: candidates of multi-service + traces have `service=NULL`, so a service scope aggregates every retained + `read_trace_parts` row, and DuckDB materializes and spills the candidate set + (275 MB). +- Maintenance grows too. The per-pass candidate MERGE joins every retained + trace: 27, 280 and 627 ms per pass at 3M, 30M and 100M traces. -Rewriting the joins does not help once the index is fully cached (603 ms -against 624 ms). The work is proportional to retained traces by construction. +The stored per-trace state is what fails to scale. Rewriting the query's joins +gave no gain once the index was fully cached. ## Goals and non-goals Goals: - Drill-down and recent-trace cost independent of retained traces, for every - scope (namespace and service set or empty) and window up to the retention - period. -- Exactly the current results, including ties (`trace_id`, then - `namespace`), late publication, compaction and retention. -- Bounded maintenance cost per cached batch. + scope (namespace and service, each optional) and windows up to retention. +- Read-cache maintenance cost per batch independent of retained traces. +- Exact results for the ranking defined below, including late publication, + compaction, retention and immediately visible uncached batches. Non-goals: -- Changing the ranking key or the definition of a trace's duration - (`max_end - min_start` over in-window, in-scope spans). -- Approximate ranking. Peers rank by root-span duration (SigNoz, Honeycomb, - Uptrace), return the first N matches (Tempo, Jaeger), or accept per-block - durations that understate split traces (Tempo). Pitot keeps its exact - definition. +- Approximate ranking. - Trace search with arbitrary attribute filters, which keeps the generic path. +## Ranking + +A trace's score in a scope and window is that of its best span among the spans +that match the scope and start in `[start, end)`: + +1. the span failed (`STATUS_CODE_ERROR`) first; +2. then longer span duration; +3. then `trace_id`, then `namespace`, ascending, as today. + +`has_error` is still true when the trace has any in-scope, in-window error +span, so the error-first selection is unchanged. + +This replaces the scoped extent (`max_end - min_start`). The extent is never +shown: the drill list displays service, operation and root duration +(`ui/host/src/dashboards/drill.tsx`), and `indexedExemplars` re-sorts the +selected traces by root status and duration. The generic exemplar path already +ranks by span `duration_ms`, as proposal 0004 documents, so both paths now +agree. Which traces fill the 20 slots can differ from today: a service with +several short disjoint spans in one trace no longer ranks by the time between +them, which included time spent in other services. Peers that rank traces +(Honeycomb, SigNoz, Uptrace) also rank by a span's duration. + +A best-span score decomposes: the best span over a union of partitions is the +best of each partition's best spans. That is what lets the cache hold no +cross-batch state. + ## Design -### Rank lists - -A new private read-cache table, `read_trace_rank`: - -| Column | Meaning | -|---|---| -| `tier` | minute, hour or day | -| `bucket` | bucket start, Unix nanoseconds | -| `namespace`, `service` | scope; `service` is `''` for the all-services list | -| `trace_id`, `min_start`, `max_start`, `max_end`, `has_error` | the trace's score in that scope | - -Each (tier, bucket, namespace, service) group holds the top -`queryrows.TraceRankDepth` (21) traces by the production ordering. A trace -belongs to the minute bucket of its `min_start`. Only short traces -(`max_start - min_start` under one minute) enter lists. - -A trace whose spans cover several services gets one row per service. Each -row's score is computed from that service's parts, because a service-scoped -query ranks only that service's spans. Rank rows store results only. The -"one incremental candidate per trace" rule still holds for -`read_trace_candidates`. - -### Why it is exact - -Each short trace lies in exactly one minute bucket per scope, so buckets -partition the traces. If a trace is in the global top K of a window, at most -K-1 traces outrank it anywhere, so at most K-1 outrank it in its own bucket. -It is therefore in its bucket's list, and the union of the lists of fully -covered buckets contains the global top K over those buckets. Hour and day -lists are the top K of their child lists, by the same argument. - -Scores come from merged candidates, never from per-batch parts. A newly -published span only lowers `min_start`, raises `max_end` or sets `has_error`, -so a trace's score never falls on publication. Updating a list is then an -insert with eviction of the lowest entry. Two events can lower a score or move -a trace between buckets: retention retiring part of a trace, and `min_start` -moving into an earlier bucket. Both rebuild the affected bucket from -`read_trace_candidates`, at a cost bounded by that bucket's traces. - -### Maintenance - -In the same `cacheBatches` transaction that inserts parts, merges candidates -and writes markers (`internal/engine/batch_cache.go`): - -1. Before the candidate MERGE, record the bucket keys of the candidates it - touches. After the MERGE, record their new keys. -2. Recompute those minute groups from `read_trace_candidates` and, for - per-service scores, from that bucket's `read_trace_parts`, using - `QUALIFY row_number() OVER (PARTITION BY namespace, service ORDER BY ...) - <= 21`. -3. Recompute affected hour groups from their minute lists and affected day - groups from their hour lists. - -Retention repair runs the same steps for the keys in `cache_retired_traces`. -A compaction covered by a cached replacement chain changes no candidate and -does no rank work. An uncovered one is treated as retention. Contributions, -rank rows and markers commit together, so a reader in one read transaction -under the pinned snapshot sees them consistently. +### Exemplar lists + +A private read-cache table, `read_trace_exemplars`, holds rows keyed by +(batch, namespace, service, minute of span start). Each partition keeps three +skyline sets of spans: + +- `full`: the top 21 distinct traces of the partition, each by its best span; +- `suffix`: every span that fewer than 21 other traces outrank using spans + that start at or after it; +- `prefix`: the mirror, using spans that start at or before it. + +For a window starting inside a partition's minute, its top 21 is contained in +the `suffix` rows with start at or after the window start. For a window ending +inside one, its top 21 is contained in the `prefix` rows that start before the +window end. List size grows with the logarithm of spans per partition: about +210 rows at 2.4k spans and 480 at 1M. ### Query -`CompletedTraceQuery` and its tests stay unchanged. `aggregateSources` and -`traceCandidateSource` bind `trace_candidates` as the union of: - -- rank rows of the day, hour and minute buckets covered by the window, - splitting it greedily into the largest covered buckets. A bucket is covered - when it starts at or after the window start and ends at least one minute - before the window end, because a short trace can extend up to a minute past - its bucket. Every short trace in a covered bucket is then fully inside the - window, as the index path requires (`min_start >= S AND max_start < E`); -- an exact candidate scan of the uncovered minutes at each edge; -- an exact scan of long traces (`max_start - min_start` of one minute or more) - inside the window, a set bounded by how many long traces exist; -- the existing partial-trace and uncached-tail sources. - -A limit above `TraceRankDepth`, or a bucket that query-time repair has marked -dirty, falls back to the exact candidate scan for those buckets. - -Rows read per request are about (day + hour + minute buckets) × groups × 21, -plus the edge minutes, the long traces and the tail. For a 24h window that is -at most about 2 + 46 + 118 buckets, independent of retention. - -### Remaining scan - -The `touched` stage still needs the cached rows of traces that also appear in -the uncached tail, and the boundary `partial` filter still reads the index. -Neither has an index today: a per-trace index ran out of memory, and -`views_test.go` asserts that the read tables have none. The first build -bounds both by pruning `read_trace_candidates` on `min_start` within the tail -and edge time ranges. Phase 2 measures whether that is enough. - -### Versioning - -The rank table changes read-cache schema and semantics, so `readCacheVersion` -goes from 19 to 20, and a mismatch rebuilds every private read table. - -## Phases - -1. **Harness, done.** The scale benchmark above, with full cache refresh and - a two-service toggle. The pitot-bench `--prefill` phase - (labstack/pitot-bench#14) runs measured reads on a populated server. -2. **Minute tier.** `read_trace_rank` with minute buckets only, maintenance - in `cacheBatches` and retention repair, and query binding. Gate: the - existing exactness suite (`trace_candidate_window_test.go`) passes - unchanged, and an added oracle test compares random windows, scopes and - limits against raw spans after late publication, compaction and retention. - The scale benchmark must show drill-down time growing no more than the - number of minutes in the window between 1.6M, 16M and 30M traces. -3. **Hour and day tiers.** Gate: a 24h window costs under 50 ms at 30M - traces in the scale benchmark, and refresh cost per batch rises by no more - than 50%. -4. **End-to-end gate.** pitot-bench on Hetzner with a full-day prefill (about - 4B rows; about 4 hours of prefill on a ccx33 pair), then 48k items/s with - 20 reads/s. The read gate must pass. A 10-minute prefill is not enough, - because it holds about 1/20 of a day's data. +A window `[S, E)` reads: + +- `full` rows of the minutes fully inside it; +- `suffix` rows of the minute containing `S`, with start at or after `S`; +- `prefix` rows of the minute containing `E`, with start before `E`; +- `UNION ALL` the raw in-scope spans of uncached batches, so new data is + visible at once. + +It then groups by trace, takes the best span, and returns the top K. The tail +is a union, not a join, so its cost is that of the uncached data only. A window +whose start and end fall in the same minute reads footer-pruned Parquet of the +overlapping files exactly, as the generic drill path already does. + +### Exactness + +If a trace's best in-window span lies in partition P and the trace is in the +global top K, then fewer than K traces outrank it anywhere in the window, so +fewer than K outrank it within P's part of the window. Its best span is +therefore in P's `full`, `suffix` or `prefix` set for that part. Every other +trace in those sets is also in the window, so no excluded trace can displace a +winner. An oracle test in the prototype compares against brute force over raw +spans on 300 random datasets × 20 windows, including traces split across +batches and minutes, and compaction. + +### Lifecycle + +- **Publication.** Lists are built from the batch's spans in the + completed-batch write, in the same transaction as the batch markers. +- **Compaction.** The output's lists are recomputed from the union of the + inputs' rows in the same transaction as the marker transfer. A superset of + the true lists keeps the query exact. An uncovered compaction builds from + Parquet like any batch. +- **Retention.** Rows are deleted by batch. + +Late spans need no update path: each is a row in its own batch's list, and the +query's maximum merges it. + +### Removed + +`read_trace_candidates`, `read_trace_parts`, the trace tail join, +`traceCandidateSource`, retired-trace repair and the partial-trace raw path. +The read-cache version goes from 19 to 20, so a mismatched cache rebuilds once +on upgrade. No telemetry is lost. ## Alternatives -- **Rewriting the joins.** No gain once the index is fully cached. -- **Per-batch top-K.** Inexact, because a trace split across batches has a - merged duration that no batch's partial shows. Its cost also grows with the - number of batches. -- **Threshold algorithms (Fagin TA/NRA) over per-batch partials.** Unseen - batches can extend any trace, so the only available upper bounds are - window-wide and too loose to stop early. -- **Ordered index on the score.** Exact, but DuckDB indexes do not serve - ordered limit scans, and a per-trace index already ran out of memory. -- **Sampling, or Jaeger-style per-bucket `LIMIT N*k`.** Approximate, with no - bound on missing the true top K. +- **Rank lists over merged per-trace candidates.** Exact for the extent key, + but they keep the per-trace candidate MERGE and tail join, which grow with + retention (about 1.4–5.8 s per pass at 700M traces). Bounding them needs a + merge horizon that splits traces with gaps into segments, which users would + see. +- **Go-side per-batch sidecars and an in-memory split-trace registry.** Exact + for the extent key, with microsecond queries in memory. But the registry + grows with retention, filters need about 2.25 GB of memory at 1B traces a + day, the on-disk form was not built, and the measurements were unstable. +- **Per-batch top-K of trace extents.** Inexact for traces split across + batches. +- **Threshold algorithms over per-batch partials.** Unseen batches can extend + any trace's extent, so the upper bounds are too loose to stop early. +- **Join rewrites.** No gain once the index is fully cached. + +## Gates + +1. An in-tree oracle against brute force over raw spans. It covers random + windows, namespace, service and empty scopes, K of 1 and 21, ties, + identical trace IDs in different namespaces, sub-minute and multi-minute + windows, late publication, compaction across generations, retention and an + uncached tail. +2. Scale benchmark: a 24h window stays flat in retained traces and under 50 ms + at 16M traces, for single- and two-service traces. +3. Maintenance: list build per batch and refresh pass time flat from 1.6M to + 16M traces. +4. `TraceSummaries` root lookup for 20 traces over 24h stays under 150 ms at a + day's batch count. If it does not, add a per-batch trace-hash filter. +5. pitot-bench on Hetzner with a full-day prefill, then 48k items/s with 20 + reads/s: the read gate passes. ## Risks -- Maintenance cost. Measured on a simulated table: 7–12 ms per 40k-part - batch, against about 25 ms per batch for today's refresh. Recomputing only - touched buckets should be cheaper; phase 2 measures it. -- Long traces bypass the lists. A workload of mostly long traces degrades - toward today's cost. Phase 2 reports their share in the benchmark and on - Hetzner. -- Planner estimates. Bad cardinality estimates already reversed a hash build - in the exemplar path. Bound the new sources with explicit predicates and - check the plans in tests. -- Materialization. DuckDB 2 may factor repeated sources into materialized - subplans regardless of hints. Keep the merged rank source small enough that - materializing it is cheap. - -## Open questions - -- Should the one-minute short-trace threshold follow the bucket width or be - measured from real span-duration distributions? -- Is zonemap pruning on `min_start` enough for the tail join, or does the - tail need its own bounded structure? -- Rank depth: 21 fits both callers. Should the trace search page reuse the - lists at a larger depth? +- **`TraceSummaries`** still opens each in-window batch's trace index. Gate 4 + measures it. +- **Sub-minute windows** on old, large compacted files decode whole columns + (estimated 0.1–0.3 s). Add a finer tier only if measurements need it. +- **Planner estimates.** Bound every source with constant time predicates and + check plans in tests. diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 5acc6020..f657fc37 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -38,6 +38,6 @@ Earlier proposals predate the template and keep their original structure. | 0004 | [Full UI without an LLM key, and PitotQL](0004-pitotql.md) | Accepted | | 0005 | [Dedicated attribute columns and batch format 5](0005-dedicated-attribute-columns.md) | Accepted | | 0006 | [Read performance under sustained ingest](0006-read-performance.md) | Draft | -| 0012 | [Bounded trace ranking for drill-down](0012-bounded-trace-ranking.md) | Draft | +| 0012 | [Bounded trace ranking for drill-down](0012-bounded-trace-ranking.md) | Accepted | Settled decisions that a proposal leads to are recorded in [../adr](../adr/README.md). From ea309f95821fcaba7707ddbbd7d093a71607d016 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 11:43:29 -0800 Subject: [PATCH 4/6] perf(engine): rank drill-down traces from per-minute exemplar skylines Service-map drill-down and the recent-trace lookup ranked every retained in-window trace candidate, and each read-cache pass merged new parts into that retained per-trace index, so both grew with retained traces. A trace now scores by its best in-scope, in-window span: an error first, then the longest span, then trace_id and namespace. The append-only read_trace_exemplars table keeps, per batch, namespace, service and minute of span start, the best span of the top 21 traces (full rows) and the spans fewer than 21 other traces outrank from either side (suffix and prefix rows). A window reads full rows of covered minutes, the partial start and end minutes' suffix and prefix rows, and the uncached files' raw spans, then takes each trace's best score. A window inside a single minute reads the overlapping files' raw spans. Rows are written with the batch markers, retention deletes them by batch, and compaction ranks the union of its inputs' rows again in the marker transfer. The candidate and part tables, the retired-trace repair and the partial-trace raw path are removed; the read cache version is now 20. --- internal/dashboard/resource_inventory_test.go | 2 +- internal/engine/batch_cache.go | 241 ++-------- .../engine/compaction_cache_transfer_test.go | 20 +- internal/engine/compaction_stall_test.go | 2 +- internal/engine/file_snapshot.go | 7 +- internal/engine/file_snapshot_bench_test.go | 46 +- internal/engine/file_snapshot_test.go | 84 ++-- internal/engine/panel_rollup_keys_test.go | 8 +- internal/engine/resource_inventory_test.go | 4 +- internal/engine/rollup_keys_test.go | 2 +- .../engine/snapshot_binding_bench_test.go | 2 +- internal/engine/sql_boundary_test.go | 6 +- internal/engine/startup_cold_bench_test.go | 14 +- internal/engine/startup_maintenance_test.go | 2 +- .../trace_candidate_window_bench_test.go | 186 -------- .../engine/trace_candidate_window_test.go | 25 +- internal/engine/trace_exemplars.go | 286 ++++++++++++ internal/engine/trace_exemplars_test.go | 420 ++++++++++++++++++ internal/engine/views_test.go | 16 +- internal/observability/batch_reads_test.go | 17 +- internal/query/exemplar_index_test.go | 5 +- internal/query/exemplars.go | 2 +- internal/query/log_redaction_test.go | 4 +- internal/queryrows/completed_traces.go | 16 + internal/queryrows/trace_candidates.go | 22 - 25 files changed, 864 insertions(+), 575 deletions(-) delete mode 100644 internal/engine/trace_candidate_window_bench_test.go create mode 100644 internal/engine/trace_exemplars.go create mode 100644 internal/engine/trace_exemplars_test.go create mode 100644 internal/queryrows/completed_traces.go delete mode 100644 internal/queryrows/trace_candidates.go diff --git a/internal/dashboard/resource_inventory_test.go b/internal/dashboard/resource_inventory_test.go index 9a3be543..5c8542e3 100644 --- a/internal/dashboard/resource_inventory_test.go +++ b/internal/dashboard/resource_inventory_test.go @@ -103,7 +103,7 @@ func TestDefaultResourcePanelsAcrossCacheStates(t *testing.T) { for _, state := range []string{"partial", "cold", "full"} { t.Run(state, func(t *testing.T) { if state == "cold" { - if _, err := d.DB.Exec(`DELETE FROM read_batches; DELETE FROM read_resource_inventory; DELETE FROM read_resource_seconds; DELETE FROM read_metric_names; DELETE FROM read_trace_parts; DELETE FROM read_trace_candidates`); err != nil { + if _, err := d.DB.Exec(`DELETE FROM read_batches; DELETE FROM read_resource_inventory; DELETE FROM read_resource_seconds; DELETE FROM read_metric_names; DELETE FROM read_trace_exemplars`); err != nil { t.Fatal(err) } if err := engine.CreateCacheTables(d.DB); err != nil { diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index e22b8234..e1a2b6dd 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -26,11 +26,11 @@ import ( // anomaly_log is not one of them: it holds detector output that nothing can // rebuild, so the drop keeps it. Its table is created once; any change to its // schema must be a separate, explicit migration step. -const readCacheVersion = 19 +const readCacheVersion = 20 const ( readCacheBatchLimit = 64 - // Amortize candidate maintenance and bounded pressure pauses without + // Amortize per-pass statements and bounded pressure pauses without // allowing a pass to grow with retained history. File count stays capped. readCacheRowBudget = 256000 ) @@ -93,11 +93,10 @@ func CreateCacheTables(db *sql.DB) error { createAnomalyLogTable, `CREATE TABLE IF NOT EXISTS read_edge_buckets (namespace VARCHAR, bucket TIMESTAMPTZ_NS, spans BIGINT)`, `CREATE TABLE IF NOT EXISTS read_metric_names (batch_id VARCHAR,name VARCHAR)`, - `CREATE TABLE IF NOT EXISTS read_batches (batch_id VARCHAR PRIMARY KEY,replacement_id VARCHAR)`, + `CREATE TABLE IF NOT EXISTS read_batches (batch_id VARCHAR PRIMARY KEY)`, `CREATE TABLE IF NOT EXISTS read_resource_seconds (batch_id VARCHAR,signal VARCHAR,namespace VARCHAR,service VARCHAR,resource VARIANT,bucket TIMESTAMP_NS,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS)`, `CREATE TABLE IF NOT EXISTS read_resource_inventory (batch_id VARCHAR,signal VARCHAR,namespace VARCHAR,service VARCHAR,resource VARIANT,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS)`, - `CREATE TABLE IF NOT EXISTS read_trace_candidates (trace_id VARCHAR,namespace VARCHAR,service VARCHAR,min_start BIGINT,max_start BIGINT,max_end BIGINT,has_error INTEGER)`, - `CREATE TABLE IF NOT EXISTS read_trace_parts (batch_id VARCHAR, namespace VARCHAR, service VARCHAR, trace_id VARCHAR, min_start BIGINT, max_start BIGINT, max_end BIGINT, has_error INTEGER)`, + createTraceExemplarsTable, } { if _, err := tx.Exec(stmt); err != nil { return err @@ -245,10 +244,6 @@ func (d *Duck) refreshReadCaches(ctx context.Context) (rows int64, backlog bool, return 0, false, nil } didWork = true - changed, err := changedRetiredBatches(ctx, d.writer(), markers, active, stale) - if err != nil { - return 0, false, err - } // Keep the bounded multi-file SQL kernel: per-file transactions spend // more time preparing statements than aggregating rows under steady ingest. // Yield between these cache batches, never before the first unit of work. @@ -258,12 +253,12 @@ func (d *Duck) refreshReadCaches(ctx context.Context) (rows int64, backlog bool, } } d.cachePassStarted = true - rows, err = d.cacheBatches(ctx, pending, stale, changed) + rows, err = d.cacheBatches(ctx, pending, stale) return rows, err == nil && selected < pendingCount, err } -func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetadata, stale, changed []string) (int64, error) { +func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetadata, stale []string) (int64, error) { tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err @@ -281,35 +276,14 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada } return err } - if len(changed) > 0 { - if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil { - return 0, err - } - } if len(stale) > 0 { - for _, table := range []string{"read_resource_inventory", "read_resource_seconds", "read_metric_names", "read_trace_parts", "read_batches"} { + for _, table := range []string{"read_resource_inventory", "read_resource_seconds", "read_metric_names", "read_trace_exemplars", "read_batches"} { if err := exec("DELETE FROM " + table + " WHERE batch_id IN (" + idsSQL(stale) + ")"); err != nil { return 0, err } } } - if len(changed) > 0 { - // Retention can change a trace's extrema or error flag. Compaction whose - // complete replacement is cached preserves them and bypasses this repair. - if err := exec(`DELETE FROM read_trace_candidates WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) AND (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM read_trace_parts)`); err != nil { - return 0, err - } - if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) GROUP BY namespace,trace_id) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id - WHEN MATCHED THEN UPDATE SET namespace=excluded.namespace,service=excluded.service,min_start=excluded.min_start,max_start=excluded.max_start,max_end=excluded.max_end,has_error=excluded.has_error - WHEN NOT MATCHED THEN INSERT ` + traceCandidateColumns); err != nil { - return 0, err - } - if err := exec(`DROP TABLE cache_retired_traces`); err != nil { - return 0, err - } - } - // Each aggregation scans only the pass's new files. Materialize trace - // parts once for both trace inserts without scanning retained parts. + // Each aggregation scans only the pass's new files. source := func(signal string) string { var selected []telemetry.BatchMetadata for _, b := range batches { @@ -353,26 +327,18 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada } } if spans := source("spans"); spans != "" { - if err := exec(`CREATE OR REPLACE TEMP TABLE cache_new_trace_parts AS SELECT batch_id,coalesce(namespace,'') AS namespace,coalesce(service,'') AS service,trace_id, - min(start_unix_nano) AS min_start,max(start_unix_nano) AS max_start,max(end_unix_nano) AS max_end,max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error - FROM (` + spans + `) WHERE trace_id<>'' GROUP BY 1,2,3,4`); err != nil { + // Rank each new partition's spans once; retained rows are never read. + if err := exec(`CREATE OR REPLACE TEMP TABLE cache_new_exemplar_spans AS SELECT row_number() OVER () AS id,batch_id,coalesce(namespace,'') AS namespace,coalesce(service,'') AS service, + ` + minuteSQL("start_unix_nano") + ` AS minute,trace_id,start_unix_nano,end_unix_nano-start_unix_nano AS duration,CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END AS has_error + FROM (` + spans + `) WHERE trace_id<>''`); err != nil { return 0, err } - if err := exec(`INSERT INTO read_trace_parts SELECT * FROM cache_new_trace_parts`); err != nil { - return 0, err - } - candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY namespace,trace_id` - if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id - WHEN MATCHED THEN UPDATE SET - service=CASE WHEN c.service=excluded.service THEN excluded.service END, - min_start=least(c.min_start,excluded.min_start), - max_start=greatest(c.max_start,excluded.max_start), - max_end=greatest(c.max_end,excluded.max_end), - has_error=greatest(c.has_error,excluded.has_error) - WHEN NOT MATCHED THEN INSERT ` + traceCandidateColumns); err != nil { + n, err := insertTraceExemplars(ctx, tx, "cache_new_exemplar_spans") + if err != nil { return 0, err } - if err := exec(`DROP TABLE cache_new_trace_parts`); err != nil { + written += n + if err := exec(`DROP TABLE cache_new_exemplar_spans`); err != nil { return 0, err } } @@ -427,60 +393,7 @@ func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []t if w.Kind == queryrows.RawRead { return nil } - var cachedIDs []string - var cached, uncached []telemetry.BatchMetadata - for _, b := range batches { - if !overlapping(b, "spans", w) { - continue - } - if markers[b.ID] { - cachedIDs = append(cachedIDs, b.ID) - cached = append(cached, b) - } else { - uncached = append(uncached, b) - } - } - // Use the incremental candidate index, repairing changed retired traces - // and ambiguous scopes from active batch parts. Compaction keeps it usable. - scope := "batch_id IN (" + idsSQL(cachedIDs) + ")" - if w.Namespace != "" { - scope += " AND namespace=" + sqlLiteral(w.Namespace) - } - if w.Service != "" { - scope += " AND service=" + sqlLiteral(w.Service) - } - index, err := d.traceCandidateSource(ctx, db, batches, markers, w) - if err != nil { - return err - } - full := fmt.Sprintf("min_start>=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano()) - partial := "SELECT * FROM (" + index + ") WHERE NOT (" + full + ") AND " + fmt.Sprintf("max_start>=%d AND min_start<%d", w.Start.UnixNano(), w.End.UnixNano()) - projection := "namespace,service,trace_id,start_time,start_unix_nano,end_unix_nano,status" - tail := "SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", uncached, batches)) + ") WHERE " + windowPredicate("start_time", w) - // Only files whose scoped trace parts straddle a boundary require raw rows. - var partialFiles []telemetry.BatchMetadata - for _, b := range cached { - bounds, _ := batchTime(b, "spans") - if !bounds.Known || bounds.MinNanos < w.Start.UnixNano() || bounds.MaxNanos >= w.End.UnixNano() { - partialFiles = append(partialFiles, b) - } - } - interior := fmt.Sprintf("min_start>=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano()) - if len(partialFiles) > 0 { - tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (" + partial + "))" - // A partial trace may also have in-window contributions in other batches. - // Include these parts as candidates; boundary rows merge them in the kernel. - sources["trace_candidates"] = `SELECT namespace,trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY namespace,trace_id` - } else { - // Every overlapping cached file lies inside the window, so a partial - // trace's other spans are in files outside it. Its in-window parts are - // still a candidate; the kernel needs one row per trace, and partial - // traces are disjoint from the fully contained index rows. - sources["trace_candidates"] = "SELECT namespace,trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + - ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + - ` AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (` + partial + `)) GROUP BY namespace,trace_id` - } - sources["trace_tail"] = tail + sources["trace_exemplars"] = d.traceExemplarSource(batches, markers, w) return nil } @@ -579,126 +492,22 @@ func (d *Duck) transferBatchCacheLocked(ctx context.Context, output string, inpu `INSERT INTO read_resource_inventory SELECT ` + sqlLiteral(output) + `,signal,namespace,service,resource,min(first_seen),max(last_seen) FROM read_resource_inventory` + where + ` GROUP BY signal,namespace,service,resource`, `INSERT INTO read_resource_seconds SELECT ` + sqlLiteral(output) + `,signal,namespace,service,resource,bucket,min(first_seen),max(last_seen) FROM read_resource_seconds` + where + ` GROUP BY signal,namespace,service,resource,bucket`, `INSERT INTO read_metric_names SELECT DISTINCT ` + sqlLiteral(output) + `,name FROM read_metric_names` + where, - `INSERT INTO read_trace_parts SELECT ` + sqlLiteral(output) + `,namespace,service,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts` + where + ` GROUP BY 2,3,4`, + // The inputs' rows hold every span the output's partitions keep, so + // ranking their union again derives the output's rows exactly. + `CREATE OR REPLACE TEMP TABLE cache_transfer_exemplar_spans AS SELECT row_number() OVER () AS id,` + sqlLiteral(output) + ` AS batch_id,namespace,service,minute,trace_id,start_unix_nano,duration,has_error FROM read_trace_exemplars` + where, `INSERT INTO read_batches(batch_id) VALUES (` + sqlLiteral(output) + `)`, - `UPDATE read_batches SET replacement_id=` + sqlLiteral(output) + where, } { if _, err := tx.ExecContext(ctx, stmt); err != nil { return err } } - return tx.Commit() -} - -// MERGE joins against the candidates instead of probing a key: an index on -// every retained trace is loaded whole to upsert or checkpoint, and at a few -// million traces that alone outgrew a 1 GB memory limit. -const traceCandidateColumns = `VALUES (excluded.trace_id,excluded.namespace,excluded.service,excluded.min_start,excluded.max_start,excluded.max_end,excluded.has_error)` - -// One candidate per namespace and trace. Mixed services are NULL and use exact -// batch parts for a service selection. -const traceCandidateSelect = `SELECT trace_id, - namespace, - CASE WHEN min(service)=max(service) THEN min(service) END AS service, - min(min_start) AS min_start,max(max_start) AS max_start,max(max_end) AS max_end,max(has_error) AS has_error` - -// Only retention or an uncached replacement changes the active cached trace -// bounds. Follow complete compaction replacements, including multiple levels. -func changedRetiredBatches(ctx context.Context, db snapshotSQL, markers, active map[string]bool, stale []string) ([]string, error) { - if len(stale) == 0 { - return nil, nil - } - rows, err := db.QueryContext(ctx, `SELECT batch_id,replacement_id FROM read_batches WHERE replacement_id IS NOT NULL`) - if err != nil { - return nil, err - } - defer rows.Close() - replacements := map[string]string{} - for rows.Next() { - var id, to string - if err := rows.Scan(&id, &to); err != nil { - return nil, err - } - replacements[id] = to - } - if err := rows.Err(); err != nil { - return nil, err - } - var changed []string - for _, id := range stale { - covered := false - visited := map[string]bool{} - for to := replacements[id]; to != "" && !visited[to]; to = replacements[to] { - visited[to] = true - if active[to] && markers[to] { - covered = true - break - } - } - if !covered { - changed = append(changed, id) - } - } - return changed, nil -} - -func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches []telemetry.BatchMetadata, markers map[string]bool, w queryrows.Window) (string, error) { - active := map[string]bool{} - var cachedIDs, stale []string - for _, b := range batches { - active[b.ID] = true - if markers[b.ID] { - cachedIDs = append(cachedIDs, b.ID) - } - } - for id := range markers { - if !active[id] { - d.cachePublishMu.Lock() - publishing := d.cachePublishing[id] - d.cachePublishMu.Unlock() - if !publishing { - stale = append(stale, id) - } - } - } - changed, err := changedRetiredBatches(ctx, db, markers, active, stale) - if err != nil { - return "", err + if _, err := insertTraceExemplars(ctx, tx, "cache_transfer_exemplar_spans"); err != nil { + return err } - index := `SELECT * FROM read_trace_candidates` - if len(changed) > 0 { - dirty := `SELECT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)` - index += ` WHERE (namespace,trace_id) NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND (namespace,trace_id) IN (` + dirty + `) GROUP BY namespace,trace_id` + if _, err := tx.ExecContext(ctx, `DROP TABLE cache_transfer_exemplar_spans`); err != nil { + return err } - var exact, possible, ambiguous []string - for _, field := range []struct{ name, value string }{{"namespace", w.Namespace}, {"service", w.Service}} { - if field.value == "" { - continue - } - match := field.name + "=" + sqlLiteral(field.value) - exact = append(exact, match) - possible = append(possible, "("+match+" OR "+field.name+" IS NULL)") - ambiguous = append(ambiguous, field.name+" IS NULL") - } - projection := `namespace,trace_id,min_start,max_start,max_end,has_error` - if len(exact) == 0 { - return "SELECT " + projection + " FROM (" + index + ")", nil - } - result := "SELECT " + projection + " FROM (" + index + ") WHERE " + strings.Join(exact, " AND ") - mixed := "SELECT namespace,trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")" - // Most traces belong to a single namespace/service. Do not put a retained - // batch-parts aggregation in the ranking/join plan when this snapshot has no - // ambiguous candidates for the requested scope. The probe shares its read - // transaction and the same retired-contribution repair. - var hasMixed bool - if err := db.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM ("+mixed+") LIMIT 1)").Scan(&hasMixed); err != nil { - return "", err - } - if !hasMixed { - return result, nil - } - scope := "batch_id IN (" + idsSQL(cachedIDs) + ") AND " + strings.Join(exact, " AND ") - return result + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND (namespace,trace_id) IN (` + mixed + `) GROUP BY namespace,trace_id`, nil + return tx.Commit() } func minTime(a, b time.Time) time.Time { diff --git a/internal/engine/compaction_cache_transfer_test.go b/internal/engine/compaction_cache_transfer_test.go index cd4b2e88..9b4ebc21 100644 --- a/internal/engine/compaction_cache_transfer_test.go +++ b/internal/engine/compaction_cache_transfer_test.go @@ -42,12 +42,12 @@ func TestCompactionDefersWarmCacheTransfer(t *testing.T) { if n != 0 { t.Fatalf("rebuilt %d contributions from already cached inputs", n) } - var parts, candidates, markers int - if err := d.DB.QueryRow(`SELECT (SELECT count(*) FROM read_trace_parts),(SELECT count(*) FROM read_trace_candidates),(SELECT count(*) FROM read_batches)`).Scan(&parts, &candidates, &markers); err != nil { + var exemplars, owners, markers int + if err := d.DB.QueryRow(exemplarOwnersQuery).Scan(&exemplars, &owners, &markers); err != nil { t.Fatal(err) } - if parts != 24 || candidates != 24 || markers != 3 { - t.Fatalf("parts=%d candidates=%d markers=%d", parts, candidates, markers) + if exemplars != 24 || owners != 3 || markers != 3 { + t.Fatalf("exemplars=%d owners=%d markers=%d", exemplars, owners, markers) } }) } @@ -80,11 +80,15 @@ func TestDeferredCacheTransferFollowsCompactionGenerations(t *testing.T) { if n != 0 { t.Fatalf("rebuilt %d contributions across warm generations", n) } - var parts, candidates, markers int - if err := d.DB.QueryRow(`SELECT (SELECT count(*) FROM read_trace_parts),(SELECT count(*) FROM read_trace_candidates),(SELECT count(*) FROM read_batches)`).Scan(&parts, &candidates, &markers); err != nil { + var exemplars, owners, markers int + if err := d.DB.QueryRow(exemplarOwnersQuery).Scan(&exemplars, &owners, &markers); err != nil { t.Fatal(err) } - if parts != 64 || candidates != 64 || markers != 1 { - t.Fatalf("parts=%d candidates=%d markers=%d", parts, candidates, markers) + if exemplars == 0 || owners != 1 || markers != 1 { + t.Fatalf("exemplars=%d owners=%d markers=%d", exemplars, owners, markers) } } + +// Exemplar rows, the batches owning them, and the cached batches: transferred +// rows belong only to the active output. +const exemplarOwnersQuery = `SELECT (SELECT count(*) FROM read_trace_exemplars),(SELECT count(DISTINCT batch_id) FROM read_trace_exemplars),(SELECT count(*) FROM read_batches)` diff --git a/internal/engine/compaction_stall_test.go b/internal/engine/compaction_stall_test.go index e38e98af..2231aa7f 100644 --- a/internal/engine/compaction_stall_test.go +++ b/internal/engine/compaction_stall_test.go @@ -83,7 +83,7 @@ func TestCompactionPublishesDuringBackgroundReads(t *testing.T) { t.Fatalf("rebuilt output markers=%d, want 3: %v", markers, err) } var parts int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts`).Scan(&parts); err != nil || parts != 24 { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars`).Scan(&parts); err != nil || parts != 24 { t.Fatalf("rebuilt trace contributions=%d, want 24: %v", parts, err) } d.unpinParquet(pin) diff --git a/internal/engine/file_snapshot.go b/internal/engine/file_snapshot.go index c83c9fb3..dec9a64a 100644 --- a/internal/engine/file_snapshot.go +++ b/internal/engine/file_snapshot.go @@ -436,7 +436,7 @@ func (d *Duck) compileSnapshot(ctx context.Context, db snapshotSQL, query string } d.snapshotMisses.Add(1) aliases := map[string]string{} - names := []string{"trace_candidates", "trace_tail", "resource_inventory", "receiver_rollup", "receiver_points", "resource_rollup", "receiver_entity_rollup"} + names := []string{"trace_exemplars", "resource_inventory", "receiver_rollup", "receiver_points", "resource_rollup", "receiver_entity_rollup"} for _, s := range telemetry.Signals { names = append(names, "telemetry."+s.Name, s.Name) } @@ -470,11 +470,10 @@ func bindCompiledSnapshot(compiled compiledSnapshot, sources map[string]string) return "", fmt.Errorf("missing snapshot relation %s", name) } materialization := " AS (" - if name == "trace_candidates" || name == "receiver_rollup" || name == "receiver_points" || name == "resource_rollup" || name == "receiver_entity_rollup" { + if name == "trace_exemplars" || name == "receiver_rollup" || name == "receiver_points" || name == "resource_rollup" || name == "receiver_entity_rollup" { // These sources can have several consumers. Inline them so their // time/key predicates reach scans instead of copying every retained - // candidate or receiver payload into a temporary relation. - // Mixed-scope candidate aggregates keep their required barriers. + // exemplar or receiver payload into a temporary relation. materialization = " AS NOT MATERIALIZED (" } definitions = append(definitions, alias+materialization+body+")") diff --git a/internal/engine/file_snapshot_bench_test.go b/internal/engine/file_snapshot_bench_test.go index 453fd77e..b935b251 100644 --- a/internal/engine/file_snapshot_bench_test.go +++ b/internal/engine/file_snapshot_bench_test.go @@ -74,9 +74,10 @@ func TestSnapshotFileBindingBenchmark(t *testing.T) { } } -// Synthetic retained-index experiment: append five disjoint 1,000-span batches -// without timing fixture creation. Existing parts/candidates model one trace per -// two retained spans. There is no stale cleanup or compaction in this interval. +// Synthetic retained-history experiment: append five disjoint 1,000-span +// batches without timing fixture creation. Existing exemplar rows model the +// retained history; a pass must not read them, so its cost stays flat. There is +// no stale cleanup or compaction in this interval. func TestReadCacheAppendScalingBenchmark(t *testing.T) { for _, retained := range []int{200000, 800000, 2000000, 8000000} { t.Run(fmt.Sprint(retained), func(t *testing.T) { @@ -100,52 +101,21 @@ func TestReadCacheAppendScalingBenchmark(t *testing.T) { t.Fatal(err) } defer d.Close() - parts := fmt.Sprintf(`INSERT INTO read_trace_parts SELECT 'retained','prod','api',md5(i::VARCHAR),1::BIGINT,1::BIGINT,2::BIGINT,0 FROM range(%d) t(i)`, retained/2) - if _, err := d.DB.Exec(parts); err != nil { - t.Fatal(err) - } - if _, err := d.DB.Exec(`INSERT INTO read_trace_candidates SELECT trace_id,namespace,service,min_start,max_start,max_end,has_error FROM read_trace_parts`); err != nil { + rows := fmt.Sprintf(`INSERT INTO read_trace_exemplars SELECT 'retained','prod','api',(i//1000)*60000000000,md5(i::VARCHAR),(i//1000)*60000000000+i%%1000,1,0,i%%50=0,true,true FROM range(%d) t(i)`, retained) + if _, err := d.DB.Exec(rows); err != nil { t.Fatal(err) } var timings []float64 for _, b := range repo.Parquet.BatchMetadata() { started := time.Now() - if _, err := d.cacheBatches(t.Context(), []telemetry.BatchMetadata{b}, nil, nil); err != nil { + if _, err := d.cacheBatches(t.Context(), []telemetry.BatchMetadata{b}, nil); err != nil { t.Fatal(err) } timings = append(timings, float64(time.Since(started))/float64(time.Millisecond)) } sort.Float64s(timings) - data, _ := json.Marshal(map[string]any{"retained_spans": retained, "retained_trace_rows": retained / 2, "added_spans_per_pass": 1000, "passes": 5, "median_ms": timings[2], "p95_ms": timings[4]}) + data, _ := json.Marshal(map[string]any{"retained_exemplar_rows": retained, "added_spans_per_pass": 1000, "passes": 5, "median_ms": timings[2], "p95_ms": timings[4]}) t.Log("RESULT", string(data)) - for _, hint := range []string{"MATERIALIZED", "NOT MATERIALIZED"} { - stmt := `WITH candidates AS ` + hint + ` (SELECT trace_id,min_start,max_end,has_error FROM read_trace_candidates), - tail AS (SELECT 'new0-0'::VARCHAR AS trace_id,1::BIGINT AS min_start,2::BIGINT AS max_end,0::INTEGER AS has_error), - untouched AS (SELECT * FROM candidates WHERE trace_id NOT IN (SELECT trace_id FROM tail) ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1), - touched AS (SELECT t.trace_id,least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end,greatest(t.has_error,c.has_error) AS has_error FROM tail t LEFT JOIN candidates c ON t.trace_id=c.trace_id), - choices AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) - SELECT trace_id FROM choices ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1` - var id string - if err := d.DB.QueryRow(stmt).Scan(&id); err != nil { - t.Fatal(err) - } - timings := []float64{} - for range 5 { - started := time.Now() - if err := d.DB.QueryRow(stmt).Scan(&id); err != nil { - t.Fatal(err) - } - timings = append(timings, float64(time.Since(started))/float64(time.Millisecond)) - } - sort.Float64s(timings) - data, _ := json.Marshal(map[string]any{"experiment": "trace_candidate_materialization", "retained_spans": retained, "hint": hint, "median_ms": timings[2], "p95_ms": timings[4]}) - t.Log("RESULT", string(data)) - var key, plan string - if err := d.DB.QueryRow("EXPLAIN (FORMAT JSON) "+stmt).Scan(&key, &plan); err != nil { - t.Fatal(err) - } - t.Logf("PLAN retained_spans=%d hint=%s %s", retained, hint, plan) - } }) } } diff --git a/internal/engine/file_snapshot_test.go b/internal/engine/file_snapshot_test.go index 515dceed..fa9a191a 100644 --- a/internal/engine/file_snapshot_test.go +++ b/internal/engine/file_snapshot_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -180,7 +181,7 @@ func TestCompletedBatchCachesIncludeLatePublicationAndPartialMinutes(t *testing. w := queryrows.Window{Start: at.Add(20 * time.Second), End: at.Add(140 * time.Second), Namespace: "prod", Service: "api"} w.Kind = queryrows.TraceCandidateRead var count int - q := `SELECT count(DISTINCT trace_id) FROM (SELECT trace_id FROM trace_candidates UNION ALL SELECT trace_id FROM trace_tail)` + q := `SELECT count(DISTINCT trace_id) FROM trace_exemplars` if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, q); err != nil || count != 5 { t.Fatalf("partial/late trace candidates=%d: %v", count, err) } @@ -193,7 +194,7 @@ func TestCompletedBatchCachesIncludeLatePublicationAndPartialMinutes(t *testing. if err := d.DB.QueryRow(`SELECT count(*) FROM read_batches`).Scan(&markers); err != nil { t.Fatal(err) } - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts`).Scan(&parts); err != nil { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars`).Scan(&parts); err != nil { t.Fatal(err) } if markers != 2 || parts != 8 { @@ -239,11 +240,12 @@ func TestBatchCachesStayExactThroughCompactionAndRetention(t *testing.T) { w.Kind = queryrows.TraceCandidateRead w.Namespace = "prod" w.Service = "api" - if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT (SELECT count(*) FROM trace_candidates)+(SELECT count(*) FROM trace_tail)`); err != nil { + if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT count(*) FROM trace_exemplars`); err != nil { t.Fatal(err) } // A replacement derived from complete inputs is immediately cached even - // while the retired input markers remain. It has one candidate and no tail. + // while the retired input markers remain: the covered minute holds the + // trace's one best span and nothing reads the inputs or a raw tail. if count != 1 { t.Fatalf("candidate coverage=%d", count) } @@ -265,13 +267,13 @@ func TestBatchCachesStayExactThroughCompactionAndRetention(t *testing.T) { } w := queryrows.Window{Start: at.Add(-time.Minute), End: at.Add(time.Minute), Kind: queryrows.TraceCandidateRead, Namespace: "prod", Service: "api"} var count int - if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT (SELECT count(*) FROM trace_candidates)+(SELECT count(*) FROM trace_tail)`); err != nil || count != 0 { + if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT count(*) FROM trace_exemplars`); err != nil || count != 0 { t.Fatalf("retired trace visible: %d: %v", count, err) } if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts`).Scan(&count); err != nil || count != 0 { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars`).Scan(&count); err != nil || count != 0 { t.Fatalf("retired index retained: %d %v", count, err) } } @@ -285,7 +287,7 @@ func TestReadCacheCancellationLeavesNoMarkerOrPartialContributions(t *testing.T) if _, err := d.RefreshReadCaches(ctx); err == nil { t.Fatal("cancelled backfill succeeded") } - for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_exemplars"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { t.Fatalf("%s partial rows=%d: %v", table, count, err) @@ -302,13 +304,13 @@ func TestFailedBatchCacheTransactionPublishesNoContributions(t *testing.T) { t.Fatal(err) } // Fail after the span writes, before acknowledgement. - if _, err := d.DB.Exec(`DROP TABLE read_trace_candidates`); err != nil { + if _, err := d.DB.Exec(`DROP TABLE read_trace_exemplars`); err != nil { t.Fatal(err) } if _, err := d.RefreshReadCaches(t.Context()); err == nil { t.Fatal("broken cache schema accepted") } - for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_resource_seconds"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { t.Fatalf("%s partial rows=%d: %v", table, count, err) @@ -405,7 +407,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - for _, stmt := range []string{`UPDATE read_trace_candidates SET min_start=0`, `UPDATE read_resource_inventory SET first_seen=make_timestamp_ns(0)::TIMESTAMPTZ_NS`, `UPDATE read_cache_version SET version=5`, `INSERT INTO read_edge_buckets VALUES ('ns',now(),10)`, `INSERT INTO rollup_state VALUES ('edge_rollup_v2_substart',1,now()),('edge_rollup_v2_subwindow_end',2,now())`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, + for _, stmt := range []string{`UPDATE read_trace_exemplars SET start_unix_nano=0`, `UPDATE read_resource_inventory SET first_seen=make_timestamp_ns(0)::TIMESTAMPTZ_NS`, `UPDATE read_cache_version SET version=5`, `INSERT INTO read_edge_buckets VALUES ('ns',now(),10)`, `INSERT INTO rollup_state VALUES ('edge_rollup_v2_substart',1,now()),('edge_rollup_v2_subwindow_end',2,now())`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, `INSERT INTO service_rollup (namespace,bucket,service,spans) VALUES ('ns',now(),'svc',1)`, `INSERT INTO edge_rollup (namespace,bucket,caller,callee,calls) VALUES ('ns',now(),'a','b',1)`, `INSERT INTO version_rollup VALUES ('ns','svc','v1',now(),now())`, `INSERT INTO version_rollup_batches VALUES ('semantic',1)`, `INSERT INTO anomaly_log VALUES ('ns','svc','latency',now(),now(),'slow','bad')`, `INSERT INTO rollup_state VALUES ('service_rollup_v3',1,now())`} { @@ -416,7 +418,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { if err := CreateCacheTables(d.DB); err != nil { t.Fatal(err) } - for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates", "read_edge_buckets", + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_exemplars", "read_edge_buckets", "service_rollup", "edge_rollup", "rollup_state", "version_rollup", "version_rollup_batches"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { @@ -443,7 +445,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { t.Fatal(err) } var minStart int64 - if err := d.DB.QueryRow(`SELECT min_start FROM read_trace_candidates`).Scan(&minStart); err != nil || minStart != at.UnixNano() { + if err := d.DB.QueryRow(`SELECT start_unix_nano FROM read_trace_exemplars`).Scan(&minStart); err != nil || minStart != at.UnixNano() { t.Fatalf("rebuilt bound=%d: %v", minStart, err) } var version int @@ -530,7 +532,7 @@ func TestBatchFilenameIdentityWithRelativeDataDir(t *testing.T) { t.Fatal(err) } var count int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts WHERE batch_id IN (SELECT batch_id FROM read_batches)`).Scan(&count); err != nil || count != readCacheBatchLimit { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE batch_id IN (SELECT batch_id FROM read_batches)`).Scan(&count); err != nil || count != readCacheBatchLimit { t.Fatalf("relative cache batch identity=%d: %v", count, err) } active := repo.Parquet.BatchMetadata() @@ -602,7 +604,9 @@ func TestPreparedReplacementDoesNotHoldCacheWriteGate(t *testing.T) { } } -func TestRetiredTraceRepairsOnlyItsActiveScopeAndBounds(t *testing.T) { +// Retention removes a retired batch's spans from a trace's score: the shared +// trace loses its retired error span and ranks below the remaining winner. +func TestRetentionRemovesRetiredSpansFromTraceRanking(t *testing.T) { d, repo := newBatchCacheTest(t) at := time.Now().UTC().Truncate(time.Hour) old := at.Add(-2 * time.Hour).UnixNano() @@ -626,9 +630,10 @@ func TestRetiredTraceRepairsOnlyItsActiveScopeAndBounds(t *testing.T) { t.Helper() for _, ns := range []string{"", "prod"} { w := queryrows.Window{Start: at.Add(-3 * time.Hour), End: at.Add(time.Minute), Kind: queryrows.TraceCandidateRead, Namespace: ns} - var id string - if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&id}, `SELECT trace_id FROM trace_candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1`); err != nil || id != "winner" { - t.Fatalf("namespace %q winner=%q: %v", ns, id, err) + got := completedTraces(t, d, w, traceExemplarDepth) + want := []oracleRow{{"prod", "winner", 0}, {"prod", "shared", 0}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("namespace %q ranking=%v", ns, got) } } } @@ -637,10 +642,9 @@ func TestRetiredTraceRepairsOnlyItsActiveScopeAndBounds(t *testing.T) { t.Fatal(err) } check() - var namespace, service string - var hasError int - if err := d.DB.QueryRow(`SELECT namespace,service,has_error FROM read_trace_candidates WHERE trace_id='shared'`).Scan(&namespace, &service, &hasError); err != nil || namespace != "prod" || service != "api" || hasError != 0 { - t.Fatalf("remaining scope=%q/%q error=%d: %v", namespace, service, hasError, err) + var retired int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE namespace='gone'`).Scan(&retired); err != nil || retired != 0 { + t.Fatalf("retired exemplars=%d: %v", retired, err) } } @@ -669,7 +673,7 @@ func TestTraceCacheFollowsMultipleCompactionLevels(t *testing.T) { w := queryrows.Window{Start: at.Add(-time.Minute), End: at.Add(2 * time.Minute), Kind: queryrows.TraceCandidateRead} assertInventoryOracle(t, d, w, 1) var count int - if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT count(*) FROM trace_candidates`); err != nil || count != 1 { + if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT count(DISTINCT trace_id) FROM trace_exemplars`); err != nil || count != 1 { t.Fatalf("candidate=%d: %v", count, err) } if _, err := d.RefreshReadCaches(t.Context()); err != nil { @@ -678,8 +682,8 @@ func TestTraceCacheFollowsMultipleCompactionLevels(t *testing.T) { if err := d.DB.QueryRow(`SELECT count(*) FROM read_batches`).Scan(&count); err != nil || count != 1 { t.Fatalf("retired markers=%d: %v", count, err) } - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_candidates`).Scan(&count); err != nil || count != 1 { - t.Fatalf("trace index=%d: %v", count, err) + if err := d.DB.QueryRow(`SELECT count(DISTINCT batch_id) FROM read_trace_exemplars`).Scan(&count); err != nil || count != 1 { + t.Fatalf("exemplar owners=%d: %v", count, err) } } @@ -739,7 +743,7 @@ func TestReadCacheRowBudgetAndOversizedFileProgress(t *testing.T) { t.Fatalf("oversized file made no progress: %v %v", markers, err) } var count int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts WHERE batch_id='oversized'`).Scan(&count); err != nil || count != len(old) { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE batch_id='oversized'`).Scan(&count); err != nil || count == 0 { t.Fatalf("oversized count=%d: %v", count, err) } } @@ -775,7 +779,7 @@ func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) { t.Fatal(err) } markers, err = batchMarkers(t.Context(), d.DB) - if err != nil || !markers["log-heavy-trace"] || markers["older-trace"] || written != 6 { + if err != nil || !markers["log-heavy-trace"] || markers["older-trace"] || written != 5 { t.Fatalf("oversized mixed batch: %v written=%d err=%v", markers, written, err) } if _, err := d.RefreshReadCaches(t.Context()); err != nil { @@ -787,7 +791,9 @@ func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) { } } -func TestRetirementRepairsOnlyMatchingNamespaceIdentity(t *testing.T) { +// The same trace_id in two namespaces ranks as two traces, and retiring one +// namespace's error span changes only that identity. +func TestRetentionRanksTraceIdentityPerNamespace(t *testing.T) { d, repo := newBatchCacheTest(t) at := time.Now().UTC().Truncate(time.Hour) old, now := at.Add(-2*time.Hour).UnixNano(), at.UnixNano() @@ -809,26 +815,8 @@ func TestRetirementRepairsOnlyMatchingNamespaceIdentity(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - for _, want := range []struct { - namespace, service string - start, end int64 - hasError int - }{ - {"a", "api", now, now + 10, 0}, - {"b", "worker", now + 20, now + 30, 1}, - } { - var service string - var start, maxStart, end int64 - var hasError int - if err := d.DB.QueryRow(`SELECT service,min_start,max_start,max_end,has_error FROM read_trace_candidates WHERE namespace=? AND trace_id='shared'`, want.namespace).Scan(&service, &start, &maxStart, &end, &hasError); err != nil { - t.Fatal(err) - } - if service != want.service || start != want.start || maxStart != want.start || end != want.end || hasError != want.hasError { - t.Fatalf("namespace=%s service=%s bounds=%d/%d/%d error=%d", want.namespace, service, start, maxStart, end, hasError) - } - } - var count int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_candidates WHERE trace_id='shared'`).Scan(&count); err != nil || count != 2 { - t.Fatalf("candidates=%d: %v", count, err) + w := queryrows.Window{Start: at.Add(-3 * time.Hour), End: at.Add(time.Minute)} + if got, want := completedTraces(t, d, w, traceExemplarDepth), []oracleRow{{"b", "shared", 1}, {"a", "shared", 0}}; !reflect.DeepEqual(got, want) { + t.Fatalf("ranking=%v want %v", got, want) } } diff --git a/internal/engine/panel_rollup_keys_test.go b/internal/engine/panel_rollup_keys_test.go index 07bdd417..c590938a 100644 --- a/internal/engine/panel_rollup_keys_test.go +++ b/internal/engine/panel_rollup_keys_test.go @@ -211,7 +211,7 @@ func TestPanelRollupKeysUpgradePreviousCacheOnReopen(t *testing.T) { } } for _, statement := range []string{ - `UPDATE read_cache_version SET version=18`, + `UPDATE read_cache_version SET version=19`, `UPDATE rollup_schema_version SET version=4`, `INSERT INTO anomaly_log VALUES ('upgrade','svc','latency',now(),now(),'witness','preserve me')`, `INSERT INTO read_edge_buckets VALUES ('upgrade',now(),10)`, @@ -244,13 +244,13 @@ func TestPanelRollupKeysUpgradePreviousCacheOnReopen(t *testing.T) { if err != nil { t.Fatal(err) } - for table, want := range map[string]int{"read_cache_version": 19, "rollup_schema_version": 5} { + for table, want := range map[string]int{"read_cache_version": 20, "rollup_schema_version": 5} { var got int if err := d.DB.QueryRowContext(t.Context(), "SELECT version FROM "+table).Scan(&got); err != nil || got != want { t.Fatalf("reopened %s=%d want=%d: %v", table, got, want, err) } } - for _, table := range []string{"span_rollup", "log_rollup", "endpoint_rollup", "service_rollup", "edge_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "rollup_state", "version_rollup", "version_rollup_batches", "read_batches", "read_trace_parts", "read_trace_candidates", "read_resource_inventory", "read_resource_seconds", "read_metric_names", "read_edge_buckets"} { + for _, table := range []string{"span_rollup", "log_rollup", "endpoint_rollup", "service_rollup", "edge_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "rollup_state", "version_rollup", "version_rollup_batches", "read_batches", "read_trace_exemplars", "read_resource_inventory", "read_resource_seconds", "read_metric_names", "read_edge_buckets"} { var count int if err := d.DB.QueryRowContext(t.Context(), "SELECT count(*) FROM "+table).Scan(&count); err != nil || count != 0 { t.Fatalf("previous %s survived rebuild: count=%d: %v", table, count, err) @@ -290,7 +290,7 @@ func TestPanelRollupKeysUpgradePreviousCacheOnReopen(t *testing.T) { t.Fatalf("rebuilt markers=%d: %v", markers, err) } var candidates int - if err := d.DB.QueryRowContext(t.Context(), `SELECT count(*) FROM read_trace_candidates WHERE min_start=?`, at.UnixNano()).Scan(&candidates); err != nil || candidates != len(batch.Spans) { + if err := d.DB.QueryRowContext(t.Context(), `SELECT count(*) FROM read_trace_exemplars WHERE start_unix_nano=?`, at.UnixNano()).Scan(&candidates); err != nil || candidates != len(batch.Spans) { t.Fatalf("rebuilt exact candidates=%d: %v", candidates, err) } if got := assertCommittedWatermarkParity(t, d); got.IsZero() { diff --git a/internal/engine/resource_inventory_test.go b/internal/engine/resource_inventory_test.go index 67e216fd..a8c21bf9 100644 --- a/internal/engine/resource_inventory_test.go +++ b/internal/engine/resource_inventory_test.go @@ -258,7 +258,7 @@ func TestResourceInventoryAtomicSignalAndMarkerFailures(t *testing.T) { } constraint := `CHECK (signal<>` + sqlLiteral(failure) + `)` if failure == "marker" { - if _, err := d.DB.Exec(`DROP TABLE read_batches; CREATE TABLE read_batches(batch_id VARCHAR PRIMARY KEY CHECK(batch_id<>'atomic'),replacement_id VARCHAR)`); err != nil { + if _, err := d.DB.Exec(`DROP TABLE read_batches; CREATE TABLE read_batches(batch_id VARCHAR PRIMARY KEY CHECK(batch_id<>'atomic'))`); err != nil { t.Fatal(err) } } else { @@ -269,7 +269,7 @@ func TestResourceInventoryAtomicSignalAndMarkerFailures(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err == nil { t.Fatal("injected failure succeeded") } - for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_exemplars"} { var count int if err := d.DB.QueryRow(`SELECT count(*) FROM ` + table).Scan(&count); err != nil || count != 0 { t.Fatalf("partial %s=%d %v", table, count, err) diff --git a/internal/engine/rollup_keys_test.go b/internal/engine/rollup_keys_test.go index 74c960ad..253ed724 100644 --- a/internal/engine/rollup_keys_test.go +++ b/internal/engine/rollup_keys_test.go @@ -73,7 +73,7 @@ func TestCachedReadScopesAcceptNUL(t *testing.T) { } w := queryrows.Window{Start: at.Add(-time.Second), End: at.Add(time.Minute), Namespace: ns, Service: svc, Kind: queryrows.TraceCandidateRead} ctx := inventoryContext(t.Context(), w) - for _, relation := range []string{"trace_candidates", "resource_inventory"} { + for _, relation := range []string{"trace_exemplars", "resource_inventory"} { var count int if err := d.QueryRowScan(ctx, []any{&count}, "SELECT count(*) FROM "+relation); err != nil || count != 1 { t.Fatalf("%s: count=%d: %v", relation, count, err) diff --git a/internal/engine/snapshot_binding_bench_test.go b/internal/engine/snapshot_binding_bench_test.go index 7f9693a2..4e443c51 100644 --- a/internal/engine/snapshot_binding_bench_test.go +++ b/internal/engine/snapshot_binding_bench_test.go @@ -120,7 +120,7 @@ func TestSnapshotPreparationRetentionScalingBenchmark(t *testing.T) { args []any }{ {"raw", `SELECT count(DISTINCT trace_id) FROM spans WHERE start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano()) - index, err := d.traceCandidateSource(t.Context(), tx, batches, markers, w) - if err != nil { - t.Fatal(err) - } - partial := "SELECT * FROM (" + index + ") WHERE NOT (" + full + ") AND " + fmt.Sprintf("max_start>=%d AND min_start<%d", w.Start.UnixNano(), w.End.UnixNano()) - // Keep both experimental expressions here so a negative result remains - // reproducible without changing product code or its materialization policy. - bodies := []string{ - `SELECT namespace,trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + full + ` GROUP BY namespace,trace_id`, - "SELECT namespace,trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + - ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + full + - ` AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (` + partial + `)) GROUP BY namespace,trace_id`, - } - if sources["trace_candidates"] != bodies[0] { - t.Fatal("benchmark baseline no longer matches the production candidate expression") - } - var queries, names []string - for i, body := range bodies { - sources["trace_candidates"] = body - bound, err := bindCompiledSnapshot(compiled, sources) - if err != nil { - t.Fatal(err) - } - marker := compiled.names["trace_candidates"] + " AS NOT MATERIALIZED (" - if strings.Count(bound, marker) != 1 { - t.Fatal("benchmark requires the current candidate inlining policy") - } - queries = append(queries, bound, strings.Replace(bound, marker, compiled.names["trace_candidates"]+" AS MATERIALIZED (", 1)) - names = append(names, fmt.Sprintf("expression_%d_inline", i), fmt.Sprintf("expression_%d_shared", i)) - } - args := []any{w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, 20} - run := func(query string) []completedTraceWindowRow { - rows, err := tx.QueryContext(t.Context(), query, args...) - if err != nil { - t.Fatal(err) - } - defer rows.Close() - var out []completedTraceWindowRow - for rows.Next() { - var row completedTraceWindowRow - if err := rows.Scan(&row.namespace, &row.trace, &row.hasError); err != nil { - t.Fatal(err) - } - out = append(out, row) - } - if err := rows.Err(); err != nil || len(out) != 20 { - t.Fatalf("completed rows=%d error=%v", len(out), err) - } - return out - } - rawWindow := w - rawWindow.Kind = queryrows.RawRead - want := completedTraceWindowRows(t, d, rawWindow, rawCompletedTraceWindowQuery, args) - // The actual completed-query API projects identities/errors. The separate - // lifecycle oracle compares its internal bounds as well. - for i := range want { - want[i].minStart, want[i].maxEnd = 0, 0 - } - for _, query := range queries { - for range 3 { - if got := run(query); !reflect.DeepEqual(got, want) { - t.Fatalf("experimental query changed ranking: got=%v want=%v", got, want) - } - } - } - timings := make([][]float64, len(queries)) - for sample := range samples { - for offset := range queries { - i := (sample + offset) % len(queries) - started := time.Now() - run(queries[i]) - timings[i] = append(timings[i], float64(time.Since(started))/float64(time.Millisecond)) - } - } - for i, values := range timings { - sort.Float64s(values) - t.Logf("scope_service=%q variant=%s bound_execution_only=true samples=%d median_ms=%.3f p95_ms=%.3f", w.Service, names[i], samples, (values[49]+values[50])/2, values[94]) - } -} diff --git a/internal/engine/trace_candidate_window_test.go b/internal/engine/trace_candidate_window_test.go index 62557f82..f69e4057 100644 --- a/internal/engine/trace_candidate_window_test.go +++ b/internal/engine/trace_candidate_window_test.go @@ -3,7 +3,6 @@ package engine import ( "fmt" "reflect" - "strings" "testing" "time" @@ -12,8 +11,8 @@ import ( telemetrystore "github.com/labstack/pitot/internal/telemetry/store" ) -// Compare the complete ranking query with raw rows, not merely the candidate -// source: boundary candidates and raw tail overlap and must merge exactly. +// Compare the complete ranking query with raw rows, not merely the exemplar +// source: cached rows of several batches and the raw tail merge per trace. func TestCompletedTraceWindowsMatchRawSpans(t *testing.T) { d, repo := newBatchCacheTest(t) span := func(namespace, service, trace string, start, end int64, failed bool) telemetry.Span { @@ -40,6 +39,7 @@ func TestCompletedTraceWindowsMatchRawSpans(t *testing.T) { } // The file straddles both boundaries, but includes an entirely contained // trace and an interior part of a trace whose other parts cross the start. + // A trace's best in-window span decides its rank, not its extent. commit("boundaries", span("prod", "api", "contained", 0, 5, false), span("prod", "api", "partial", -15, -10, true), @@ -66,10 +66,7 @@ func TestCompletedTraceWindowsMatchRawSpans(t *testing.T) { for _, limit := range []int{1, 3, 20} { w := queryrows.Window{Start: time.Unix(0, bounds[0]).UTC(), End: time.Unix(0, bounds[1]).UTC(), Namespace: scope[0], Service: scope[1], Kind: queryrows.TraceCandidateRead} args := []any{w.Start, w.End, scope[0], scope[0], scope[1], scope[1], limit} - // Expose the kernel's aggregate bounds as well as its public - // identity/error projection, without replacing any merge logic. - query := strings.Replace(queryrows.CompletedTraceQuery, "SELECT namespace,trace_id,has_error FROM candidates", "SELECT namespace,trace_id,has_error,min_start,max_end FROM candidates", 1) - got := completedTraceWindowRows(t, d, w, query, args) + got := completedTraceWindowRows(t, d, w, queryrows.CompletedTraceQuery, args) w.Kind = queryrows.RawRead want := completedTraceWindowRows(t, d, w, rawCompletedTraceWindowQuery, args) if !reflect.DeepEqual(got, want) { @@ -110,19 +107,19 @@ func TestCompletedTraceWindowsMatchRawSpans(t *testing.T) { check("retired marker cleanup") } -const rawCompletedTraceWindowQuery = `SELECT namespace,trace_id, - max(CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error, - min(start_unix_nano),max(end_unix_nano) +// The best span is the longest error span when there is one, otherwise the +// longest span: an independent formulation of the production ranking. +const rawCompletedTraceWindowQuery = `SELECT namespace,trace_id,max(has_error) AS has_error FROM ( + SELECT namespace,trace_id,CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END AS has_error,end_unix_nano-start_unix_nano AS duration FROM spans WHERE start_time >= $1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < $2::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' + AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'') GROUP BY namespace,trace_id - ORDER BY has_error DESC,max(end_unix_nano)-min(start_unix_nano) DESC,trace_id,namespace LIMIT $7` + ORDER BY max(has_error) DESC,coalesce(max(duration) FILTER (WHERE has_error=1),max(duration)) DESC,trace_id,namespace LIMIT $7` type completedTraceWindowRow struct { namespace, trace string hasError int - minStart, maxEnd int64 } func completedTraceWindowRows(t testing.TB, d *Duck, w queryrows.Window, sql string, args []any) []completedTraceWindowRow { @@ -135,7 +132,7 @@ func completedTraceWindowRows(t testing.TB, d *Duck, w queryrows.Window, sql str var out []completedTraceWindowRow for rows.Next() { var row completedTraceWindowRow - if err := rows.Scan(&row.namespace, &row.trace, &row.hasError, &row.minStart, &row.maxEnd); err != nil { + if err := rows.Scan(&row.namespace, &row.trace, &row.hasError); err != nil { t.Fatal(err) } out = append(out, row) diff --git a/internal/engine/trace_exemplars.go b/internal/engine/trace_exemplars.go new file mode 100644 index 00000000..dcb7deb2 --- /dev/null +++ b/internal/engine/trace_exemplars.go @@ -0,0 +1,286 @@ +package engine + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" +) + +// read_trace_exemplars ranks traces by their best in-scope, in-window span: +// an error first, then a longer span, then trace_id and namespace. Rows are +// kept per (batch, namespace, service, minute of span start) partition: +// +// - in_full: the best span of each of the partition's top traces, enough to +// rank any window that covers the whole minute; +// - in_suffix: spans that fewer than traceExemplarDepth other traces outrank +// using spans starting at or after them, enough for a window starting in +// the minute; +// - in_prefix: the mirror, enough for a window ending in the minute. +// +// A trace's score is a maximum over its spans, so it decomposes over any +// union of partitions and raw spans. A span another trace's in-window spans +// outrank at least traceExemplarDepth times can never be a top trace's best, +// so any superset of these rows ranks exactly. +const createTraceExemplarsTable = `CREATE TABLE IF NOT EXISTS read_trace_exemplars (batch_id VARCHAR,namespace VARCHAR,service VARCHAR,minute BIGINT,trace_id VARCHAR,start_unix_nano BIGINT,duration BIGINT,has_error INTEGER,in_full BOOLEAN,in_suffix BOOLEAN,in_prefix BOOLEAN)` + +// traceExemplarDepth bounds every completed-trace ranking limit. +const traceExemplarDepth = queryrows.CompletedTraceLimit + +const minuteNanos = int64(60_000_000_000) + +// floorMinute is floor division, also for instants before the epoch. +func floorMinute(nanos int64) int64 { + m := nanos % minuteNanos + if m < 0 { + m += minuteNanos + } + return nanos - m +} + +// minuteSQL matches floorMinute: DuckDB's % keeps the dividend's sign. +func minuteSQL(column string) string { + return column + "-((" + column + "%60000000000)+60000000000)%60000000000" +} + +const ( + exemplarFull = 1 + exemplarSuffix = 2 + exemplarPrefix = 4 +) + +type exemplarSpan struct { + id int64 + trace int64 // dense rank of trace_id within its partition + start int64 + dur int64 + err bool +} + +// better reports whether a outranks b. In one partition the namespace is +// fixed, so the trace rank completes the global tie order. +func (a exemplarSpan) better(b exemplarSpan) bool { + if a.err != b.err { + return a.err + } + if a.dur != b.dur { + return a.dur > b.dur + } + return a.trace < b.trace +} + +// exemplarSkyline returns each span's flags for one partition. +func exemplarSkyline(spans []exemplarSpan) []uint8 { + flags := make([]uint8, len(spans)) + traces := int64(0) + for _, s := range spans { + traces = max(traces, s.trace) + } + best := make([]int, traces+1) + for i := range best { + best[i] = -1 + } + for i, s := range spans { + if b := best[s.trace]; b < 0 || s.better(spans[b]) { + best[s.trace] = i + } + } + var top []int + for _, b := range best { + if b >= 0 { + top = append(top, b) + } + } + sort.Slice(top, func(i, j int) bool { return spans[top[i]].better(spans[top[j]]) }) + for _, i := range top[:min(len(top), traceExemplarDepth)] { + flags[i] |= exemplarFull + } + order := make([]int, len(spans)) + for i := range order { + order[i] = i + } + sort.Slice(order, func(i, j int) bool { return spans[order[i]].start > spans[order[j]].start }) + exemplarPass(spans, order, best, flags, exemplarSuffix) + for i, j := 0, len(order)-1; i < j; i, j = i+1, j-1 { + order[i], order[j] = order[j], order[i] + } + exemplarPass(spans, order, best, flags, exemplarPrefix) + return flags +} + +// exemplarPass scans spans in order, grouping equal start times, and keeps a +// span when fewer than traceExemplarDepth other traces outrank it with spans +// already scanned. It tracks the depth+2 best per-trace scores; per-trace bests +// only rise, so a trace outside that set is outranked by depth+1 others and +// cannot change any count below the depth. +func exemplarPass(spans []exemplarSpan, order, best []int, flags []uint8, bit uint8) { + for i := range best { + best[i] = -1 + } + capacity := traceExemplarDepth + 2 + top := make([]int, 0, capacity) // indexes of per-trace bests + for g := 0; g < len(order); { + e := g + for e < len(order) && spans[order[e]].start == spans[order[g]].start { + e++ + } + for _, i := range order[g:e] { + s := spans[i] + if b := best[s.trace]; b >= 0 && !s.better(spans[b]) { + continue + } + best[s.trace] = i + found := false + for j, o := range top { + if spans[o].trace == s.trace { + top[j], found = i, true + break + } + } + if found { + continue + } + if len(top) < capacity { + top = append(top, i) + continue + } + worst := 0 + for j := range top { + if spans[top[worst]].better(spans[top[j]]) { + worst = j + } + } + if s.better(spans[top[worst]]) { + top[worst] = i + } + } + for _, i := range order[g:e] { + n := 0 + for _, o := range top { + if spans[o].trace != spans[i].trace && spans[o].better(spans[i]) { + n++ + } + } + if n < traceExemplarDepth { + flags[i] |= bit + } + } + g = e + } +} + +// insertTraceExemplars reads source's projected spans (id, batch_id, +// namespace, service, minute, trace_id, start_unix_nano, duration, has_error), +// computes each partition's rows and appends them. Only integers cross into +// Go; the kept ids return as one bound string. +func insertTraceExemplars(ctx context.Context, tx *sql.Tx, source string) (int64, error) { + rows, err := tx.QueryContext(ctx, `SELECT id,dense_rank() OVER (ORDER BY batch_id,namespace,service,minute) AS part, + dense_rank() OVER (PARTITION BY batch_id,namespace,service,minute ORDER BY trace_id) AS trace,start_unix_nano,duration,has_error + FROM `+source+` ORDER BY part`) + if err != nil { + return 0, err + } + var kept strings.Builder + var spans []exemplarSpan + flush := func() { + for i, f := range exemplarSkyline(spans) { + if f != 0 { + if kept.Len() > 0 { + kept.WriteByte(',') + } + kept.WriteString(strconv.FormatInt(spans[i].id, 10)) + kept.WriteByte(':') + kept.WriteByte('0' + f) + } + } + spans = spans[:0] + } + current := int64(-1) + for rows.Next() { + var s exemplarSpan + var part, failed int64 + if err := rows.Scan(&s.id, &part, &s.trace, &s.start, &s.dur, &failed); err != nil { + rows.Close() + return 0, err + } + s.err = failed != 0 + if part != current && len(spans) > 0 { + flush() + } + current = part + spans = append(spans, s) + } + err = rows.Err() + rows.Close() + if err != nil { + return 0, err + } + if len(spans) > 0 { + flush() + } + if kept.Len() == 0 { + return 0, nil + } + result, err := tx.ExecContext(ctx, `INSERT INTO read_trace_exemplars SELECT s.batch_id,s.namespace,s.service,s.minute,s.trace_id,s.start_unix_nano,s.duration,s.has_error, + (k.f&1)<>0,(k.f&2)<>0,(k.f&4)<>0 + FROM `+source+` s JOIN (SELECT split_part(e,':',1)::BIGINT AS id,split_part(e,':',2)::INTEGER AS f FROM unnest(string_split(?::VARCHAR,',')) AS u(e)) k ON s.id=k.id`, kept.String()) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// traceExemplarSource is the private trace_exemplars relation: the in-window, +// in-scope spans that can decide a ranking. Covered minutes contribute their +// full rows, a partial start minute its suffix rows, a partial end minute its +// prefix rows, and uncached files their raw spans. A window inside one minute +// reads the overlapping files' raw spans. +func (d *Duck) traceExemplarSource(batches []telemetry.BatchMetadata, markers map[string]bool, w queryrows.Window) string { + start, end := w.Start.UnixNano(), w.End.UnixNano() + startMinute, endMinute := floorMinute(start), floorMinute(end) + partialStart, partialEnd := start != startMinute, end != endMinute + exact := partialStart && partialEnd && startMinute == endMinute + var cachedIDs []string + var raw []telemetry.BatchMetadata + for _, b := range batches { + if !overlapping(b, "spans", w) { + continue + } + if markers[b.ID] && !exact { + cachedIDs = append(cachedIDs, b.ID) + } else { + raw = append(raw, b) + } + } + var scope strings.Builder + if w.Namespace != "" { + scope.WriteString(" AND namespace=" + sqlLiteral(w.Namespace)) + } + if w.Service != "" { + scope.WriteString(" AND service=" + sqlLiteral(w.Service)) + } + tail := `SELECT namespace,service,trace_id,start_time,start_unix_nano,duration,has_error FROM (SELECT coalesce(namespace,'') AS namespace,coalesce(service,'') AS service,trace_id,start_time,start_unix_nano,end_unix_nano-start_unix_nano AS duration, + CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END AS has_error FROM (` + cleanSource("spans", d.snapshotSource("spans", raw, batches)) + `) WHERE ` + windowPredicate("start_time", w) + ` AND trace_id<>'') WHERE true` + scope.String() + if len(cachedIDs) == 0 { + return tail + } + fullFrom := startMinute + if partialStart { + fullFrom += minuteNanos + } + ranges := []string{fmt.Sprintf("(in_full AND minute>=%d AND minute<%d)", fullFrom, endMinute)} + if partialStart { + ranges = append(ranges, fmt.Sprintf("(in_suffix AND minute=%d AND start_unix_nano>=%d)", startMinute, start)) + } + if partialEnd { + ranges = append(ranges, fmt.Sprintf("(in_prefix AND minute=%d AND start_unix_nano<%d)", endMinute, end)) + } + cached := `SELECT namespace,service,trace_id,make_timestamp_ns(start_unix_nano)::TIMESTAMPTZ_NS AS start_time,start_unix_nano,duration,has_error FROM read_trace_exemplars WHERE batch_id IN (SELECT unnest([` + idsSQL(cachedIDs) + `]))` + + fmt.Sprintf(" AND minute>=%d AND minute<=%d", startMinute, endMinute) + scope.String() + ` AND (` + strings.Join(ranges, " OR ") + `)` + return cached + ` UNION ALL ` + tail +} diff --git a/internal/engine/trace_exemplars_test.go b/internal/engine/trace_exemplars_test.go new file mode 100644 index 00000000..02014d6d --- /dev/null +++ b/internal/engine/trace_exemplars_test.go @@ -0,0 +1,420 @@ +package engine + +import ( + "fmt" + "math/rand" + "reflect" + "sort" + "testing" + "time" + + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +type skylineCase struct { + part int // partition: batch*minutes + minute + trace string + span exemplarSpan +} + +// bruteTopTraces ranks traces by their best span among in-window spans. +func bruteTopTraces(spans []skylineCase, in func(skylineCase) bool, limit int) []string { + best := map[string]skylineCase{} + for _, s := range spans { + if !in(s) { + continue + } + if b, ok := best[s.trace]; !ok || skylineBetter(s, b) { + best[s.trace] = s + } + } + list := make([]skylineCase, 0, len(best)) + for _, s := range best { + list = append(list, s) + } + sort.Slice(list, func(i, j int) bool { return skylineBetter(list[i], list[j]) }) + out := []string{} + for i := 0; i < len(list) && i < limit; i++ { + out = append(out, list[i].trace) + } + return out +} + +func skylineBetter(a, b skylineCase) bool { + if a.span.err != b.span.err { + return a.span.err + } + if a.span.dur != b.span.dur { + return a.span.dur > b.span.dur + } + return a.trace < b.trace +} + +// keepSkyline returns each partition's rows with their flags, ranking traces +// within a partition by their names as the SQL dense rank does. +func keepSkyline(spans []skylineCase) ([]skylineCase, []uint8) { + parts := map[int][]int{} + for i, s := range spans { + parts[s.part] = append(parts[s.part], i) + } + var rows []skylineCase + var flags []uint8 + for _, members := range parts { + names := map[string]bool{} + for _, i := range members { + names[spans[i].trace] = true + } + sorted := make([]string, 0, len(names)) + for name := range names { + sorted = append(sorted, name) + } + sort.Strings(sorted) + rank := map[string]int64{} + for i, name := range sorted { + rank[name] = int64(i + 1) + } + in := make([]exemplarSpan, len(members)) + for j, i := range members { + in[j] = spans[i].span + in[j].trace = rank[spans[i].trace] + } + for j, f := range exemplarSkyline(in) { + if f != 0 { + rows = append(rows, spans[members[j]]) + flags = append(flags, f) + } + } + } + return rows, flags +} + +// The pure skyline must answer every window exactly from full rows of covered +// minutes, suffix rows of the start minute and prefix rows of the end minute, +// including after compaction ranks the union of its inputs' rows again. +func TestTraceExemplarSkylineMatchesBruteForce(t *testing.T) { + r := rand.New(rand.NewSource(7)) + const minutes, batches = 4, 3 + for iter := range 200 { + var all []skylineCase + perBatch := make([][]skylineCase, batches) + for b := range batches { + for range 60 + r.Intn(500) { + m := r.Intn(minutes) + s := skylineCase{part: b*minutes + m, trace: fmt.Sprintf("t%02d", r.Intn(30+r.Intn(60)))} + s.span = exemplarSpan{start: int64(m*60 + r.Intn(60)), err: r.Intn(25) == 0, dur: int64(r.Intn(20))} + all = append(all, s) + perBatch[b] = append(perBatch[b], s) + } + } + rows, flags := keepSkyline(all) + // Compaction: batches 0 and 1 merge; rank the union of their rows. + var inputs []skylineCase + for i, row := range rows { + if row.part/minutes < 2 { + _ = flags[i] + row.part = row.part % minutes + inputs = append(inputs, row) + } + } + merged, mergedFlags := keepSkyline(inputs) + for i, row := range rows { + if row.part/minutes == 2 { + merged = append(merged, row) + mergedFlags = append(mergedFlags, flags[i]) + } + } + for range 30 { + s0 := int64(r.Intn(minutes * 60)) + e0 := s0 + 1 + int64(r.Intn(minutes*60-int(s0))) + if r.Intn(4) == 0 { + s0 -= s0 % 60 + } + if r.Intn(4) == 0 && e0%60 != 0 && e0+60-e0%60 <= minutes*60 { + e0 += 60 - e0%60 + } + if s0/60 == (e0-1)/60 && s0%60 != 0 && e0%60 != 0 { + continue // one minute reads raw spans + } + in := func(s skylineCase) bool { return s.span.start >= s0 && s.span.start < e0 } + for _, limit := range []int{1, traceExemplarDepth} { + want := bruteTopTraces(all, in, limit) + for name, set := range map[string][2]any{"batches": {rows, flags}, "compacted": {merged, mergedFlags}} { + rs, fs := set[0].([]skylineCase), set[1].([]uint8) + var read []skylineCase + for i, row := range rs { + m := row.span.start / 60 * 60 + switch { + case m >= s0 && m+60 <= e0: + if fs[i]&exemplarFull != 0 { + read = append(read, row) + } + case m < s0 && m+60 > s0: + if fs[i]&exemplarSuffix != 0 && in(row) { + read = append(read, row) + } + case m < e0 && m+60 > e0: + if fs[i]&exemplarPrefix != 0 && in(row) { + read = append(read, row) + } + } + } + if got := bruteTopTraces(read, in, limit); !reflect.DeepEqual(got, want) { + t.Fatalf("iter %d %s window [%d,%d) limit %d: got %v want %v", iter, name, s0, e0, limit, got, want) + } + } + } + } + } +} + +type oracleSpan struct { + namespace, service, trace string + start, end int64 + failed bool +} + +type oracleRow struct { + namespace, trace string + hasError int +} + +// bruteCompletedTraces ranks raw spans in Go, independently of SQL. +func bruteCompletedTraces(spans []oracleSpan, start, end int64, namespace, service string, limit int) []oracleRow { + type key struct{ namespace, trace string } + type score struct { + failed bool + dur int64 + } + best := map[key]score{} + for _, s := range spans { + if s.start < start || s.start >= end || s.trace == "" || namespace != "" && s.namespace != namespace || service != "" && s.service != service { + continue + } + k, sc := key{s.namespace, s.trace}, score{s.failed, s.end - s.start} + if b, ok := best[k]; !ok || sc.failed && !b.failed || sc.failed == b.failed && sc.dur > b.dur { + best[k] = sc + } + } + keys := make([]key, 0, len(best)) + for k := range best { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, b := best[keys[i]], best[keys[j]] + if a.failed != b.failed { + return a.failed + } + if a.dur != b.dur { + return a.dur > b.dur + } + if keys[i].trace != keys[j].trace { + return keys[i].trace < keys[j].trace + } + return keys[i].namespace < keys[j].namespace + }) + out := []oracleRow{} + for _, k := range keys[:min(limit, len(keys))] { + flag := 0 + if best[k].failed { + flag = 1 + } + out = append(out, oracleRow{k.namespace, k.trace, flag}) + } + return out +} + +func completedTraces(t *testing.T, d *Duck, w queryrows.Window, limit int) []oracleRow { + t.Helper() + w.Kind = queryrows.TraceCandidateRead + rows, err := d.QueryContext(queryrows.WithWindow(t.Context(), w), queryrows.CompletedTraceQuery, w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, limit) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + out := []oracleRow{} + for rows.Next() { + var row oracleRow + if err := rows.Scan(&row.namespace, &row.trace, &row.hasError); err != nil { + t.Fatal(err) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} + +// rawOracleSpans reads every retained span, so the oracle follows retention. +func rawOracleSpans(t *testing.T, d *Duck) []oracleSpan { + t.Helper() + all := queryrows.Window{Start: time.Unix(0, -1<<62).UTC(), End: time.Unix(0, 1<<62).UTC(), Kind: queryrows.RawRead} + rows, err := d.QueryContext(queryrows.WithWindow(t.Context(), all), `SELECT coalesce(namespace,''),coalesce(service,''),trace_id,start_unix_nano,end_unix_nano,status='STATUS_CODE_ERROR' FROM spans`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []oracleSpan + for rows.Next() { + var s oracleSpan + if err := rows.Scan(&s.namespace, &s.service, &s.trace, &s.start, &s.end, &s.failed); err != nil { + t.Fatal(err) + } + out = append(out, s) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} + +// Compare the production ranking with brute force over raw spans through +// late publication, an uncached tail, covered and uncovered compaction, +// compaction of compaction outputs, and retention. Traces span batches, +// minutes, namespaces and services; starts and durations tie; windows are +// shorter than, about, and longer than a minute, aligned or not. +func TestCompletedTraceQueryMatchesBruteForce(t *testing.T) { + d, repo := newBatchCacheTest(t) + r := rand.New(rand.NewSource(11)) + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + const minutes = 5 + hour := int64(time.Hour) + ingest := base.UnixNano() + solo := "" + commit := func(id string, ingestedAt int64, n int, minute func() int) { + t.Helper() + batch := telemetrystore.Batch{ID: id} + for i := range n { + namespace := "prod" + if r.Intn(6) == 0 { + namespace = "edge" + } + service := "api" + if r.Intn(4) == 0 { + service = "db" + } + if solo != "" { + namespace, service = "prod", solo + } + // Coarse starts and durations force ties on both; some starts sit on + // a minute boundary. + start := base.UnixNano() + int64(minute())*minuteNanos + int64(r.Intn(12))*5*int64(time.Second) + status := "STATUS_CODE_OK" + if r.Intn(25) == 0 { + status = "STATUS_CODE_ERROR" + } + dur := int64(1+r.Intn(12)) * int64(time.Millisecond) + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: namespace, ServiceName: service, + TraceID: fmt.Sprintf("trace-%03d", r.Intn(400)), SpanID: fmt.Sprintf("%s-%d", id, i), + StartUnixNanos: start, EndUnixNanos: start + dur, DurationMS: float64(dur) / 1e6, + StatusCode: status, IngestedAt: ingestedAt}) + } + if err := commitRestoredFixture(repo, t.Context(), batch); err != nil { + t.Fatal(err) + } + } + refresh := func() { + t.Helper() + for { + rows, err := d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) + } + if rows == 0 { + return + } + } + } + anyMinute := func() int { return r.Intn(minutes) } + type scope struct{ namespace, service string } + // A scope one batch owns ranks from a single partition per minute, so + // every kept or missing span decides the result. + scopes := []scope{{"", ""}, {"prod", ""}, {"prod", "api"}, {"edge", "db"}, {"", "db"}, {"prod", "solo"}} + check := func(stage string) { + t.Helper() + raw := rawOracleSpans(t, d) + at := func(seconds int64) time.Time { return base.Add(time.Duration(seconds) * time.Second) } + windows := [][2]int64{{0, minutes * 60}, {-30, minutes*60 + 30}, {60, 120}, {65, 115}, {55, 65}, {60, 65}, {55, 60}, {30, 90}, {5, 245}, {20, 200}} + for range 6 { + s := int64(r.Intn(minutes*12)) * 5 + windows = append(windows, [2]int64{s, s + int64(1+r.Intn(minutes*12))*5}) + } + for _, bounds := range windows { + for _, sc := range scopes { + for _, limit := range []int{1, traceExemplarDepth} { + w := queryrows.Window{Start: at(bounds[0]), End: at(bounds[1]), Namespace: sc.namespace, Service: sc.service} + got := completedTraces(t, d, w, limit) + want := bruteCompletedTraces(raw, w.Start.UnixNano(), w.End.UnixNano(), sc.namespace, sc.service, limit) + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s window=%v scope=%v limit=%d:\n got %v\nwant %v", stage, bounds, sc, limit, got, want) + } + } + } + } + } + // Like live ingest, each batch covers about a minute. + for i := range 12 { + commit(fmt.Sprintf("a%02d", i), ingest+int64(i), 2000, func() int { return (i + r.Intn(2)) % minutes }) + } + solo = "solo" + commit("solo", ingest+12, 3000, func() int { return 1 + r.Intn(3) }) + solo = "" + refresh() + check("cached") + // Late publication: recent ingest, old event times, not yet cached. + for i := range 4 { + commit(fmt.Sprintf("late%02d", i), ingest+int64(20+i), 1000, func() int { return r.Intn(2) }) + } + check("uncached tail") + refresh() + check("acknowledged late publication") + if n, err := repo.CompactParquet(t.Context(), d, 8); err != nil || n == 0 { + t.Fatalf("compaction inputs=%d error=%v", n, err) + } + check("covered compaction before marker cleanup") + refresh() + check("covered compaction") + // An uncovered compaction: inputs that were never cached. + for i := range 8 { + commit(fmt.Sprintf("b%02d", i), ingest+int64(40+i), 800, anyMinute) + } + if n, err := repo.CompactParquet(t.Context(), d, 8); err != nil || n == 0 { + t.Fatalf("uncovered compaction inputs=%d error=%v", n, err) + } + check("uncovered compaction") + refresh() + check("uncovered compaction cached") + // Once their hour settles, the covered and the uncovered compaction + // outputs merge again from their transferred rows. + for i := range 3 { + commit(fmt.Sprintf("c%02d", i), ingest+int64(60+i), 800, anyMinute) + } + refresh() + commit("newest", ingest+48*hour, 20, anyMinute) + refresh() + if n, err := repo.CompactParquet(t.Context(), d, 64); err != nil || n == 0 { + t.Fatalf("day compaction inputs=%d error=%v", n, err) + } + merged := false + for _, b := range repo.Parquet.BatchMetadata() { + merged = merged || b.Generation >= 2 + } + if !merged { + t.Fatal("compaction outputs did not merge again") + } + check("mixed generation compaction") + refresh() + check("mixed generation compaction cached") + if n, err := repo.PruneParquet(t.Context(), d, ingest+30, 64); err != nil || n == 0 { + t.Fatalf("retention removed=%d error=%v", n, err) + } + check("retention before marker cleanup") + refresh() + check("retention") + var stale int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE batch_id NOT IN (SELECT batch_id FROM read_batches)`).Scan(&stale); err != nil || stale != 0 { + t.Fatalf("orphaned exemplar rows=%d error=%v", stale, err) + } +} diff --git a/internal/engine/views_test.go b/internal/engine/views_test.go index 1f14afbe..32455885 100644 --- a/internal/engine/views_test.go +++ b/internal/engine/views_test.go @@ -56,7 +56,7 @@ func TestCreateTables_CacheTablesIncludePartitionColumns(t *testing.T) { rows, err := db.Query(` SELECT table_name, column_name FROM duckdb_columns() -WHERE table_name IN ('service_rollup', 'edge_rollup', 'read_trace_parts', 'read_resource_inventory', 'rollup_state') +WHERE table_name IN ('service_rollup', 'edge_rollup', 'read_trace_exemplars', 'read_resource_inventory', 'rollup_state') ORDER BY table_name, column_name`) if err != nil { t.Fatalf("query duckdb_columns failed: %v", err) @@ -81,7 +81,7 @@ ORDER BY table_name, column_name`) required := map[string][]string{ "service_rollup": {"namespace", "bucket", "service"}, "edge_rollup": {"namespace", "bucket", "caller", "callee", "edge_type"}, - "read_trace_parts": {"batch_id", "namespace", "service", "trace_id", "min_start", "max_start", "max_end", "has_error"}, + "read_trace_exemplars": {"batch_id", "namespace", "service", "minute", "trace_id", "start_unix_nano", "duration", "has_error", "in_full", "in_suffix", "in_prefix"}, "read_resource_inventory": {"batch_id", "signal", "namespace", "service", "resource", "first_seen", "last_seen"}, "rollup_state": {"cache_key", "last_ingested_unix_nano", "updated_at"}, } @@ -114,18 +114,18 @@ func TestCreateViews_AttrMacroWorks(t *testing.T) { } } -// An index on the per-trace candidate table grows with every retained trace, -// and DuckDB loads all of it to upsert or checkpoint. At a few million traces -// that exceeded a 1 GB memory limit, failed the checkpoint and invalidated the -// database, so the read cache keeps no index on per-trace rows. +// An index on per-trace rows grows with every retained trace, and DuckDB +// loads all of it to upsert or checkpoint. At a few million traces that +// exceeded a 1 GB memory limit, failed the checkpoint and invalidated the +// database, so the append-only exemplar rows keep no index. func TestCreateTables_ReadCacheHasNoTraceIndex(t *testing.T) { db := openTestDuck(t) if err := CreateTables(db); err != nil { t.Fatalf("CreateTables failed: %v", err) } var tables []string - rows, err := db.Query(`SELECT DISTINCT table_name FROM duckdb_constraints() WHERE table_name IN ('read_trace_candidates','read_trace_parts') AND constraint_type IN ('PRIMARY KEY','UNIQUE') -UNION SELECT DISTINCT table_name FROM duckdb_indexes() WHERE table_name IN ('read_trace_candidates','read_trace_parts')`) + rows, err := db.Query(`SELECT DISTINCT table_name FROM duckdb_constraints() WHERE table_name='read_trace_exemplars' AND constraint_type IN ('PRIMARY KEY','UNIQUE') +UNION SELECT DISTINCT table_name FROM duckdb_indexes() WHERE table_name='read_trace_exemplars'`) if err != nil { t.Fatal(err) } diff --git a/internal/observability/batch_reads_test.go b/internal/observability/batch_reads_test.go index 91dd6b5b..54d7c7b3 100644 --- a/internal/observability/batch_reads_test.go +++ b/internal/observability/batch_reads_test.go @@ -29,14 +29,16 @@ func newCompletedReadTest(t *testing.T) (*engine.Duck, *telemetrystore.Repositor return d, repo, New(d, d, 30) } -// rawTraceQuery ranks a window's traces straight from its spans: the oracle -// the completed-batch candidates must match. +// rawTraceQuery ranks a window's traces straight from its spans by their +// best span, the longest error span or else the longest span: the oracle the +// completed-batch exemplars must match. const rawTraceQuery = `SELECT trace_id FROM spans WHERE start_time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (? = '' OR namespace = ?) AND trace_id <> '' AND (? = '' OR service = ?) GROUP BY namespace,trace_id ORDER BY MAX(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) DESC, - MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC, namespace ASC + coalesce(MAX(end_unix_nano - start_unix_nano) FILTER (WHERE status = 'STATUS_CODE_ERROR'), MAX(end_unix_nano - start_unix_nano)) DESC, + trace_id ASC, namespace ASC LIMIT 1` func rawTraceID(t testing.TB, d *engine.Duck, scope Scope, service string) (string, error) { @@ -133,9 +135,8 @@ func TestCompletedTraceReadsMatchColdAndLateData(t *testing.T) { } } -// When every cached file overlapping the window lies inside it, a trace whose -// other cached spans sit in a file outside the window is still a candidate: -// its in-window spans decide the ranking, as they do for the raw query. +// A trace whose other cached spans sit in a file outside the window is still +// ranked: its in-window spans decide the ranking, as they do for the raw query. func TestCompletedTraceKeepsTraceWithSpansOutsideWindow(t *testing.T) { d, repo, svc := newCompletedReadTest(t) t0 := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) @@ -169,8 +170,8 @@ func TestCompletedTraceKeepsTraceWithSpansOutsideWindow(t *testing.T) { } } -// Time-local batches put most window edges between files, so cached candidates -// must combine index rows with the in-window parts of partially covered traces. +// Time-local batches put most window edges between files and minutes, so +// cached exemplars must combine partial minutes of several batches. func TestCompletedTraceMatchesRawAcrossTimeLocalBatches(t *testing.T) { d, repo, svc := newCompletedReadTest(t) rng := rand.New(rand.NewPCG(11, 104729)) diff --git a/internal/query/exemplar_index_test.go b/internal/query/exemplar_index_test.go index 4a0412f5..20781ffe 100644 --- a/internal/query/exemplar_index_test.go +++ b/internal/query/exemplar_index_test.go @@ -22,9 +22,10 @@ func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { } spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: fmt.Sprintf("trace-%02d", i), SpanID: "root", Name: "root", StartUnixNanos: n, EndUnixNanos: n + int64(i+1)*1000000, DurationMS: float64(i + 1), StatusCode: status}) } - // Select a long scoped trace, but order its displayed short root last. + // Select a trace by its longest scoped span, but order its displayed + // short root last. late := fixtureStart.Add(45 * time.Minute).UnixNano() - spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: "trace-01", SpanID: "late", ParentSpanID: "root", StartUnixNanos: late, EndUnixNanos: late + 1000000, DurationMS: 1}) + spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: "trace-01", SpanID: "late", ParentSpanID: "root", StartUnixNanos: late, EndUnixNanos: late + 100000000, DurationMS: 100}) // A much longer trace in another namespace and a different service must not win. for _, s := range []struct{ ns, service string }{{"other", "checkout"}, {"shop", "other"}} { id := s.ns + s.service diff --git a/internal/query/exemplars.go b/internal/query/exemplars.go index a785fbf1..612af408 100644 --- a/internal/query/exemplars.go +++ b/internal/query/exemplars.go @@ -349,7 +349,7 @@ func outsideSelection() error { return Problems{{Path: "from", Message: "selection does not overlap the panel window"}} } -// Only exact namespace/service equalities can use trace candidates. Every +// Only exact namespace/service equalities can use trace exemplars. Every // remaining selection keeps its generic span/log predicates. func (e *Executor) exemplarTraceScope(ctx context.Context, p *Panel, filters []Filter, scope Scope, dimensions map[string]string, bucket *SelectionBucket) (queryrows.Window, bool, bool, error) { w := queryrows.Window{Start: scope.Start, End: scope.End, Kind: queryrows.TraceCandidateRead} diff --git a/internal/query/log_redaction_test.go b/internal/query/log_redaction_test.go index 45721c35..32754b87 100644 --- a/internal/query/log_redaction_test.go +++ b/internal/query/log_redaction_test.go @@ -72,8 +72,8 @@ func TestSQLPanelKeepsQualifiedNamesThatACTEShadows(t *testing.T) { commit(t, repo, nil, []telemetry.Log{{Body: "token=private failed", EventUnixNanos: n, TimeUnixNanos: n, IngestedAt: n}}) e := NewExecutor(engine, 30) e.now = func() time.Time { return fixtureStart.Add(time.Hour) } - // DuckDB binds main.trace_candidates.x to the struct alias, never to the CTE. - text := "WITH trace_candidates AS (SELECT 'cte' AS x) SELECT main.trace_candidates.x AS b FROM (SELECT {'x': 42} AS trace_candidates) AS main, trace_candidates, logs WHERE $__window(logs.time)" + // DuckDB binds main.trace_exemplars.x to the struct alias, never to the CTE. + text := "WITH trace_exemplars AS (SELECT 'cte' AS x) SELECT main.trace_exemplars.x AS b FROM (SELECT {'x': 42} AS trace_exemplars) AS main, trace_exemplars, logs WHERE $__window(logs.time)" d := Dashboard{Name: "CTE shadow", Time: Time{Range: "1h"}, Panels: []Panel{{ID: "p", Title: "SQL", Viz: "table", SQL: text}}} results, err := e.Run(t.Context(), RunRequest{Dashboard: d}) if err != nil || len(results) != 1 || results[0].Status != StatusOK || results[0].Frame.Values[0][0] != float64(42) { diff --git a/internal/queryrows/completed_traces.go b/internal/queryrows/completed_traces.go new file mode 100644 index 00000000..eff800be --- /dev/null +++ b/internal/queryrows/completed_traces.go @@ -0,0 +1,16 @@ +package queryrows + +// CompletedTraceLimit is the deepest ranking CompletedTraceQuery answers +// exactly; the read cache keeps enough spans per partition for this depth. +const CompletedTraceLimit = 21 + +// CompletedTraceQuery ranks scoped traces by their best in-window span: an +// error first, then the longest span, then trace_id and namespace. Arguments +// are start, end, namespace twice, service twice, and a limit of at most +// CompletedTraceLimit. Results are namespace, trace_id, and whether the trace +// has an in-scope, in-window error span. +const CompletedTraceQuery = `SELECT namespace,trace_id,max(has_error) AS has_error FROM trace_exemplars + WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' + GROUP BY namespace,trace_id + ORDER BY max([has_error::BIGINT,duration]) DESC,trace_id,namespace LIMIT $7` diff --git a/internal/queryrows/trace_candidates.go b/internal/queryrows/trace_candidates.go deleted file mode 100644 index 8d8edb79..00000000 --- a/internal/queryrows/trace_candidates.go +++ /dev/null @@ -1,22 +0,0 @@ -package queryrows - -// CompletedTraceQuery ranks scoped traces from completed batch candidates and -// the uncached/boundary tail. Arguments are start, end, namespace twice, service -// twice, and limit. Both top-N stages must use the same limit. Results are -// namespace, trace_id, and the selected scope's has_error flag. -const CompletedTraceQuery = `WITH tail AS ( - SELECT namespace,trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, - max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error - FROM trace_tail - WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY namespace,trace_id -), untouched AS ( - SELECT namespace,trace_id,min_start,max_end,has_error FROM trace_candidates - WHERE (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM tail) - ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7 -), touched AS ( - SELECT t.namespace,t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, - greatest(t.has_error,c.has_error) AS has_error - FROM tail t LEFT JOIN trace_candidates c ON t.namespace=c.namespace AND t.trace_id=c.trace_id -), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) -SELECT namespace,trace_id,has_error FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7` From e1deec2083717f222662c8bb0fd35ff8d9837680 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 11:47:58 -0800 Subject: [PATCH 5/6] test(engine): time exemplar ranking and cache passes against retention Spread the scale fixture over PITOT_SCALE_HOURS, rank the newest unaligned 24 hours, and report read-cache pass times, exemplar rows, and snapshot binding apart from execution. --- .../trace_candidate_scale_bench_test.go | 94 ++++++++++++++++--- 1 file changed, 79 insertions(+), 15 deletions(-) diff --git a/internal/engine/trace_candidate_scale_bench_test.go b/internal/engine/trace_candidate_scale_bench_test.go index 5c8b7276..17ce62ac 100644 --- a/internal/engine/trace_candidate_scale_bench_test.go +++ b/internal/engine/trace_candidate_scale_bench_test.go @@ -15,10 +15,13 @@ import ( telemetrystore "github.com/labstack/pitot/internal/telemetry/store" ) -// Time the completed-trace kernel as retained traces grow, with a raw tail of -// two uncached recent batches like live ingest. PITOT_SCALE_BATCHES sets the -// fully cached batches of 20,000 two-span traces each; PITOT_SCALE_MIXED=1 puts -// each trace's spans in two services, as real request traces do. +// Time the completed-trace ranking over an unaligned 24-hour window as +// retained traces grow, with a raw tail of two uncached recent batches like +// live ingest, and the read-cache passes that build its rows. +// PITOT_SCALE_BATCHES sets the fully cached batches of 20,000 two-span traces +// each, spread over PITOT_SCALE_HOURS (default 24) so that retention can exceed +// the window; PITOT_SCALE_MIXED=1 puts each trace's spans in two services, as +// real request traces do. The window is the newest 24 hours. func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { mixed := os.Getenv("PITOT_SCALE_MIXED") == "1" batches := 20 @@ -27,12 +30,18 @@ func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { } const traces = 20000 d, repo := newBatchCacheTest(t) - base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + hours := 24 + if v, err := strconv.Atoi(os.Getenv("PITOT_SCALE_HOURS")); err == nil && v >= 24 { + hours = v + } + retained := time.Duration(hours) * time.Hour + base := time.Now().UTC().Add(-retained - time.Hour).Truncate(time.Second) + spacing := retained / time.Duration(batches+2) commit := func(part int) { batch := telemetrystore.Batch{ID: fmt.Sprintf("part_%d", part)} for i := range traces { id := uint64(part*traces+i+1) * 0x9e3779b97f4a7c15 - start := base.Add(time.Duration(part)*time.Second + time.Duration(i%1000)*time.Millisecond) + start := base.Add(time.Duration(part)*spacing + time.Duration(i%1000)*time.Millisecond) status := "STATUS_CODE_OK" if i%97 == 0 { status = "STATUS_CODE_ERROR" @@ -56,20 +65,39 @@ func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { commit(part) } // One pass stops at the row budget; repeat until every batch is cached. - for { - rows, err := d.RefreshReadCaches(t.Context()) - if err != nil { - t.Fatal(err) - } - if rows == 0 { - break + var passes []float64 + refresh := func() { + for { + started := time.Now() + rows, err := d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) + } + if rows == 0 { + return + } + passes = append(passes, float64(time.Since(started))/float64(time.Millisecond)) } } + refresh() + first, last := passes[:min(5, len(passes))], passes[max(0, len(passes)-5):] + var exemplars int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars`).Scan(&exemplars); err != nil { + t.Fatal(err) + } + // The pass that caches two more batches over the whole retained history. + for part := batches + 2; part < batches+4; part++ { + commit(part) + } + passes = passes[:0] + refresh() + t.Logf("passes first_ms=%.0f last_ms=%.0f exemplar_rows=%d two_batch_pass_ms=%.0f", first, last, exemplars, passes) // Recent batches the read cache has not reached yet. for part := batches; part < batches+2; part++ { commit(part) } - w := queryrows.Window{Start: base.Add(-time.Hour), End: base.Add(2 * time.Hour), Namespace: "prod", Service: "api", Kind: queryrows.TraceCandidateRead} + start := base.Add(retained - 24*time.Hour + 17*time.Second) + w := queryrows.Window{Start: start, End: start.Add(24 * time.Hour), Namespace: "prod", Service: "api", Kind: queryrows.TraceCandidateRead} args := []any{w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, 21} run := func(query string) []string { rows, err := d.QueryContext(queryrows.WithWindow(t.Context(), w), query, args...) @@ -102,5 +130,41 @@ func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { ms = append(ms, float64(time.Since(started))/float64(time.Millisecond)) } sort.Float64s(ms) - t.Logf("cached_traces=%d tail_traces=%d mixed_services=%v median_ms=%.1f max_ms=%.1f", batches*traces, 2*traces, mixed, ms[samples/2], ms[samples-1]) + // Split snapshot binding from execution of the bound statement. + pin := d.repository.Parquet.Pin() + defer d.repository.Parquet.Unpin(pin) + tx, err := d.DB.BeginTx(t.Context(), nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + var bind, exec []float64 + var bound string + for range samples { + started := time.Now() + bound, err = d.bindSnapshot(t.Context(), tx, queryrows.CompletedTraceQuery, w) + if err != nil { + t.Fatal(err) + } + bind = append(bind, float64(time.Since(started))/float64(time.Millisecond)) + started = time.Now() + rows, err := tx.QueryContext(t.Context(), bound, args...) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + } + rows.Close() + exec = append(exec, float64(time.Since(started))/float64(time.Millisecond)) + } + sort.Float64s(bind) + sort.Float64s(exec) + if os.Getenv("PITOT_SCALE_EXPLAIN") == "1" { + var key, plan string + if err := tx.QueryRowContext(t.Context(), "EXPLAIN ANALYZE "+bound, args...).Scan(&key, &plan); err != nil { + t.Fatal(err) + } + t.Log(plan) + } + t.Logf("retained_hours=%d cached_traces=%d tail_traces=%d mixed_services=%v median_ms=%.1f max_ms=%.1f bind_ms=%.1f exec_ms=%.1f", hours, batches*traces, 2*traces, mixed, ms[samples/2], ms[samples-1], bind[samples/2], exec[samples/2]) } From 7f1201696f334151fc128014c3898eeb882d9265 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 12:05:11 -0800 Subject: [PATCH 6/6] test(engine): cover same-scope retirement and pre-epoch exemplar minutes Retire an error span from the same namespace and service as the trace's kept span and assert its rank and error flag drop. Rank windows across negative minute boundaries against brute force and check each row's floored minute. Build the trace_exemplars source only for statements that reference it. --- internal/engine/batch_cache.go | 8 ++-- internal/engine/file_snapshot.go | 6 ++- internal/engine/file_snapshot_test.go | 31 ++++++------- internal/engine/trace_exemplars_test.go | 59 +++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index e1a2b6dd..e1bce3b2 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -362,7 +362,7 @@ func timeLiteral(t time.Time) string { func windowPredicate(column string, w queryrows.Window) string { return column + ">=" + timeLiteral(w.Start) + " AND " + column + "<" + timeLiteral(w.End) } -func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []telemetry.BatchMetadata, w queryrows.Window, inventory bool, sources map[string]string) error { +func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []telemetry.BatchMetadata, w queryrows.Window, inventory, traces bool, sources map[string]string) error { markers, err := batchMarkers(ctx, db) if err != nil { return err @@ -390,10 +390,10 @@ func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []t queryrows.RecordCoverage(ctx, fresh) sources["resource_inventory"] = d.resourceInventorySource(batches, markers, window) } - if w.Kind == queryrows.RawRead { - return nil + // Only statements that rank traces pay for the per-batch exemplar source. + if traces { + sources["trace_exemplars"] = d.traceExemplarSource(batches, markers, w) } - sources["trace_exemplars"] = d.traceExemplarSource(batches, markers, w) return nil } diff --git a/internal/engine/file_snapshot.go b/internal/engine/file_snapshot.go index dec9a64a..e2ea4337 100644 --- a/internal/engine/file_snapshot.go +++ b/internal/engine/file_snapshot.go @@ -209,8 +209,10 @@ func (d *Duck) bindSnapshot(ctx context.Context, db snapshotSQL, query string, w sources[signal] = cleanSource(signal, physical) } _, inventory := compiled.names["resource_inventory"] - if w.Kind != queryrows.RawRead || inventory { - if err := d.aggregateSources(ctx, db, batches, w, inventory, sources); err != nil { + _, traces := compiled.names["trace_exemplars"] + traces = traces && w.Kind != queryrows.RawRead + if traces || inventory { + if err := d.aggregateSources(ctx, db, batches, w, inventory, traces, sources); err != nil { return "", err } } diff --git a/internal/engine/file_snapshot_test.go b/internal/engine/file_snapshot_test.go index fa9a191a..a259dd4d 100644 --- a/internal/engine/file_snapshot_test.go +++ b/internal/engine/file_snapshot_test.go @@ -605,12 +605,13 @@ func TestPreparedReplacementDoesNotHoldCacheWriteGate(t *testing.T) { } // Retention removes a retired batch's spans from a trace's score: the shared -// trace loses its retired error span and ranks below the remaining winner. +// trace loses its retired error span, in the same scope as its kept span, and +// ranks below the remaining winner. func TestRetentionRemovesRetiredSpansFromTraceRanking(t *testing.T) { d, repo := newBatchCacheTest(t) at := time.Now().UTC().Truncate(time.Hour) old := at.Add(-2 * time.Hour).UnixNano() - if err := commitRestoredFixture(repo, t.Context(), telemetrystore.Batch{ID: "retire-scope", Spans: []telemetry.Span{{Namespace: "gone", ServiceName: "retired", TraceID: "shared", StartUnixNanos: old, EndUnixNanos: old + 1, StatusCode: "STATUS_CODE_ERROR", IngestedAt: old}}}); err != nil { + if err := commitRestoredFixture(repo, t.Context(), telemetrystore.Batch{ID: "retire-scope", Spans: []telemetry.Span{{Namespace: "prod", ServiceName: "api", TraceID: "shared", StartUnixNanos: old, EndUnixNanos: old + 1, StatusCode: "STATUS_CODE_ERROR", IngestedAt: old}}}); err != nil { t.Fatal(err) } now := at.UnixNano() @@ -623,27 +624,27 @@ func TestRetentionRemovesRetiredSpansFromTraceRanking(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - if count, err := repo.PruneParquet(t.Context(), d, at.Add(-time.Hour).UnixNano(), 8); err != nil || count != 1 { - t.Fatalf("prune=%d: %v", count, err) - } - check := func() { + check := func(want []oracleRow) { t.Helper() - for _, ns := range []string{"", "prod"} { - w := queryrows.Window{Start: at.Add(-3 * time.Hour), End: at.Add(time.Minute), Kind: queryrows.TraceCandidateRead, Namespace: ns} - got := completedTraces(t, d, w, traceExemplarDepth) - want := []oracleRow{{"prod", "winner", 0}, {"prod", "shared", 0}} - if !reflect.DeepEqual(got, want) { - t.Fatalf("namespace %q ranking=%v", ns, got) + for _, scope := range [][2]string{{"", ""}, {"prod", ""}, {"prod", "api"}} { + w := queryrows.Window{Start: at.Add(-3 * time.Hour), End: at.Add(time.Minute), Kind: queryrows.TraceCandidateRead, Namespace: scope[0], Service: scope[1]} + if got := completedTraces(t, d, w, traceExemplarDepth); !reflect.DeepEqual(got, want) { + t.Fatalf("scope %v ranking=%v want %v", scope, got, want) } } } - check() + check([]oracleRow{{"prod", "shared", 1}, {"prod", "winner", 0}}) + if count, err := repo.PruneParquet(t.Context(), d, at.Add(-time.Hour).UnixNano(), 8); err != nil || count != 1 { + t.Fatalf("prune=%d: %v", count, err) + } + retained := []oracleRow{{"prod", "winner", 0}, {"prod", "shared", 0}} + check(retained) if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - check() + check(retained) var retired int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE namespace='gone'`).Scan(&retired); err != nil || retired != 0 { + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE batch_id='retire-scope'`).Scan(&retired); err != nil || retired != 0 { t.Fatalf("retired exemplars=%d: %v", retired, err) } } diff --git a/internal/engine/trace_exemplars_test.go b/internal/engine/trace_exemplars_test.go index 02014d6d..199a7240 100644 --- a/internal/engine/trace_exemplars_test.go +++ b/internal/engine/trace_exemplars_test.go @@ -418,3 +418,62 @@ func TestCompletedTraceQueryMatchesBruteForce(t *testing.T) { t.Fatalf("orphaned exemplar rows=%d error=%v", stale, err) } } + +// Minutes before the epoch use floor division in both Go and SQL: rows land +// in the minute their start belongs to, and windows across negative minute +// boundaries rank exactly. +func TestCompletedTraceQueryMatchesBruteForceBeforeEpoch(t *testing.T) { + d, repo := newBatchCacheTest(t) + r := rand.New(rand.NewSource(5)) + for b := range 3 { + batch := telemetrystore.Batch{ID: fmt.Sprintf("pre%d", b)} + for i := range 900 { + // Starts from -180s to +60s on a 5s grid, never exactly zero. + start := int64(r.Intn(48)-36)*5*int64(time.Second) + int64(r.Intn(2)) + if start == 0 { + start = -1 + } + status := "STATUS_CODE_OK" + if r.Intn(30) == 0 { + status = "STATUS_CODE_ERROR" + } + dur := int64(1+r.Intn(9)) * int64(time.Millisecond) + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "prod", ServiceName: "api", TraceID: fmt.Sprintf("trace-%03d", r.Intn(300)), + SpanID: fmt.Sprintf("%d-%d", b, i), StartUnixNanos: start, EndUnixNanos: start + dur, DurationMS: float64(dur) / 1e6, StatusCode: status, IngestedAt: int64(b + 1)}) + } + if err := commitRestoredFixture(repo, t.Context(), batch); err != nil { + t.Fatal(err) + } + } + for { + rows, err := d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) + } + if rows == 0 { + break + } + } + var misplaced int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_exemplars WHERE minute>start_unix_nano OR minute+60000000000<=start_unix_nano OR minute%60000000000<>0`).Scan(&misplaced); err != nil || misplaced != 0 { + t.Fatalf("misplaced minutes=%d: %v", misplaced, err) + } + for _, s := range []int64{-180, -125, -120, -61, -60, -35, -1} { + if got := floorMinute(s * int64(time.Second)); got > s*int64(time.Second) || got%minuteNanos != 0 || s*int64(time.Second)-got >= minuteNanos { + t.Fatalf("floorMinute(%ds)=%d", s, got) + } + } + raw := rawOracleSpans(t, d) + for _, bounds := range [][2]int64{{-180, 60}, {-125, -5}, {-120, -60}, {-95, -65}, {-65, -55}, {-62, 3}, {-1, 30}, {-170, -61}} { + for _, limit := range []int{1, traceExemplarDepth} { + for _, service := range []string{"", "api"} { + w := queryrows.Window{Start: time.Unix(0, bounds[0]*int64(time.Second)).UTC(), End: time.Unix(0, bounds[1]*int64(time.Second)).UTC(), Namespace: "prod", Service: service} + got := completedTraces(t, d, w, limit) + want := bruteCompletedTraces(raw, w.Start.UnixNano(), w.End.UnixNano(), "prod", service, limit) + if !reflect.DeepEqual(got, want) { + t.Fatalf("window=%v limit=%d service=%q:\n got %v\nwant %v", bounds, limit, service, got, want) + } + } + } + } +}