diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd0cc52e53..18adf16c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -543,7 +543,7 @@ jobs: MAPLE_REQUIRE_LIBCHDB: "1" MAPLE_TELEMETRY: off run: | - for file in tenant-isolation scoped-delete local-service-map-rollup checkpoint-merge-freeze local-store-traces-rekey local-query-guard local-query-guard.fuzz funnel-scenarios users-identity histogram-metrics exponential-histogram-metrics release-verification profile-sample-types; do + for file in tenant-isolation scoped-delete local-service-map-rollup checkpoint-merge-freeze local-store-traces-rekey local-query-guard local-query-guard.fuzz funnel-scenarios users-identity histogram-metrics exponential-histogram-metrics release-verification profile-sample-types service-usage-profiles; do bun test --timeout 30000 "test/$file.test.ts" done diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 48ad8a6bcc..4bdf32a55b 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -232,6 +232,7 @@ describe("current local schema identity", () => { "service_map_db_edges_minutely_mv", "service_runtimes_hourly", "service_runtimes_hourly_mv", + "service_usage_profiles_mv", "source_usage_hourly", "source_usage_logs_mv", "source_usage_metrics_exp_histogram_mv", @@ -389,6 +390,7 @@ describe("current local schema identity", () => { "service_map_db_edges_minutely_mv", "service_runtimes_hourly", "service_runtimes_hourly_mv", + "service_usage_profiles_mv", "source_usage_hourly", "source_usage_logs_mv", "source_usage_metrics_exp_histogram_mv", diff --git a/apps/cli/test/service-usage-profiles.test.ts b/apps/cli/test/service-usage-profiles.test.ts new file mode 100644 index 0000000000..8bd465a897 --- /dev/null +++ b/apps/cli/test/service-usage-profiles.test.ts @@ -0,0 +1,105 @@ +// Profiles reach service_usage on a real ClickHouse engine: the view counts each stack row and +// estimates its bytes from its frames, per org, service and hour, beside the other signals. + +import { afterAll, beforeAll, expect, test } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as CH from "@maple/query-engine/ch" +import { Schema, Array as Arr, String as Str } from "effect" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" +import { localSchemaSnapshot } from "@maple/local-server/local-schema-history" +import { describeWithChdb } from "./libchdb-test-support" +import { migrationStep, openStore } from "./local-store-step-test-support" + +const ORG = "org_service_usage_profiles" +/** An hour ago on the hour: recent, so every TTL keeps it. */ +const HOUR_MS = Math.floor(Date.now() / 3_600_000) * 3_600_000 - 3_600_000 + +const seconds = (ms: number) => Str.slice(0, 19)(Str.replace("T", " ")(new Date(ms).toISOString())) + +const insertProfile = (db: Chdb, orgId: string, atMs: number, frames: ReadonlyArray) => { + const list = (items: ReadonlyArray) => `[${Arr.join(items, ", ")}]` + db.exec( + `INSERT INTO profiles (OrgId, Timestamp, ServiceName, ProfileId, SampleType, SampleUnit, Value, Samples, Frames, FrameFiles, FrameLines) VALUES ('${orgId}', '${seconds(atMs)}', 'checkout', 'p1', 'cpu', 'nanoseconds', 10, 1, ${list(Arr.map(frames, (frame) => `'${frame}'`))}, ${list(Arr.map(frames, () => "''"))}, ${list(Arr.map(frames, () => "0"))})`, + ) +} + +const UsageRow = Schema.Struct({ + hour: Schema.String, + profileRows: CH.CHNumber, + profileBytes: CH.CHNumber, + logBytes: CH.CHNumber, +}) + +const usage = (db: Chdb, orgId: string) => + decodeJsonEachRow(UsageRow)( + db.query( + `SELECT toString(Hour) AS hour, sum(ProfileRowCount) AS profileRows, sum(ProfileSizeBytes) AS profileBytes, sum(LogSizeBytes) AS logBytes FROM service_usage WHERE OrgId = '${orgId}' AND ServiceName = 'checkout' GROUP BY Hour ORDER BY Hour`, + ), + ) + +describeWithChdb("service_usage profiles view", () => { + let root = "" + let db: Chdb + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "maple-service-usage-profiles-")) + db = Chdb.open({ dataDir: join(root, "data"), schemaSql }) + }) + + afterAll(() => { + db.close() + rmSync(root, { recursive: true, force: true }) + }) + + test("counts each stack row and its frame-based bytes into its org's hour", () => { + insertProfile(db, ORG, HOUR_MS + 60_000, ["main", "work"]) + insertProfile(db, ORG, HOUR_MS + 120_000, ["main", "work", "parse"]) + insertProfile(db, ORG, HOUR_MS + 3_600_000 + 60_000, ["main"]) + insertProfile(db, "org_someone_else", HOUR_MS + 60_000, ["main"]) + + const rows = usage(db, ORG) + // 16 bytes per frame plus 200 per row: (2 * 16 + 200) + (3 * 16 + 200), then 1 * 16 + 200. + expect(rows).toEqual([ + { hour: seconds(HOUR_MS), profileRows: 2, profileBytes: 480, logBytes: 0 }, + { hour: seconds(HOUR_MS + 3_600_000), profileRows: 1, profileBytes: 216, logBytes: 0 }, + ]) + }) +}) + +describeWithChdb("v59 -> v60 service_usage profile columns", () => { + const STEP_ID = "local-0059-to-0060-service-usage-profiles" + const MIGRATED_ORG = "local" + + test("keeps existing usage hours and counts the profiles ingested after it", async () => { + const root = await mkdtemp(join(tmpdir(), "maple-service-usage-profiles-step-")) + const sourceDataDir = join(root, "source", "data") + const targetDataDir = join(root, "target", "data") + const db = Chdb.open({ dataDir: sourceDataDir, schemaSql: localSchemaSnapshot(59).sql }) + db.exec( + `INSERT INTO service_usage (OrgId, ServiceName, Hour, LogCount, LogSizeBytes) VALUES ('${MIGRATED_ORG}', 'checkout', '${seconds(HOUR_MS)}', 3, 900)`, + ) + db.close() + + const { migration, context } = migrationStep(STEP_ID, sourceDataDir, targetDataDir, Date.now()) + const state = await migration.preflight(context) + await migration.prepareTarget(context, state) + const progress = await migration.apply(context, state, undefined) + await migration.verify(context, state, progress) + + const rows = await openStore( + targetDataDir, + (db) => { + insertProfile(db, MIGRATED_ORG, HOUR_MS + 60_000, ["main"]) + return usage(db, MIGRATED_ORG) + }, + { schemaSql: localSchemaSnapshot(60).sql, bootstrapSchema: false }, + ) + await rm(root, { recursive: true, force: true }) + expect(rows).toEqual([{ hour: seconds(HOUR_MS), profileRows: 1, profileBytes: 216, logBytes: 900 }]) + }) +}) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index cc6e85864b..cd41d69c0b 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162"; +pub const PROJECT_REVISION: &str = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/backend/src/services/billing/DailySpendService.test.ts b/packages/backend/src/services/billing/DailySpendService.test.ts new file mode 100644 index 0000000000..73dafa62c4 --- /dev/null +++ b/packages/backend/src/services/billing/DailySpendService.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "@effect/vitest" +import { Array as Arr, Effect, Layer, Schema, String as Str } from "effect" +import { OrgId, UserId, WarehouseSchemaDriftError, type TenantContext } from "@maple/domain/http" +import { compiledQueryOf } from "@maple/query-engine/execution" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { DailySpendService } from "./DailySpendService" + +// The billing chart's daily volume: profile bytes from service_usage become profilesGB, and a +// cluster whose service_usage predates the profile columns still answers, with no profiles. + +const tenant: TenantContext = { + orgId: Schema.decodeSync(OrgId)("org_daily_spend"), + userId: Schema.decodeSync(UserId)("user_admin"), + roles: ["org:admin"], + authMode: "self_hosted", +} + +const cycle = { + startMs: Date.parse("2026-07-01T00:00:00Z"), + endMs: Date.parse("2026-07-03T23:59:59Z"), + nowMs: Date.parse("2026-07-02T12:00:00Z"), +} + +const drift = (column: string) => + new WarehouseSchemaDriftError({ + message: `Code: 47. Unknown expression identifier \`${column}\` in scope SELECT ...`, + pipeName: "billingDailySignalVolume", + clickhouseType: "UNKNOWN_IDENTIFIER", + }) + +const signalRow = (day: string, profileBytes: string | number) => ({ + day: `${day} 00:00:00`, + logBytes: "2000000000", + traceBytes: "0", + metricBytes: "0", + profileBytes, +}) + +/** Answers the signal read with `signalRows`; `failWith` fails any SQL naming the profile columns. */ +const layer = (signalRows: ReadonlyArray>, failWith?: WarehouseSchemaDriftError) => + Layer.effect(DailySpendService, DailySpendService.make).pipe( + Layer.provide( + Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + compiledQuery: (_tenant, input, options) => { + const compiled = compiledQueryOf(input) + if (failWith !== undefined && Str.includes("ProfileSizeBytes")(compiled.sql)) + return Effect.fail(failWith) + const rows = options?.context === "billingDailySignalVolume" ? signalRows : [] + return compiled.decodeRows(rows).pipe(Effect.orDie) + }, + }), + ), + ), + ) + +describe("DailySpendService", () => { + it.effect("reports each day's profile bytes as profilesGB, and 0 on days without any", () => + Effect.gen(function* () { + const spend = yield* DailySpendService + const response = yield* spend.get(tenant, cycle) + + expect( + Arr.map(response.days, (day) => ({ + date: day.date, + logsGB: day.logsGB, + profilesGB: day.profilesGB, + })), + ).toEqual([ + { date: "2026-07-01", logsGB: 2, profilesGB: 1.5 }, + { date: "2026-07-02", logsGB: 0, profilesGB: 0 }, + { date: "2026-07-03", logsGB: 2, profilesGB: 0.25 }, + ]) + }).pipe( + Effect.provide( + layer([signalRow("2026-07-01", "1500000000"), signalRow("2026-07-03", "250000000")]), + ), + ), + ) + + it.effect("falls back to no profile usage when service_usage predates the profile columns", () => + Effect.gen(function* () { + const spend = yield* DailySpendService + const response = yield* spend.get(tenant, cycle) + + expect(Arr.map(response.days, (day) => [day.date, day.logsGB, day.profilesGB])).toEqual([ + ["2026-07-01", 2, 0], + ["2026-07-02", 0, 0], + ["2026-07-03", 0, 0], + ]) + }).pipe(Effect.provide(layer([signalRow("2026-07-01", 0)], drift("ProfileSizeBytes")))), + ) + + it.effect("still fails the chart on drift that names another column", () => + Effect.gen(function* () { + const spend = yield* DailySpendService + const error = yield* Effect.flip(spend.get(tenant, cycle)) + + expect(error.pipeName).toBe("billingDailySpend") + expect(error.message).toContain("LogSizeBytes") + }).pipe(Effect.provide(layer([], drift("LogSizeBytes")))), + ) +}) diff --git a/packages/backend/src/services/billing/DailySpendService.ts b/packages/backend/src/services/billing/DailySpendService.ts index 314afd417a..958bb75de3 100644 --- a/packages/backend/src/services/billing/DailySpendService.ts +++ b/packages/backend/src/services/billing/DailySpendService.ts @@ -2,14 +2,17 @@ import { DailySpendResponse, DailyVolume, WarehouseQueryError, type TenantContex import { CH, formatWarehouseDateTime } from "@maple/query-engine" import { Context, DateTime, Effect, Layer, String as Str } from "effect" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" -import { isMissingProductEvents } from "@maple/backend/services/warehouse/missing-table" +import { + isMissingProductEvents, + isMissingServiceUsageProfiles, +} from "@maple/backend/services/warehouse/missing-table" import * as Integrations from "@maple/query-engine-integrations" /** * The daily volume series behind the billing page's cumulative spend chart. * * Autumn reports cycle totals only, so the daily shape comes from the warehouse: - * `service_usage` for log/trace/metric bytes and `session_replays` for browser + * `service_usage` for log/trace/metric/profile bytes and `session_replays` for browser * sessions. Volume only — this service never prices anything. Dollars are the * client's job, computed from the same catalog rates the cost breakdown uses, so * the chart and the invoice can't tell two stories. @@ -68,12 +71,17 @@ export class DailySpendService extends Context.Service + warehouse.compiledQuery( + tenant, + CH.compile(Integrations.dailySignalVolumeQuery({ profiles }), params), + { profile: "list", context: "billingDailySignalVolume" }, + ) + // A cluster whose service_usage predates migration 0068 has no profile bytes to show. + const signalRows = yield* signalVolume(true).pipe( + Effect.catchIf(isMissingServiceUsageProfiles, () => signalVolume(false)), + Effect.mapError(toQueryError), + ) const sessionRows = yield* warehouse .compiledQuery(tenant, CH.compile(Integrations.dailySessionCountQuery(), params), { @@ -82,12 +90,16 @@ export class DailySpendService extends Context.Service() + const byDay = new Map< + string, + { logsGB: number; tracesGB: number; metricsGB: number; profilesGB: number } + >() for (const row of signalRows) { byDay.set(toUtcDateKey(DateTime.toEpochMillis(row.day)), { logsGB: row.logBytes / BYTES_PER_BILLED_GB, tracesGB: row.traceBytes / BYTES_PER_BILLED_GB, metricsGB: row.metricBytes / BYTES_PER_BILLED_GB, + profilesGB: row.profileBytes / BYTES_PER_BILLED_GB, }) } @@ -125,6 +137,7 @@ export class DailySpendService extends Context.Service> = { service_usage: [ "Hourly per-service rollup of STORED rows, 365-day TTL — far longer than the 30-day `logs`/`traces` retention, so it is the right table for long-range volume trends.", "`Hour` is a top-of-hour DateTime. Snap both range bounds to their hour floor (`toStartOfHour`) or sub-hour windows return no rows at all; this necessarily over-reports at the window edges.", - "The `*SizeBytes` columns are STORAGE ESTIMATES, not measured bytes — the materialized views compute them as `length(Body) + 200` (logs), `length(SpanName) + 300` (spans) and flat per-point constants (metrics). They will NOT reconcile with an Autumn invoice and must never be presented as billed usage.", + "The `*SizeBytes` columns are STORAGE ESTIMATES, not measured bytes: the materialized views compute them as `length(Body) + 200` (logs), `length(SpanName) + 300` (spans), flat per-point constants (metrics) and `length(Frames) * 16 + 200` per stack row (profiles, `ProfileRowCount` / `ProfileSizeBytes`, 0 before migration 0068). They will NOT reconcile with an Autumn invoice and must never be presented as billed usage.", "For real billed bytes use `metrics_sum` / `maple_ingest_org_bytes_total`, which records the decoded payload size the gateway actually metered. Counts here are also post-sampling and post-drop, i.e. what was stored, not what was accepted.", "`TraceCount` counts stored SPANS (not distinct traces) and `LogCount` log records; there is no `SpanCount` column.", "Metric counts are split across four columns by point type (`SumMetricCount`, `GaugeMetricCount`, `HistogramMetricCount`, `ExpHistogramMetricCount`) — add all four for a total metric-point count.", diff --git a/packages/domain/src/clickhouse/migrations/0068_service_usage_profiles.ts b/packages/domain/src/clickhouse/migrations/0068_service_usage_profiles.ts new file mode 100644 index 0000000000..d92fde24f1 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0068_service_usage_profiles.ts @@ -0,0 +1,27 @@ +// Migration 0068: profile stack rows and estimated bytes on `service_usage`, fed by a view from `profiles`, so the +// billing page's daily usage includes continuous profiling. The six signal views are recreated to write 0 to the +// new columns. No backfill: earlier hours hold 0. Frozen v68 DDL. +export const migration_0068_service_usage_profiles = { + version: 68, + description: + "Add ProfileRowCount and ProfileSizeBytes to service_usage, recreate its signal views and create its view from profiles", + requiredForIngest: false, + statements: [ + "ALTER TABLE service_usage ADD COLUMN IF NOT EXISTS ProfileRowCount UInt64 DEFAULT 0", + "ALTER TABLE service_usage ADD COLUMN IF NOT EXISTS ProfileSizeBytes UInt64 DEFAULT 0", + "DROP VIEW IF EXISTS service_usage_logs_mv", + "DROP VIEW IF EXISTS service_usage_traces_mv", + "DROP VIEW IF EXISTS service_usage_metrics_sum_mv", + "DROP VIEW IF EXISTS service_usage_metrics_gauge_mv", + "DROP VIEW IF EXISTS service_usage_metrics_histogram_mv", + "DROP VIEW IF EXISTS service_usage_metrics_exp_histogram_mv", + "DROP VIEW IF EXISTS service_usage_profiles_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_profiles_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n count() AS ProfileRowCount,\n sum(length(Frames) * 16 + 200) AS ProfileSizeBytes\n FROM profiles\n GROUP BY OrgId, ServiceName, Hour", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index ed73a65349..16d9230212 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -17,6 +17,7 @@ import { migration_0034_trace_facets_hourly, traceFacetsHourlyBackfill } from ". import { migration_0037_service_overview_spans_span_kind } from "./0037_service_overview_spans_span_kind" import { migration_0038_external_edges_legacy_db_system } from "./0038_external_edges_legacy_db_system" import { migration_0045_ttl_only_drop_parts } from "./0045_ttl_only_drop_parts" +import { migration_0068_service_usage_profiles } from "./0068_service_usage_profiles" import { latestSnapshotStatements } from "../../generated/clickhouse-schema" import { migrations } from "./index" @@ -389,6 +390,27 @@ describe("migration 0038: legacy db.system spans stay off the external edges", ( }) }) +describe("migration 0068: service_usage counts profile rows", () => { + it("adds the columns, then recreates every service_usage view from the snapshot, outside the ingest gate", () => { + const statements = migration_0068_service_usage_profiles.statements + const serviceUsageViews = Arr.filter(latestSnapshotStatements, (stmt) => + Str.includes(" TO service_usage AS")(stmt), + ) + expect(statements.slice(0, 2)).toEqual([ + "ALTER TABLE service_usage ADD COLUMN IF NOT EXISTS ProfileRowCount UInt64 DEFAULT 0", + "ALTER TABLE service_usage ADD COLUMN IF NOT EXISTS ProfileSizeBytes UInt64 DEFAULT 0", + ]) + const drops = Arr.filter(statements, (stmt) => Str.startsWith("DROP VIEW")(stmt)) + const creates = Arr.filter(statements, (stmt) => Str.startsWith("CREATE MATERIALIZED VIEW")(stmt)) + // Every view writing service_usage must emit the new columns, so all of them are replaced. + expect(drops).toHaveLength(serviceUsageViews.length) + expect([...creates].sort()).toEqual([...serviceUsageViews].sort()) + expect(statements).toEqual([...statements.slice(0, 2), ...drops, ...creates]) + for (const view of creates) expect(view).toContain("AS ProfileRowCount") + expect(migration_0068_service_usage_profiles.requiredForIngest).toBe(false) + }) +}) + describe("migration 0045: TTL eviction drops whole parts", () => { const tables = Arr.flatMap(latestSnapshotStatements, (stmt) => { const name = firstGroup(/^CREATE TABLE IF NOT EXISTS (\w+) \(/)(stmt) diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 9bbf12df02..24b7e1e918 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -66,6 +66,7 @@ import { migration_0064_audit_trail_events } from "./0064_audit_trail_events" import { migration_0065_user_activity_identity } from "./0065_user_activity_identity" import { migration_0066_aws_metrics_hourly } from "./0066_aws_metrics_hourly" import { migration_0067_profiles } from "./0067_profiles" +import { migration_0068_service_usage_profiles } from "./0068_service_usage_profiles" import { Array as Arr } from "effect" /** @@ -165,6 +166,7 @@ export const migrations: ReadonlyArray = [ migration_0065_user_activity_identity, migration_0066_aws_metrics_hourly, migration_0067_profiles, + migration_0068_service_usage_profiles, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 3a00e9b26d..f4666838ac 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162" as const +export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -48,7 +48,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS service_overview_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String),\n SpanKind LowCardinality(String) DEFAULT '',\n IsRoot UInt8 DEFAULT 0,\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_runtimes_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n RuntimeName LowCardinality(String),\n RuntimeVersion LowCardinality(String),\n ExecutableName LowCardinality(String),\n SdkLanguage LowCardinality(String),\n SdkName LowCardinality(String),\n SdkVersion LowCardinality(String),\n SpanCount UInt64\n)\nENGINE = SummingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion)\nTTL toDate(Hour) + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replays (\n OrgId LowCardinality(String),\n SessionId String,\n StartTime DateTime64(9),\n EndTime Nullable(DateTime64(9)),\n DurationMs Nullable(UInt32),\n Status LowCardinality(String),\n UserId String,\n UrlInitial String,\n UserAgent String,\n BrowserName LowCardinality(String),\n OsName LowCardinality(String),\n DeviceType LowCardinality(String),\n Country LowCardinality(String) DEFAULT '',\n ServiceName LowCardinality(String),\n PageViews UInt32 DEFAULT 0,\n ClickCount UInt32 DEFAULT 0,\n ErrorCount UInt32 DEFAULT 0,\n TraceIds Array(String) DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String),\n Version UInt32,\n VisitorId String DEFAULT '',\n VisitorIsNew UInt8 DEFAULT 0,\n UserEmail String DEFAULT '',\n UserName String DEFAULT '',\n GroupId String DEFAULT '',\n GroupName String DEFAULT '',\n UserTraits Map(String, String) DEFAULT map(),\n Referrer String DEFAULT '',\n ReferrerHost LowCardinality(String) DEFAULT '',\n UtmSource LowCardinality(String) DEFAULT '',\n UtmMedium LowCardinality(String) DEFAULT '',\n UtmCampaign LowCardinality(String) DEFAULT '',\n UtmTerm String DEFAULT '',\n UtmContent String DEFAULT '',\n Host LowCardinality(String) DEFAULT '',\n EntryPath String DEFAULT '',\n ExitPath String DEFAULT '',\n Language LowCardinality(String) DEFAULT '',\n LastActivityAt Nullable(DateTime64(9))\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(StartTime)\nORDER BY (OrgId, SessionId)\nTTL toDate(StartTime) + INTERVAL 30 DAY", @@ -98,12 +98,13 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['vcs.ref.head.revision'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n SpanKind,\n toUInt8(ParentSpanId = '') AS IsRoot\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_runtimes_hourly_mv TO service_runtimes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n leftUTF8(ResourceAttributes['process.runtime.name'], 64) AS RuntimeName,\n leftUTF8(ResourceAttributes['process.runtime.version'], 64) AS RuntimeVersion,\n leftUTF8(ResourceAttributes['process.executable.name'], 64) AS ExecutableName,\n leftUTF8(ResourceAttributes['telemetry.sdk.language'], 64) AS SdkLanguage,\n leftUTF8(ResourceAttributes['telemetry.sdk.name'], 64) AS SdkName,\n leftUTF8(ResourceAttributes['telemetry.sdk.version'], 64) AS SdkVersion,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_profiles_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n count() AS ProfileRowCount,\n sum(length(Frames) * 16 + 200) AS ProfileSizeBytes\n FROM profiles\n GROUP BY OrgId, ServiceName, Hour", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_logs_mv TO source_usage_hourly AS\nSELECT\n OrgId,\n ResourceAttributes['maple_source_id'] AS SourceId,\n toStartOfHour(TimestampTime) AS Hour,\n 0 AS SpanCount,\n 0 AS SpanSizeBytes,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS MetricCount,\n 0 AS MetricSizeBytes\n FROM logs\n WHERE ResourceAttributes['maple_source_id'] != ''\n GROUP BY OrgId, SourceId, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_exp_histogram_mv TO source_usage_hourly AS\nSELECT\n OrgId,\n ResourceAttributes['maple_source_id'] AS SourceId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS SpanCount,\n 0 AS SpanSizeBytes,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS MetricCount,\n count() * 300 AS MetricSizeBytes\n FROM metrics_exponential_histogram\n WHERE ResourceAttributes['maple_source_id'] != ''\n GROUP BY OrgId, SourceId, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_gauge_mv TO source_usage_hourly AS\nSELECT\n OrgId,\n ResourceAttributes['maple_source_id'] AS SourceId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS SpanCount,\n 0 AS SpanSizeBytes,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS MetricCount,\n count() * 150 AS MetricSizeBytes\n FROM metrics_gauge\n WHERE ResourceAttributes['maple_source_id'] != ''\n GROUP BY OrgId, SourceId, Hour", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index a8ed5e91ac..a976769094 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162" as const +export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const export const datasources = [ { @@ -227,7 +227,7 @@ export const datasources = [ { name: "service_usage", content: - 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT *', + 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes,\n \t\tSumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes,\n \t\tHistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileRowCount,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileSizeBytes', }, { name: "session_events", @@ -480,32 +480,37 @@ export const pipes = [ { name: "service_usage_logs_mv", content: - "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "service_usage_metrics_exp_histogram_mv", content: - "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "service_usage_metrics_gauge_mv", content: - "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "service_usage_metrics_histogram_mv", content: - "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "service_usage_metrics_sum_mv", content: - "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + }, + { + name: "service_usage_profiles_mv", + content: + "DESCRIPTION >\n Materialized view to aggregate continuous profiling usage statistics per service per hour\n\nNODE service_usage_profiles_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n count() AS ProfileRowCount,\n sum(length(Frames) * 16 + 200) AS ProfileSizeBytes\n FROM profiles\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "service_usage_traces_mv", content: - "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", + "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes,\n 0 AS ProfileRowCount,\n 0 AS ProfileSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", }, { name: "source_usage_logs_mv", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index f0a6fec7f2..a9254716cd 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -319,8 +319,18 @@ export const serviceUsage = defineDatasource("service_usage", { HistogramMetricSizeBytes: t.uint64(), ExpHistogramMetricCount: t.uint64(), ExpHistogramMetricSizeBytes: t.uint64(), + // Continuous profiling (migration 0068, appended last). Hours written before it hold 0. + ProfileRowCount: t.uint64().default(0), + ProfileSizeBytes: t.uint64().default(0), }, - forwardQuery: `SELECT *`, + // One-shot: changing the views makes Tinybird rebuild service_usage, and without a forward + // query it would refill it from 14 to 90 day sources. Back to `SELECT *` after the 0068 deploy. + forwardQuery: `SELECT + OrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes, + SumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes, + HistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes, + defaultValueOfTypeName('UInt64') AS ProfileRowCount, + defaultValueOfTypeName('UInt64') AS ProfileSizeBytes`, engine: engine.summingMergeTree({ sortingKey: ["OrgId", "ServiceName", "Hour"], ttl: "Hour + INTERVAL 365 DAY", diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 60d6de29f2..395f03e558 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -174,7 +174,9 @@ export const serviceUsageLogsMv = defineMaterializedView("service_usage_logs_mv" 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM logs GROUP BY OrgId, ServiceName, Hour `, @@ -204,7 +206,9 @@ export const serviceUsageTracesMv = defineMaterializedView("service_usage_traces 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM traces GROUP BY OrgId, ServiceName, Hour `, @@ -234,7 +238,9 @@ export const serviceUsageMetricsSumMv = defineMaterializedView("service_usage_me 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_sum GROUP BY OrgId, ServiceName, Hour `, @@ -264,7 +270,9 @@ export const serviceUsageMetricsGaugeMv = defineMaterializedView("service_usage_ 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_gauge GROUP BY OrgId, ServiceName, Hour `, @@ -294,7 +302,9 @@ export const serviceUsageMetricsHistogramMv = defineMaterializedView("service_us count() AS HistogramMetricCount, count() * 250 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_histogram GROUP BY OrgId, ServiceName, Hour `, @@ -327,7 +337,9 @@ export const serviceUsageMetricsExpHistogramMv = defineMaterializedView( 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, count() AS ExpHistogramMetricCount, - count() * 300 AS ExpHistogramMetricSizeBytes + count() * 300 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_exponential_histogram GROUP BY OrgId, ServiceName, Hour `, @@ -336,6 +348,41 @@ export const serviceUsageMetricsExpHistogramMv = defineMaterializedView( }, ) +// One profiles row is one call stack. Like logs (body + 200), the estimate is the variable part, +// 16 bytes per frame reference, plus 200 for the value, attributes and links. The billing chart +// rescales it to the metered total, so only the daily shape matters. +export const serviceUsageProfilesMv = defineMaterializedView("service_usage_profiles_mv", { + description: "Materialized view to aggregate continuous profiling usage statistics per service per hour", + datasource: serviceUsage, + nodes: [ + node({ + name: "service_usage_profiles_mv_node", + sql: ` + SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + count() AS ProfileRowCount, + sum(length(Frames) * 16 + 200) AS ProfileSizeBytes + FROM profiles + GROUP BY OrgId, ServiceName, Hour + `, + }), + ], +}) + /** * One `source_usage_hourly` row per (org, source, hour) from a signal table. * Consumed by `sourceUsageHourlyQuery` and `sourceUsageTotalsQuery`; byte diff --git a/packages/domain/src/tinybird/materialized-projection-order.test.ts b/packages/domain/src/tinybird/materialized-projection-order.test.ts index 2ff4304fe0..a44dfb77af 100644 --- a/packages/domain/src/tinybird/materialized-projection-order.test.ts +++ b/packages/domain/src/tinybird/materialized-projection-order.test.ts @@ -115,6 +115,13 @@ describe("materialized projection order", () => { it("keeps materialized projections aligned with target datasource column order", () => { const targets = [ ["ai_trace_index", "ai_trace_index_mv"], + ["service_usage", "service_usage_logs_mv"], + ["service_usage", "service_usage_traces_mv"], + ["service_usage", "service_usage_metrics_sum_mv"], + ["service_usage", "service_usage_metrics_gauge_mv"], + ["service_usage", "service_usage_metrics_histogram_mv"], + ["service_usage", "service_usage_metrics_exp_histogram_mv"], + ["service_usage", "service_usage_profiles_mv"], ["ai_crawler_requests", "ai_crawler_requests_mv"], ["scanner_requests", "scanner_requests_mv"], ["service_runtimes_hourly", "service_runtimes_hourly_mv"], diff --git a/packages/local-server/src/local-schema-history.ts b/packages/local-server/src/local-schema-history.ts index 0e9fd007b0..7338bee4a8 100644 --- a/packages/local-server/src/local-schema-history.ts +++ b/packages/local-server/src/local-schema-history.ts @@ -57,6 +57,7 @@ import schemaV56Sql from "./schema/history/local-schema-v56.sql" with { type: "t import schemaV57Sql from "./schema/history/local-schema-v57.sql" with { type: "text" } import schemaV58Sql from "./schema/history/local-schema-v58.sql" with { type: "text" } import schemaV59Sql from "./schema/history/local-schema-v59.sql" with { type: "text" } +import schemaV60Sql from "./schema/history/local-schema-v60.sql" with { type: "text" } import { Array as Arr } from "effect" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LEGACY_LOCAL_SCHEMA, LEGACY_LOCAL_SCHEMA_VERSION, type LocalSchemaIdentity } from "./schema-identity" @@ -130,6 +131,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV57Sql, schemaV58Sql, schemaV59Sql, + schemaV60Sql, ] export interface LocalSchemaSnapshot { diff --git a/packages/local-server/src/local-schema-version.ts b/packages/local-server/src/local-schema-version.ts index 613f688a5f..7803281e54 100644 --- a/packages/local-server/src/local-schema-version.ts +++ b/packages/local-server/src/local-schema-version.ts @@ -1,7 +1,7 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 59 as const +export const LOCAL_SCHEMA_VERSION = 60 as const /** SQLite eventing state has its own independent version sequence. */ export const LOCAL_CONTROL_SCHEMA_VERSION = 1 as const diff --git a/packages/local-server/src/local-store-migrations/steps.ts b/packages/local-server/src/local-store-migrations/steps.ts index 1e9b2ce90c..98e64f710c 100644 --- a/packages/local-server/src/local-store-migrations/steps.ts +++ b/packages/local-server/src/local-store-migrations/steps.ts @@ -2040,6 +2040,49 @@ const STEP_ROWS: ReadonlyArray = [ }, ], }, + { + // Profile stack rows and bytes on service_usage, written by a new view the bootstrap creates; the six + // signal views are dropped first so the bootstrap recreates them writing 0 there. Nothing is backfilled. + slug: "service-usage-profiles", + description: "Add profile counts and bytes to service_usage and create its view from profiles", + clonedBefore: "any DDL runs", + beforeBootstrap: [ + addColumns("service_usage", [ + ["ProfileRowCount", "UInt64 DEFAULT 0"], + ["ProfileSizeBytes", "UInt64 DEFAULT 0"], + ]), + dropViews( + "service_usage_logs_mv", + "service_usage_traces_mv", + "service_usage_metrics_sum_mv", + "service_usage_metrics_gauge_mv", + "service_usage_metrics_histogram_mv", + "service_usage_metrics_exp_histogram_mv", + ), + ], + plan: [ + [ + "add-service-usage-profiles", + "Add ProfileRowCount and ProfileSizeBytes to service_usage, recreate its six signal views and create service_usage_profiles_mv via the v60 bootstrap", + ], + ], + dispositions: [ + { + name: "service_usage", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "The profile columns are added with a 0 default; existing rows keep every value and read no profile usage.", + }, + { + name: "profiles", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The new view's source is neither read nor rewritten; only rows ingested after it count.", + }, + ], + }, // local-schema:bump appends the next step above this line. ] diff --git a/packages/local-server/src/schema/history/local-schema-v60.sql b/packages/local-server/src/schema/history/local-schema-v60.sql new file mode 100644 index 0000000000..78cfbbd705 --- /dev/null +++ b/packages/local-server/src/schema/history/local-schema-v60.sql @@ -0,0 +1,3140 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5 + +CREATE TABLE IF NOT EXISTS ai_crawler_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Crawler LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64, + ResponseId String, + VendorVersion LowCardinality(String), + InputTokens Float64, + CacheReadTokens Float64, + CacheWriteTokens Float64, + OutputTokens Float64, + ReasoningTokens Float64, + ErrorType LowCardinality(String), + StatusMessage String, + ToolDescription String, + FailedToolCallResult String, + ErrorFingerprint UInt64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String), + SkipReason LowCardinality(String) DEFAULT '', + Level LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS audit_trail_events ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + IngestedAt DateTime64(3), + Provider LowCardinality(String), + Account String, + ProviderEventId String, + ActorType LowCardinality(String), + ActorId String, + ActorName String, + ActorEmail String, + Action LowCardinality(String), + Category LowCardinality(String), + Outcome LowCardinality(String), + OutcomeReason String, + TargetType LowCardinality(String), + TargetId String, + TargetName String, + OriginIp String, + OriginCountry LowCardinality(String), + UserAgent String, + Raw String, + Version UInt64, + INDEX idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1 +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 400 DAY; + +CREATE TABLE IF NOT EXISTS aws_metrics_hourly ( + OrgId LowCardinality(String), + MetricType LowCardinality(String), + MetricName LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + Attributes Map(LowCardinality(String), String), + CloudRegion LowCardinality(String), + CloudAccountId LowCardinality(String), + Total SimpleAggregateFunction(sum, Float64), + Samples SimpleAggregateFunction(sum, UInt64), + Peak SimpleAggregateFunction(max, Float64), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, MetricName, Hour, ServiceName, Attributes, CloudRegion, CloudAccountId) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS ci_jobs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + Attempt UInt16, + Job LowCardinality(String), + Result LowCardinality(String), + StatusCode LowCardinality(String), + Duration UInt64, + QueueMs UInt32, + Url String, + FailureReason String, + RunnerName LowCardinality(String), + RunnerLabels LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS ci_runs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + RunNumber UInt64, + Attempt UInt16, + Result LowCardinality(String), + Duration UInt64, + Url String, + Branch String, + Revision String, + TriggerEvent LowCardinality(String), + Actor LowCardinality(String), + PullRequest UInt32, + WorkflowPath LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS cost_daily ( + OrgId LowCardinality(String), + SourceId LowCardinality(String), + Provider LowCardinality(String), + AccountId LowCardinality(String), + Date DateTime, + CloudService LowCardinality(String), + ServiceCategory LowCardinality(String), + UsageType LowCardinality(String), + Region LowCardinality(String), + ResourceId String, + ResourceType LowCardinality(String), + ServiceName LowCardinality(String), + ChargeCategory LowCardinality(String), + Basis LowCardinality(String), + Currency LowCardinality(String), + BilledCost Float64, + EffectiveCost Float64, + ListCost Float64, + ConsumedQuantity Float64, + ConsumedUnit LowCardinality(String), + Tags Map(LowCardinality(String), String), + Estimated UInt8, + IngestedAt DateTime64(3) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(Date) +ORDER BY (OrgId, SourceId, Date, CloudService, UsageType, Region, ResourceId, ChargeCategory, AccountId, ServiceName, Currency) +TTL toDate(Date) + INTERVAL 760 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String), + UserId String DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String), + UserId String DEFAULT '', + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprint_lineage ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + PreviousFingerprintHash UInt64 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, Minute, FingerprintHash, PreviousFingerprintHash) +TTL toDate(Minute) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS profiles ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + ServiceName LowCardinality(String), + ServiceVersion LowCardinality(String) DEFAULT '', + Environment LowCardinality(String) DEFAULT '', + ServiceInstanceId String DEFAULT '', + ResourceAttributes Map(LowCardinality(String), String), + ProfileId String DEFAULT '', + SampleType LowCardinality(String), + SampleUnit LowCardinality(String), + Value Int64, + Samples UInt64 DEFAULT 0, + DurationNano UInt64 DEFAULT 0, + Frames Array(LowCardinality(String)), + FrameFiles Array(LowCardinality(String)), + FrameLines Array(Int32), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + SampleAttributes Map(LowCardinality(String), String), + StackHash UInt64 DEFAULT cityHash64(Frames, FrameFiles, FrameLines), + INDEX idx_span_id SpanId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SampleType, toStartOfHour(Timestamp), StackHash) +TTL toDate(Timestamp) + INTERVAL 14 DAY; + +CREATE TABLE IF NOT EXISTS scanner_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Signature LowCardinality(String), + Scanner LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16, + ClientAddress String, + UserAgent String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS search_console_daily ( + OrgId LowCardinality(String), + SiteUrl LowCardinality(String), + Grain LowCardinality(String), + SearchType LowCardinality(String), + Date DateTime, + Page String, + Query String, + Country LowCardinality(String), + Device LowCardinality(String), + Clicks UInt64, + Impressions UInt64, + PositionSum Float64, + IngestedAt DateTime64(3) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(Date) +ORDER BY (OrgId, SiteUrl, Grain, SearchType, Date, Page, Query, Country, Device) +TTL toDate(Date) + INTERVAL 520 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_endpoint_details_hourly ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + DeploymentEnv LowCardinality(String), + SpanName String, + HttpStatusCode LowCardinality(String), + ErrorType LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + BodySizeQuantiles AggregateFunction(quantilesTDigestIf(0.5, 0.95, 0.99), UInt64, UInt8), + BodySizeMax SimpleAggregateFunction(max, UInt64), + BodyCount SimpleAggregateFunction(sum, UInt64), + UserAgents AggregateFunction(approx_top_k(10), String), + SlowestDuration SimpleAggregateFunction(max, UInt64), + SlowestTraceId AggregateFunction(argMax, String, UInt64), + SlowestTimestamp AggregateFunction(argMax, DateTime64(9), UInt64), + SlowestBodySize AggregateFunction(argMax, Int64, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, SpanName, HttpStatusCode, ErrorType) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), TraceId, ParentSpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DbNamespace LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + FirstInsertedAt SimpleAggregateFunction(min, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, Minute, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Minute) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpStatusCount SimpleAggregateFunction(sum, UInt64), + Http4xxCount SimpleAggregateFunction(sum, UInt64), + Http5xxCount SimpleAggregateFunction(sum, UInt64), + RequestBodyBytes SimpleAggregateFunction(sum, UInt64), + RequestBodyCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpStatusCount SimpleAggregateFunction(sum, UInt64), + Http4xxCount SimpleAggregateFunction(sum, UInt64), + Http5xxCount SimpleAggregateFunction(sum, UInt64), + RequestBodyBytes SimpleAggregateFunction(sum, UInt64), + RequestBodyCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64), + EstimatedLadderSpanCount SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder50Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder100Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder250Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder1000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder2500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder5000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder10000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64), + EstimatedLadderSpanCount SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder50Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder100Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder250Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder1000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder2500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder5000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder10000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + SpanKind LowCardinality(String) DEFAULT '', + IsRoot UInt8 DEFAULT 0, + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_runtimes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + RuntimeName LowCardinality(String), + RuntimeVersion LowCardinality(String), + ExecutableName LowCardinality(String), + SdkLanguage LowCardinality(String), + SdkName LowCardinality(String), + SdkVersion LowCardinality(String), + SpanCount UInt64 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64, + ProfileRowCount UInt64 DEFAULT 0, + ProfileSizeBytes UInt64 DEFAULT 0 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS source_usage_hourly ( + OrgId LowCardinality(String), + SourceId String, + Hour DateTime, + SpanCount UInt64, + SpanSizeBytes UInt64, + LogCount UInt64, + LogSizeBytes UInt64, + MetricCount UInt64, + MetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, SourceId, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_facets_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName String, + HttpMethod LowCardinality(String), + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + HasError UInt8, + TraceCount SimpleAggregateFunction(sum, UInt64), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError) +TTL toDate(Hour) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_index ( + OrgId LowCardinality(String), + TraceId String, + ServiceName LowCardinality(String), + Bucket DateTime, + MinTimestamp SimpleAggregateFunction(min, DateTime64(9)), + MaxTimestamp SimpleAggregateFunction(max, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Bucket) +ORDER BY (OrgId, TraceId, ServiceName, Bucket) +TTL toDate(Bucket) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_name SpanName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, toStartOfFiveMinutes(Timestamp), TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS user_activity_hourly ( + OrgId LowCardinality(String), + ServiceNamespace LowCardinality(String), + UserId String, + Hour DateTime, + ServiceName LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + ErrorSpanCount SimpleAggregateFunction(sum, UInt64), + RequestCount SimpleAggregateFunction(sum, UInt64), + FailedRequestCount SimpleAggregateFunction(sum, UInt64), + TraceIds AggregateFunction(uniqIf, String, UInt8), + SessionIds AggregateFunction(uniqIf, String, UInt8), + RequestDurationQuantiles AggregateFunction(quantilesTDigestIf(0.5, 0.95), UInt64, UInt8), + TopRoutes AggregateFunction(approx_top_k(10), String), + LogCount SimpleAggregateFunction(sum, UInt64), + WarnLogCount SimpleAggregateFunction(sum, UInt64), + ErrorLogCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ClassifiedRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0, + WebRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0, + MobileRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0, + UserName AggregateFunction(argMaxIf, String, UInt32, UInt8), + UserEmail AggregateFunction(argMaxIf, String, UInt32, UInt8), + UserNameAt SimpleAggregateFunction(max, UInt32) DEFAULT 0, + UserEmailAt SimpleAggregateFunction(max, UInt32) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, ServiceNamespace, UserId, Hour, ServiceName) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler, + lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus + FROM traces + WHERE SpanKind = 'Server' + AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0 + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + SpanAttributes['maple_ai.model'] AS Model, + SpanAttributes['maple_ai.agent.name'] AS AgentName, + SpanAttributes['maple_ai.tool.name'] AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError, + toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall, + toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost, + SpanAttributes['maple_ai.response.id'] AS ResponseId, + SpanAttributes['maple_ai.vendor.version'] AS VendorVersion, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens, + SpanAttributes['error.type'] AS ErrorType, + leftUTF8(StatusMessage, 400) AS StatusMessage, + SpanAttributes['maple_ai.tool.description'] AS ToolDescription, + SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult, + if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS aws_metrics_hourly_gauge_mv TO aws_metrics_hourly AS +SELECT + OrgId, + 'gauge' AS MetricType, + MetricName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + Attributes, + ResourceAttributes['cloud.region'] AS CloudRegion, + ResourceAttributes['cloud.account.id'] AS CloudAccountId, + sum(Value) AS Total, + count() AS Samples, + max(Value) AS Peak, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + WHERE startsWith(MetricName, 'aws.') + GROUP BY OrgId, MetricType, MetricName, Hour, ServiceName, Attributes, CloudRegion, CloudAccountId; + +CREATE MATERIALIZED VIEW IF NOT EXISTS aws_metrics_hourly_sum_mv TO aws_metrics_hourly AS +SELECT + OrgId, + 'sum' AS MetricType, + MetricName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + Attributes, + ResourceAttributes['cloud.region'] AS CloudRegion, + ResourceAttributes['cloud.account.id'] AS CloudAccountId, + sum(Value) AS Total, + count() AS Samples, + max(Value) AS Peak, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + WHERE startsWith(MetricName, 'aws.') + GROUP BY OrgId, MetricType, MetricName, Hour, ServiceName, Attributes, CloudRegion, CloudAccountId; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.task.name'] AS Job, + SpanAttributes['cicd.pipeline.task.run.result'] AS Result, + StatusCode, + Duration, + toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs, + SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url, + SpanAttributes['maple.cicd.failure.reason'] AS FailureReason, + SpanAttributes['cicd.worker.name'] AS RunnerName, + SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId != '' + AND SpanAttributes['cicd.pipeline.task.name'] != '' + AND SpanAttributes['maple.cicd.step.number'] = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.result'] AS Result, + Duration, + SpanAttributes['cicd.pipeline.run.url.full'] AS Url, + SpanAttributes['vcs.ref.head.name'] AS Branch, + SpanAttributes['vcs.ref.head.revision'] AS Revision, + SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent, + SpanAttributes['maple.cicd.actor'] AS Actor, + toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest, + SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + _hash AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + _hash AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprint_lineage_mv TO error_fingerprint_lineage AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + _hash AS FingerprintHash, + _prevHash AS PreviousFingerprintHash + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ) + AND _hash != _prevHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS scanner_requests_mv TO scanner_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + if(multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci']) > 0, arrayElement(['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', '/actuator/env', '/actuator/heapdump', '/actuator/configprops', '/actuator/gateway', '/server-status', '/server-info', 'phpinfo', '/info.php', '/_profiler', '/debug/pprof', '/telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci'], multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci'])), arrayElement(['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/'], multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/']))) AS Signature, + arrayElement(['ZGrab', 'Masscan', 'Nuclei', 'sqlmap', 'Nikto', 'Nmap', 'Gobuster', 'DirBuster', 'ffuf', 'WPScan', 'WhatWeb', 'Acunetix', 'Nessus', 'Censys', 'Palo Alto Expanse', 'LeakIX', 'LeakIX', 'Modat', 'Driftnet', 'ARTEX', 'norma agent SDK'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['zgrab', 'masscan', 'nuclei', 'sqlmap', 'nikto', 'nmap', 'gobuster', 'dirbuster', 'fuzz faster u fool', 'wpscan', 'whatweb', 'acunetix', 'nessus', 'censysinspect', 'palo alto networks', 'l9explore', 'l9tcpid', 'modatscanner', 'internetmeasurement', 'artex-enrich', 'norma/'])) AS Scanner, + leftUTF8(lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')), 255) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus, + leftUTF8(coalesce(nullIf(SpanAttributes['client.address'], ''), nullIf(SpanAttributes['http.client_ip'], ''), nullIf(SpanAttributes['net.peer.ip'], ''), ''), 64) AS ClientAddress, + leftUTF8(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), 256) AS UserAgent + FROM traces + WHERE SpanKind = 'Server' + AND (multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci']) > 0 OR multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/']) > 0 OR multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['zgrab', 'masscan', 'nuclei', 'sqlmap', 'nikto', 'nmap', 'gobuster', 'dirbuster', 'fuzz faster u fool', 'wpscan', 'whatweb', 'acunetix', 'nessus', 'censysinspect', 'palo alto networks', 'l9explore', 'l9tcpid', 'modatscanner', 'internetmeasurement', 'artex-enrich', 'norma/']) > 0) + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_endpoint_details_hourly_mv TO service_endpoint_details_hourly AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + SpanAttributes['error.type'] AS ErrorType, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + quantilesTDigestIfState(0.5, 0.95, 0.99)(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodySizeQuantiles, + maxIf(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodySizeMax, + countIf(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodyCount, + approx_top_kState(10)(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), '')) AS UserAgents, + max(Duration) AS SlowestDuration, + argMaxState(TraceId, Duration) AS SlowestTraceId, + argMaxState(Timestamp, Duration) AS SlowestTimestamp, + argMaxState(ifNull(toInt64(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length']))), -1), Duration) AS SlowestBodySize + FROM traces + WHERE SpanKind = 'Server' + AND if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) != '' + GROUP BY OrgId, ServiceName, Hour, DeploymentEnv, SpanName, HttpStatusCode, ErrorType; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_minutely_mv TO service_map_db_edges_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles, + now() AS FirstInsertedAt + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Minute, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount, + sum(HttpClassifiedSpanCount) AS HttpClassifiedSpanCount, + sum(HttpStatusCount) AS HttpStatusCount, + sum(Http4xxCount) AS Http4xxCount, + sum(Http5xxCount) AS Http5xxCount, + sum(RequestBodyBytes) AS RequestBodyBytes, + sum(RequestBodyCount) AS RequestBodyCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount, + count() AS HttpClassifiedSpanCount, + countIf(SpanKind = 'Server' AND if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) != '') AS HttpStatusCount, + countIf(SpanKind = 'Server' AND toUInt16OrZero(if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code'])) BETWEEN 400 AND 499) AS Http4xxCount, + countIf(SpanKind = 'Server' AND toUInt16OrZero(if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code'])) BETWEEN 500 AND 599) AS Http5xxCount, + sumIf(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), SpanKind = 'Server' AND toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS RequestBodyBytes, + countIf(SpanKind = 'Server' AND toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS RequestBodyCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount, + sum(SampleRate) AS EstimatedLadderSpanCount, + sumIf(SampleRate, Duration < 50000000) AS EstimatedUnder50Ms, + sumIf(SampleRate, Duration < 100000000) AS EstimatedUnder100Ms, + sumIf(SampleRate, Duration < 250000000) AS EstimatedUnder250Ms, + sumIf(SampleRate, Duration < 500000000) AS EstimatedUnder500Ms, + sumIf(SampleRate, Duration < 1000000000) AS EstimatedUnder1000Ms, + sumIf(SampleRate, Duration < 2500000000) AS EstimatedUnder2500Ms, + sumIf(SampleRate, Duration < 5000000000) AS EstimatedUnder5000Ms, + sumIf(SampleRate, Duration < 10000000000) AS EstimatedUnder10000Ms + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount, + sum(SampleRate) AS EstimatedLadderSpanCount, + sumIf(SampleRate, Duration < 50000000) AS EstimatedUnder50Ms, + sumIf(SampleRate, Duration < 100000000) AS EstimatedUnder100Ms, + sumIf(SampleRate, Duration < 250000000) AS EstimatedUnder250Ms, + sumIf(SampleRate, Duration < 500000000) AS EstimatedUnder500Ms, + sumIf(SampleRate, Duration < 1000000000) AS EstimatedUnder1000Ms, + sumIf(SampleRate, Duration < 2500000000) AS EstimatedUnder2500Ms, + sumIf(SampleRate, Duration < 5000000000) AS EstimatedUnder5000Ms, + sumIf(SampleRate, Duration < 10000000000) AS EstimatedUnder10000Ms + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + SpanKind, + toUInt8(ParentSpanId = '') AS IsRoot + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_runtimes_hourly_mv TO service_runtimes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + leftUTF8(ResourceAttributes['process.runtime.name'], 64) AS RuntimeName, + leftUTF8(ResourceAttributes['process.runtime.version'], 64) AS RuntimeVersion, + leftUTF8(ResourceAttributes['process.executable.name'], 64) AS ExecutableName, + leftUTF8(ResourceAttributes['telemetry.sdk.language'], 64) AS SdkLanguage, + leftUTF8(ResourceAttributes['telemetry.sdk.name'], 64) AS SdkName, + leftUTF8(ResourceAttributes['telemetry.sdk.version'], 64) AS SdkVersion, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_profiles_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + count() AS ProfileRowCount, + sum(length(Frames) * 16 + 200) AS ProfileSizeBytes + FROM profiles + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_logs_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(TimestampTime) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM logs + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_exp_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 300 AS MetricSizeBytes + FROM metrics_exponential_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_gauge_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_gauge + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 250 AS MetricSizeBytes + FROM metrics_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_sum_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_sum + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_traces_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + count() AS SpanCount, + sum(length(SpanName) + 300) AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM traces + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_facets_hourly_mv TO trace_facets_hourly AS +SELECT + OrgId, + toStartOfHour(Timestamp) AS Hour, + ServiceName, + SpanName, + HttpMethod, + HttpStatusCode, + DeploymentEnv, + ServiceNamespace, + HasError, + count() AS TraceCount, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM trace_list_mv + GROUP BY OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_index_mv TO trace_index AS +SELECT + OrgId, + TraceId, + ServiceName, + toStartOfFiveMinutes(Timestamp) AS Bucket, + min(Timestamp) AS MinTimestamp, + max(Timestamp) AS MaxTimestamp + FROM traces + GROUP BY OrgId, TraceId, ServiceName, Bucket; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS user_activity_hourly_logs_mv TO user_activity_hourly AS +SELECT + OrgId, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + if(LogAttributes['user.id'] != '', LogAttributes['user.id'], LogAttributes['enduser.id']) AS UserId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + toUInt64(0) AS SpanCount, + toUInt64(0) AS ErrorSpanCount, + toUInt64(0) AS RequestCount, + toUInt64(0) AS FailedRequestCount, + uniqIfState(TraceId, toUInt8(0)) AS TraceIds, + uniqIfState(LogAttributes['session.id'], LogAttributes['session.id'] != '') AS SessionIds, + quantilesTDigestIfState(0.5, 0.95)(toUInt64(0), toUInt8(0)) AS RequestDurationQuantiles, + approx_top_kState(10)('') AS TopRoutes, + count() AS LogCount, + countIf(SeverityNumber >= 13 AND SeverityNumber < 17) AS WarnLogCount, + countIf(SeverityNumber >= 17) AS ErrorLogCount, + min(TimestampTime) AS FirstSeen, + max(TimestampTime) AS LastSeen, + toUInt64(0) AS ClassifiedRequestCount, + toUInt64(0) AS WebRequestCount, + toUInt64(0) AS MobileRequestCount, + argMaxIfState(LogAttributes['user.name'], toUInt32(toUnixTimestamp(TimestampTime)), LogAttributes['user.name'] != '') AS UserName, + argMaxIfState(LogAttributes['user.email'], toUInt32(toUnixTimestamp(TimestampTime)), LogAttributes['user.email'] != '') AS UserEmail, + maxIf(toUInt32(toUnixTimestamp(TimestampTime)), LogAttributes['user.name'] != '') AS UserNameAt, + maxIf(toUInt32(toUnixTimestamp(TimestampTime)), LogAttributes['user.email'] != '') AS UserEmailAt + FROM logs + WHERE if(LogAttributes['user.id'] != '', LogAttributes['user.id'], LogAttributes['enduser.id']) != '' + GROUP BY OrgId, ServiceNamespace, UserId, Hour, ServiceName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS user_activity_hourly_traces_mv TO user_activity_hourly AS +SELECT + OrgId, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + count() AS SpanCount, + countIf(StatusCode = 'Error') AS ErrorSpanCount, + countIf(IsEntryPoint = 1) AS RequestCount, + countIf(IsEntryPoint = 1 AND StatusCode = 'Error') AS FailedRequestCount, + uniqIfState(TraceId, TraceId != '') AS TraceIds, + uniqIfState(SpanAttributes['session.id'], SpanAttributes['session.id'] != '') AS SessionIds, + quantilesTDigestIfState(0.5, 0.95)(Duration, IsEntryPoint = 1) AS RequestDurationQuantiles, + approx_top_kState(10)(if(IsEntryPoint = 1, if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName), '')) AS TopRoutes, + toUInt64(0) AS LogCount, + toUInt64(0) AS WarnLogCount, + toUInt64(0) AS ErrorLogCount, + min(toDateTime(Timestamp)) AS FirstSeen, + max(toDateTime(Timestamp)) AS LastSeen, + countIf(IsEntryPoint = 1) AS ClassifiedRequestCount, + countIf(IsEntryPoint = 1 AND (ResourceAttributes['maple.sdk.type'] IN ('browser', 'client') OR if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']) LIKE 'Mozilla/%')) AS WebRequestCount, + countIf(IsEntryPoint = 1 AND (NOT (ResourceAttributes['maple.sdk.type'] IN ('browser', 'client') OR if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']) LIKE 'Mozilla/%') AND (ResourceAttributes['telemetry.sdk.language'] IN ('swift', 'kotlin') OR ResourceAttributes['os.name'] IN ('iOS', 'iPadOS', 'Android') OR match(if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']), '^(Dalvik|okhttp)/| CFNetwork/')))) AS MobileRequestCount, + argMaxIfState(SpanAttributes['user.name'], toUInt32(toUnixTimestamp(Timestamp)), SpanAttributes['user.name'] != '') AS UserName, + argMaxIfState(SpanAttributes['user.email'], toUInt32(toUnixTimestamp(Timestamp)), SpanAttributes['user.email'] != '') AS UserEmail, + maxIf(toUInt32(toUnixTimestamp(Timestamp)), SpanAttributes['user.name'] != '') AS UserNameAt, + maxIf(toUInt32(toUnixTimestamp(Timestamp)), SpanAttributes['user.email'] != '') AS UserEmailAt + FROM traces + WHERE if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) != '' + GROUP BY OrgId, ServiceNamespace, UserId, Hour, ServiceName; diff --git a/packages/local-server/src/schema/local-inserts.json b/packages/local-server/src/schema/local-inserts.json index d47b097fe1..b029e728c2 100644 --- a/packages/local-server/src/schema/local-inserts.json +++ b/packages/local-server/src/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162", + "projectRevision": "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/packages/local-server/src/schema/local-schema.sql b/packages/local-server/src/schema/local-schema.sql index fe55e04795..78cfbbd705 100644 --- a/packages/local-server/src/schema/local-schema.sql +++ b/packages/local-server/src/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162 +-- projectRevision: 50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String), @@ -1034,7 +1034,9 @@ CREATE TABLE IF NOT EXISTS service_usage ( HistogramMetricCount UInt64, HistogramMetricSizeBytes UInt64, ExpHistogramMetricCount UInt64, - ExpHistogramMetricSizeBytes UInt64 + ExpHistogramMetricSizeBytes UInt64, + ProfileRowCount UInt64 DEFAULT 0, + ProfileSizeBytes UInt64 DEFAULT 0 ) ENGINE = SummingMergeTree ORDER BY (OrgId, ServiceName, Hour) @@ -2682,7 +2684,9 @@ SELECT 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM logs GROUP BY OrgId, ServiceName, Hour; @@ -2702,7 +2706,9 @@ SELECT 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, count() AS ExpHistogramMetricCount, - count() * 300 AS ExpHistogramMetricSizeBytes + count() * 300 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_exponential_histogram GROUP BY OrgId, ServiceName, Hour; @@ -2722,7 +2728,9 @@ SELECT 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_gauge GROUP BY OrgId, ServiceName, Hour; @@ -2742,7 +2750,9 @@ SELECT count() AS HistogramMetricCount, count() * 250 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_histogram GROUP BY OrgId, ServiceName, Hour; @@ -2762,10 +2772,34 @@ SELECT 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM metrics_sum GROUP BY OrgId, ServiceName, Hour; +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_profiles_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes, + count() AS ProfileRowCount, + sum(length(Frames) * 16 + 200) AS ProfileSizeBytes + FROM profiles + GROUP BY OrgId, ServiceName, Hour; + CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS SELECT OrgId, @@ -2782,7 +2816,9 @@ SELECT 0 AS HistogramMetricCount, 0 AS HistogramMetricSizeBytes, 0 AS ExpHistogramMetricCount, - 0 AS ExpHistogramMetricSizeBytes + 0 AS ExpHistogramMetricSizeBytes, + 0 AS ProfileRowCount, + 0 AS ProfileSizeBytes FROM traces GROUP BY OrgId, ServiceName, Hour; diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index b3686d5595..acdd63859c 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -3452,7 +3452,8 @@ SELECT toStartOfInterval(service_usage.Hour, INTERVAL 86400 SECOND) AS day, sum(service_usage.LogSizeBytes) AS logBytes, sum(service_usage.TraceSizeBytes) AS traceBytes, - sum(service_usage.SumMetricSizeBytes) + sum(service_usage.GaugeMetricSizeBytes) + sum(service_usage.HistogramMetricSizeBytes) + sum(service_usage.ExpHistogramMetricSizeBytes) AS metricBytes + sum(service_usage.SumMetricSizeBytes) + sum(service_usage.GaugeMetricSizeBytes) + sum(service_usage.HistogramMetricSizeBytes) + sum(service_usage.ExpHistogramMetricSizeBytes) AS metricBytes, + sum(service_usage.ProfileSizeBytes) AS profileBytes FROM service_usage WHERE service_usage.OrgId = 'org_sql_catalog' AND service_usage.Hour >= toStartOfHour(toDateTime('2026-01-01 10:30:00')) diff --git a/packages/query-engine-integrations/src/product/billing-usage.test.ts b/packages/query-engine-integrations/src/product/billing-usage.test.ts index 49bb20e770..ef5f379119 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.test.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.test.ts @@ -53,6 +53,7 @@ describe("dailySignalVolumeQuery", () => { logBytes: "13421772800", traceBytes: "4294967296", metricBytes: "1073741824", + profileBytes: "536870912", }, ]) @@ -62,8 +63,26 @@ describe("dailySignalVolumeQuery", () => { logBytes: 13_421_772_800, traceBytes: 4_294_967_296, metricBytes: 1_073_741_824, + profileBytes: 536_870_912, }) }) + + it("sums the profile bytes the profiles view writes into service_usage", () => { + const { sql } = compileUnsafe(dailySignalVolumeQuery(), params) + + expect(sql).toContain("sum(service_usage.ProfileSizeBytes) AS profileBytes") + }) + + it("reads 0 profile bytes without naming the profile columns, for a cluster that predates them", () => { + const compiled = compileUnsafe(dailySignalVolumeQuery({ profiles: false }), params) + + expect(compiled.sql).not.toContain("ProfileSizeBytes") + expect(compiled.sql).toContain("OrgId = 'org_123'") + const [row] = decodeRows(compiled, [ + { day: "2026-07-01 00:00:00", logBytes: "1", traceBytes: "2", metricBytes: "3", profileBytes: 0 }, + ]) + expect(row?.profileBytes).toBe(0) + }) }) describe("dailySessionCountQuery", () => { diff --git a/packages/query-engine-integrations/src/product/billing-usage.ts b/packages/query-engine-integrations/src/product/billing-usage.ts index 323443d9ad..f47036116c 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.ts @@ -4,11 +4,11 @@ // per-day volume per billable signal. Autumn only reports cycle *totals* // (`aggregateEvents`), so the daily shape comes from the warehouse: // -// `dailySignalVolumeQuery` — logs/traces/metrics bytes per UTC day, from the +// `dailySignalVolumeQuery`: logs/traces/metrics/profiles bytes per UTC day, from the // hourly `service_usage` MV. Same source the usage cards read, so the // chart's totals reconcile with them rather than telling a second story. // -// `dailySessionCountQuery` — browser sessions per UTC day, from +// `dailySessionCountQuery`: browser sessions per UTC day, from // `session_replays` (several rows per session). Kept as a separate query rather // than a UNION branch: the two tables disagree on column types (byte sums // are UInt64, the session count is UInt64 but the bucket column comes from @@ -33,10 +33,11 @@ export interface DailySignalVolumeOutput { readonly logBytes: number readonly traceBytes: number readonly metricBytes: number + readonly profileBytes: number } /** - * Per-UTC-day log/trace/metric bytes for one org. + * Per-UTC-day log/trace/metric/profile bytes for one org. * * `service_usage` is keyed on top-of-hour `Hour`, so both bounds snap to their * hour floor — the same correction `serviceUsageQuery` makes. A billing cycle @@ -45,8 +46,11 @@ export interface DailySignalVolumeOutput { * "Metrics" sums all four metric-type columns: the plan meters one `metrics` * feature, and splitting sum/gauge/histogram/exp-histogram would price bytes * the customer is never billed for separately. + * + * `profiles: false` reads 0 profile bytes, for a cluster whose `service_usage` + * predates migration 0068's profile columns. */ -export function dailySignalVolumeQuery() { +export function dailySignalVolumeQuery(options: { readonly profiles?: boolean } = {}) { return from(Datasources.serviceUsage) .select(($) => ({ day: CH.toStartOfInterval($.Hour, DAY_SECONDS), @@ -56,6 +60,7 @@ export function dailySignalVolumeQuery() { .add(CH.sum($.GaugeMetricSizeBytes)) .add(CH.sum($.HistogramMetricSizeBytes)) .add(CH.sum($.ExpHistogramMetricSizeBytes)), + profileBytes: options.profiles === false ? CH.lit(0) : CH.sum($.ProfileSizeBytes), })) .where(($) => [ $.OrgId.eq(orgIdParam),