From c760eda866b6d1593e46327ce8e6be1b08974298 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 02:12:17 +0100 Subject: [PATCH] fix(warehouse): service_usage forward query back to SELECT * after the 0068 deploy The explicit column list was a one-shot for the deploy that added the profiles columns. Left in, the next rebuild of service_usage would reset ProfileRowCount and ProfileSizeBytes to 0. --- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- packages/domain/src/generated/clickhouse-schema.ts | 2 +- .../domain/src/generated/tinybird-project-manifest.ts | 4 ++-- packages/domain/src/tinybird/datasources.ts | 10 ++-------- packages/local-server/src/schema/local-inserts.json | 2 +- packages/local-server/src/schema/local-schema.sql | 2 +- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index cd41d69c0b..be36c4b392 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5"; +pub const PROJECT_REVISION: &str = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index f4666838ac..09361fe806 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const +export const projectRevision = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index a976769094..9d25d05d82 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const +export const projectRevision = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0" as const export const datasources = [ { @@ -227,7 +227,7 @@ export const datasources = [ { name: "service_usage", content: - 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes,\n \t\tSumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes,\n \t\tHistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileRowCount,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileSizeBytes', + 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT *', }, { name: "session_events", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index a9254716cd..e7b54574cf 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -323,14 +323,8 @@ export const serviceUsage = defineDatasource("service_usage", { ProfileRowCount: t.uint64().default(0), ProfileSizeBytes: t.uint64().default(0), }, - // One-shot: changing the views makes Tinybird rebuild service_usage, and without a forward - // query it would refill it from 14 to 90 day sources. Back to `SELECT *` after the 0068 deploy. - forwardQuery: `SELECT - OrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes, - SumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes, - HistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes, - defaultValueOfTypeName('UInt64') AS ProfileRowCount, - defaultValueOfTypeName('UInt64') AS ProfileSizeBytes`, + // A rebuild refills from the live table: the source tables keep 14 to 90 days, this one 365. + forwardQuery: `SELECT *`, engine: engine.summingMergeTree({ sortingKey: ["OrgId", "ServiceName", "Hour"], ttl: "Hour + INTERVAL 365 DAY", diff --git a/packages/local-server/src/schema/local-inserts.json b/packages/local-server/src/schema/local-inserts.json index b029e728c2..56b558e207 100644 --- a/packages/local-server/src/schema/local-inserts.json +++ b/packages/local-server/src/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5", + "projectRevision": "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/packages/local-server/src/schema/local-schema.sql b/packages/local-server/src/schema/local-schema.sql index 78cfbbd705..0d9a7975be 100644 --- a/packages/local-server/src/schema/local-schema.sql +++ b/packages/local-server/src/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5 +-- projectRevision: 3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String),