diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index cd41d69c0b..be36c4b392 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5"; +pub const PROJECT_REVISION: &str = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index f4666838ac..09361fe806 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const +export const projectRevision = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index a976769094..9d25d05d82 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5" as const +export const projectRevision = "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0" as const export const datasources = [ { @@ -227,7 +227,7 @@ export const datasources = [ { name: "service_usage", content: - 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes,\n \t\tSumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes,\n \t\tHistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileRowCount,\n \t\tdefaultValueOfTypeName(\'UInt64\') AS ProfileSizeBytes', + 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64,\n ProfileRowCount UInt64 DEFAULT 0,\n ProfileSizeBytes UInt64 DEFAULT 0\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT *', }, { name: "session_events", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index a9254716cd..e7b54574cf 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -323,14 +323,8 @@ export const serviceUsage = defineDatasource("service_usage", { ProfileRowCount: t.uint64().default(0), ProfileSizeBytes: t.uint64().default(0), }, - // One-shot: changing the views makes Tinybird rebuild service_usage, and without a forward - // query it would refill it from 14 to 90 day sources. Back to `SELECT *` after the 0068 deploy. - forwardQuery: `SELECT - OrgId, ServiceName, Hour, LogCount, LogSizeBytes, TraceCount, TraceSizeBytes, - SumMetricCount, SumMetricSizeBytes, GaugeMetricCount, GaugeMetricSizeBytes, - HistogramMetricCount, HistogramMetricSizeBytes, ExpHistogramMetricCount, ExpHistogramMetricSizeBytes, - defaultValueOfTypeName('UInt64') AS ProfileRowCount, - defaultValueOfTypeName('UInt64') AS ProfileSizeBytes`, + // A rebuild refills from the live table: the source tables keep 14 to 90 days, this one 365. + forwardQuery: `SELECT *`, engine: engine.summingMergeTree({ sortingKey: ["OrgId", "ServiceName", "Hour"], ttl: "Hour + INTERVAL 365 DAY", diff --git a/packages/local-server/src/schema/local-inserts.json b/packages/local-server/src/schema/local-inserts.json index b029e728c2..56b558e207 100644 --- a/packages/local-server/src/schema/local-inserts.json +++ b/packages/local-server/src/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5", + "projectRevision": "3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/packages/local-server/src/schema/local-schema.sql b/packages/local-server/src/schema/local-schema.sql index 78cfbbd705..0d9a7975be 100644 --- a/packages/local-server/src/schema/local-schema.sql +++ b/packages/local-server/src/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5 +-- projectRevision: 3fd7108014c7e271bf9654a09fb0a781d3fbb132aba36deef08ee3a687c84be0 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String),