Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ jobs:
MAPLE_REQUIRE_LIBCHDB: "1"
MAPLE_TELEMETRY: off
run: |
for file in tenant-isolation scoped-delete local-service-map-rollup checkpoint-merge-freeze local-store-traces-rekey local-query-guard local-query-guard.fuzz funnel-scenarios users-identity histogram-metrics exponential-histogram-metrics release-verification profile-sample-types; do
for file in tenant-isolation scoped-delete local-service-map-rollup checkpoint-merge-freeze local-store-traces-rekey local-query-guard local-query-guard.fuzz funnel-scenarios users-identity histogram-metrics exponential-histogram-metrics release-verification profile-sample-types service-usage-profiles; do
bun test --timeout 30000 "test/$file.test.ts"
done

Expand Down
2 changes: 2 additions & 0 deletions apps/cli/test/local-store-migrations.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ describe("current local schema identity", () => {
"service_map_db_edges_minutely_mv",
"service_runtimes_hourly",
"service_runtimes_hourly_mv",
"service_usage_profiles_mv",
"source_usage_hourly",
"source_usage_logs_mv",
"source_usage_metrics_exp_histogram_mv",
Expand Down Expand Up @@ -389,6 +390,7 @@ describe("current local schema identity", () => {
"service_map_db_edges_minutely_mv",
"service_runtimes_hourly",
"service_runtimes_hourly_mv",
"service_usage_profiles_mv",
"source_usage_hourly",
"source_usage_logs_mv",
"source_usage_metrics_exp_histogram_mv",
Expand Down
105 changes: 105 additions & 0 deletions apps/cli/test/service-usage-profiles.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Profiles reach service_usage on a real ClickHouse engine: the view counts each stack row and
// estimates its bytes from its frames, per org, service and hour, beside the other signals.

import { afterAll, beforeAll, expect, test } from "bun:test"
import { mkdtempSync, rmSync } from "node:fs"
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
import * as CH from "@maple/query-engine/ch"
import { Schema, Array as Arr, String as Str } from "effect"
import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" }
import { Chdb } from "@maple/local-server/chdb"
import { decodeJsonEachRow } from "@maple/local-server/chdb-rows"
import { localSchemaSnapshot } from "@maple/local-server/local-schema-history"
import { describeWithChdb } from "./libchdb-test-support"
import { migrationStep, openStore } from "./local-store-step-test-support"

const ORG = "org_service_usage_profiles"
/** An hour ago on the hour: recent, so every TTL keeps it. */
const HOUR_MS = Math.floor(Date.now() / 3_600_000) * 3_600_000 - 3_600_000

const seconds = (ms: number) => Str.slice(0, 19)(Str.replace("T", " ")(new Date(ms).toISOString()))

const insertProfile = (db: Chdb, orgId: string, atMs: number, frames: ReadonlyArray<string>) => {
const list = (items: ReadonlyArray<string>) => `[${Arr.join(items, ", ")}]`
db.exec(
`INSERT INTO profiles (OrgId, Timestamp, ServiceName, ProfileId, SampleType, SampleUnit, Value, Samples, Frames, FrameFiles, FrameLines) VALUES ('${orgId}', '${seconds(atMs)}', 'checkout', 'p1', 'cpu', 'nanoseconds', 10, 1, ${list(Arr.map(frames, (frame) => `'${frame}'`))}, ${list(Arr.map(frames, () => "''"))}, ${list(Arr.map(frames, () => "0"))})`,
)
}

const UsageRow = Schema.Struct({
hour: Schema.String,
profileRows: CH.CHNumber,
profileBytes: CH.CHNumber,
logBytes: CH.CHNumber,
})

const usage = (db: Chdb, orgId: string) =>
decodeJsonEachRow(UsageRow)(
db.query(
`SELECT toString(Hour) AS hour, sum(ProfileRowCount) AS profileRows, sum(ProfileSizeBytes) AS profileBytes, sum(LogSizeBytes) AS logBytes FROM service_usage WHERE OrgId = '${orgId}' AND ServiceName = 'checkout' GROUP BY Hour ORDER BY Hour`,
),
)

describeWithChdb("service_usage profiles view", () => {
let root = ""
let db: Chdb

beforeAll(() => {
root = mkdtempSync(join(tmpdir(), "maple-service-usage-profiles-"))
db = Chdb.open({ dataDir: join(root, "data"), schemaSql })
})

afterAll(() => {
db.close()
rmSync(root, { recursive: true, force: true })
})

test("counts each stack row and its frame-based bytes into its org's hour", () => {
insertProfile(db, ORG, HOUR_MS + 60_000, ["main", "work"])
insertProfile(db, ORG, HOUR_MS + 120_000, ["main", "work", "parse"])
insertProfile(db, ORG, HOUR_MS + 3_600_000 + 60_000, ["main"])
insertProfile(db, "org_someone_else", HOUR_MS + 60_000, ["main"])

const rows = usage(db, ORG)
// 16 bytes per frame plus 200 per row: (2 * 16 + 200) + (3 * 16 + 200), then 1 * 16 + 200.
expect(rows).toEqual([
{ hour: seconds(HOUR_MS), profileRows: 2, profileBytes: 480, logBytes: 0 },
{ hour: seconds(HOUR_MS + 3_600_000), profileRows: 1, profileBytes: 216, logBytes: 0 },
])
})
})

describeWithChdb("v59 -> v60 service_usage profile columns", () => {
const STEP_ID = "local-0059-to-0060-service-usage-profiles"
const MIGRATED_ORG = "local"

test("keeps existing usage hours and counts the profiles ingested after it", async () => {
const root = await mkdtemp(join(tmpdir(), "maple-service-usage-profiles-step-"))
const sourceDataDir = join(root, "source", "data")
const targetDataDir = join(root, "target", "data")
const db = Chdb.open({ dataDir: sourceDataDir, schemaSql: localSchemaSnapshot(59).sql })
db.exec(
`INSERT INTO service_usage (OrgId, ServiceName, Hour, LogCount, LogSizeBytes) VALUES ('${MIGRATED_ORG}', 'checkout', '${seconds(HOUR_MS)}', 3, 900)`,
)
db.close()

const { migration, context } = migrationStep(STEP_ID, sourceDataDir, targetDataDir, Date.now())
const state = await migration.preflight(context)
await migration.prepareTarget(context, state)
const progress = await migration.apply(context, state, undefined)
await migration.verify(context, state, progress)

const rows = await openStore(
targetDataDir,
(db) => {
insertProfile(db, MIGRATED_ORG, HOUR_MS + 60_000, ["main"])
return usage(db, MIGRATED_ORG)
},
{ schemaSql: localSchemaSnapshot(60).sql, bootstrapSchema: false },
)
await rm(root, { recursive: true, force: true })
expect(rows).toEqual([{ hour: seconds(HOUR_MS), profileRows: 1, profileBytes: 216, logBytes: 900 }])
})
})
2 changes: 1 addition & 1 deletion apps/ingest/src/clickhouse_insert_mappings.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This file is generated by scripts/generate-clickhouse-insert-mappings.ts
// Do not edit manually.

pub const PROJECT_REVISION: &str = "4a524118c4076a53c7dde36f6252c55ba6e527ceba622866fc29b63abdc1d162";
pub const PROJECT_REVISION: &str = "50c656350d470d86a59a4124114b3b336e5335b9196aa745ddefca82602283b5";
// Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the
// Tinybird-coupled PROJECT_REVISION. Compared against
// org_clickhouse_settings.schema_version. See @maple/domain/clickhouse
Expand Down
105 changes: 105 additions & 0 deletions packages/backend/src/services/billing/DailySpendService.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { describe, expect, it } from "@effect/vitest"
import { Array as Arr, Effect, Layer, Schema, String as Str } from "effect"
import { OrgId, UserId, WarehouseSchemaDriftError, type TenantContext } from "@maple/domain/http"
import { compiledQueryOf } from "@maple/query-engine/execution"
import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService"
import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support"
import { DailySpendService } from "./DailySpendService"

// The billing chart's daily volume: profile bytes from service_usage become profilesGB, and a
// cluster whose service_usage predates the profile columns still answers, with no profiles.

const tenant: TenantContext = {
orgId: Schema.decodeSync(OrgId)("org_daily_spend"),
userId: Schema.decodeSync(UserId)("user_admin"),
roles: ["org:admin"],
authMode: "self_hosted",
}

const cycle = {
startMs: Date.parse("2026-07-01T00:00:00Z"),
endMs: Date.parse("2026-07-03T23:59:59Z"),
nowMs: Date.parse("2026-07-02T12:00:00Z"),
}

const drift = (column: string) =>
new WarehouseSchemaDriftError({
message: `Code: 47. Unknown expression identifier \`${column}\` in scope SELECT ...`,
pipeName: "billingDailySignalVolume",
clickhouseType: "UNKNOWN_IDENTIFIER",
})

const signalRow = (day: string, profileBytes: string | number) => ({
day: `${day} 00:00:00`,
logBytes: "2000000000",
traceBytes: "0",
metricBytes: "0",
profileBytes,
})

/** Answers the signal read with `signalRows`; `failWith` fails any SQL naming the profile columns. */
const layer = (signalRows: ReadonlyArray<Record<string, unknown>>, failWith?: WarehouseSchemaDriftError) =>
Layer.effect(DailySpendService, DailySpendService.make).pipe(
Layer.provide(
Layer.succeed(
WarehouseQueryService,
makeWarehouseServiceStub({
compiledQuery: (_tenant, input, options) => {
const compiled = compiledQueryOf(input)
if (failWith !== undefined && Str.includes("ProfileSizeBytes")(compiled.sql))
return Effect.fail(failWith)
const rows = options?.context === "billingDailySignalVolume" ? signalRows : []
return compiled.decodeRows(rows).pipe(Effect.orDie)
},
}),
),
),
)

describe("DailySpendService", () => {
it.effect("reports each day's profile bytes as profilesGB, and 0 on days without any", () =>
Effect.gen(function* () {
const spend = yield* DailySpendService
const response = yield* spend.get(tenant, cycle)

expect(
Arr.map(response.days, (day) => ({
date: day.date,
logsGB: day.logsGB,
profilesGB: day.profilesGB,
})),
).toEqual([
{ date: "2026-07-01", logsGB: 2, profilesGB: 1.5 },
{ date: "2026-07-02", logsGB: 0, profilesGB: 0 },
{ date: "2026-07-03", logsGB: 2, profilesGB: 0.25 },
])
}).pipe(
Effect.provide(
layer([signalRow("2026-07-01", "1500000000"), signalRow("2026-07-03", "250000000")]),
),
),
)

it.effect("falls back to no profile usage when service_usage predates the profile columns", () =>
Effect.gen(function* () {
const spend = yield* DailySpendService
const response = yield* spend.get(tenant, cycle)

expect(Arr.map(response.days, (day) => [day.date, day.logsGB, day.profilesGB])).toEqual([
["2026-07-01", 2, 0],
["2026-07-02", 0, 0],
["2026-07-03", 0, 0],
])
}).pipe(Effect.provide(layer([signalRow("2026-07-01", 0)], drift("ProfileSizeBytes")))),
)

it.effect("still fails the chart on drift that names another column", () =>
Effect.gen(function* () {
const spend = yield* DailySpendService
const error = yield* Effect.flip(spend.get(tenant, cycle))

expect(error.pipeName).toBe("billingDailySpend")
expect(error.message).toContain("LogSizeBytes")
}).pipe(Effect.provide(layer([], drift("LogSizeBytes")))),
)
})
31 changes: 22 additions & 9 deletions packages/backend/src/services/billing/DailySpendService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ import { DailySpendResponse, DailyVolume, WarehouseQueryError, type TenantContex
import { CH, formatWarehouseDateTime } from "@maple/query-engine"
import { Context, DateTime, Effect, Layer, String as Str } from "effect"
import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService"
import { isMissingProductEvents } from "@maple/backend/services/warehouse/missing-table"
import {
isMissingProductEvents,
isMissingServiceUsageProfiles,
} from "@maple/backend/services/warehouse/missing-table"
import * as Integrations from "@maple/query-engine-integrations"

/**
* The daily volume series behind the billing page's cumulative spend chart.
*
* Autumn reports cycle totals only, so the daily shape comes from the warehouse:
* `service_usage` for log/trace/metric bytes and `session_replays` for browser
* `service_usage` for log/trace/metric/profile bytes and `session_replays` for browser
* sessions. Volume only — this service never prices anything. Dollars are the
* client's job, computed from the same catalog rates the cost breakdown uses, so
* the chart and the invoice can't tell two stories.
Expand Down Expand Up @@ -68,12 +71,17 @@ export class DailySpendService extends Context.Service<DailySpendService, DailyS
// Two queries rather than a UNION: the tables disagree on the bucket
// column's type and unifying branches has produced 502s before. Both are
// pre-aggregated reads, so the second round-trip is cheap.
const signalRows = yield* warehouse
.compiledQuery(tenant, CH.compile(Integrations.dailySignalVolumeQuery(), params), {
profile: "list",
context: "billingDailySignalVolume",
})
.pipe(Effect.mapError(toQueryError))
const signalVolume = (profiles: boolean) =>
warehouse.compiledQuery(
tenant,
CH.compile(Integrations.dailySignalVolumeQuery({ profiles }), params),
{ profile: "list", context: "billingDailySignalVolume" },
)
// A cluster whose service_usage predates migration 0068 has no profile bytes to show.
const signalRows = yield* signalVolume(true).pipe(
Effect.catchIf(isMissingServiceUsageProfiles, () => signalVolume(false)),
Effect.mapError(toQueryError),
)

const sessionRows = yield* warehouse
.compiledQuery(tenant, CH.compile(Integrations.dailySessionCountQuery(), params), {
Expand All @@ -82,12 +90,16 @@ export class DailySpendService extends Context.Service<DailySpendService, DailyS
})
.pipe(Effect.mapError(toQueryError))

const byDay = new Map<string, { logsGB: number; tracesGB: number; metricsGB: number }>()
const byDay = new Map<
string,
{ logsGB: number; tracesGB: number; metricsGB: number; profilesGB: number }
>()
for (const row of signalRows) {
byDay.set(toUtcDateKey(DateTime.toEpochMillis(row.day)), {
logsGB: row.logBytes / BYTES_PER_BILLED_GB,
tracesGB: row.traceBytes / BYTES_PER_BILLED_GB,
metricsGB: row.metricBytes / BYTES_PER_BILLED_GB,
profilesGB: row.profileBytes / BYTES_PER_BILLED_GB,
})
}

Expand Down Expand Up @@ -125,6 +137,7 @@ export class DailySpendService extends Context.Service<DailySpendService, DailyS
logsGB: signals?.logsGB ?? 0,
tracesGB: signals?.tracesGB ?? 0,
metricsGB: signals?.metricsGB ?? 0,
profilesGB: signals?.profilesGB ?? 0,
browserSessions: sessionsByDay.get(key) ?? 0,
productEvents: eventsByDay.get(key) ?? 0,
}),
Expand Down
4 changes: 4 additions & 0 deletions packages/backend/src/services/warehouse/missing-table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,7 @@ export const isMissingUserActivityIdentity = namesDriftedColumn(/\bUser(?:Name|E
/** `error_events.UserId` (migration 0057) is missing: schema drift naming `UserId` only, so every
* other drift still surfaces. User issues and affected users then answer unattributed. */
export const isMissingErrorEventsUserId = namesDriftedColumn(/\bUserId\b/)

/** `service_usage` predates its profile columns (migration 0068): schema drift naming one of them.
* The billing chart then reads no profile usage. */
export const isMissingServiceUsageProfiles = namesDriftedColumn(/\bProfile(?:RowCount|SizeBytes)\b/)
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ const TABLE_NOTES: Record<string, ReadonlyArray<string>> = {
service_usage: [
"Hourly per-service rollup of STORED rows, 365-day TTL — far longer than the 30-day `logs`/`traces` retention, so it is the right table for long-range volume trends.",
"`Hour` is a top-of-hour DateTime. Snap both range bounds to their hour floor (`toStartOfHour`) or sub-hour windows return no rows at all; this necessarily over-reports at the window edges.",
"The `*SizeBytes` columns are STORAGE ESTIMATES, not measured bytes — the materialized views compute them as `length(Body) + 200` (logs), `length(SpanName) + 300` (spans) and flat per-point constants (metrics). They will NOT reconcile with an Autumn invoice and must never be presented as billed usage.",
"The `*SizeBytes` columns are STORAGE ESTIMATES, not measured bytes: the materialized views compute them as `length(Body) + 200` (logs), `length(SpanName) + 300` (spans), flat per-point constants (metrics) and `length(Frames) * 16 + 200` per stack row (profiles, `ProfileRowCount` / `ProfileSizeBytes`, 0 before migration 0068). They will NOT reconcile with an Autumn invoice and must never be presented as billed usage.",
"For real billed bytes use `metrics_sum` / `maple_ingest_org_bytes_total`, which records the decoded payload size the gateway actually metered. Counts here are also post-sampling and post-drop, i.e. what was stored, not what was accepted.",
"`TraceCount` counts stored SPANS (not distinct traces) and `LogCount` log records; there is no `SpanCount` column.",
"Metric counts are split across four columns by point type (`SumMetricCount`, `GaugeMetricCount`, `HistogramMetricCount`, `ExpHistogramMetricCount`) — add all four for a total metric-point count.",
Expand Down
Loading
Loading