diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index b5e48032..49ea4d66 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -1249,6 +1249,22 @@ One new table, `suggest_usage`. Additive; apply it before or after the deploy: `npx --yes wrangler@4 d1 migrations apply genclass-dashboard --remote`. Then set the key, as a secret, from your own terminal: `npx --yes wrangler@4 secret put OPENAI_API_KEY`. +## Extra keys (two apps merged into one) + +When an onboarding left one person with two apps for one site (one with the data, one with the plan), they are merged +into one app and the key of the app that goes away keeps working: `app_keys` (migration 0022) maps that key to the +remaining app's own key, and a batch that carries it is aggregated, exported and forwarded under the app's key +(`src/dashboard.ts` `canonicalToken`; the raw batch in R2 keeps the key the page sent). An app's own key always wins; +deleting the app deletes its extra keys. No route writes the table; add a key by hand: + +```bash +CLOUDFLARE_ACCOUNT_ID=144bd5f5270b51dbe7faf46227a154f0 npx --yes wrangler@4 d1 execute genclass-dashboard --remote \ + --command "INSERT INTO app_keys (token, project_token, created) VALUES ('', (SELECT token FROM projects WHERE id = ''), strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" +``` + +A worker keeps a key's answer in memory for up to 5 minutes (30 seconds for an unknown key), so a new extra key counts +within 30 seconds. + ## Reports by email A GenClass report about an account's apps goes to the account's email on the schedule it chooses, and an alert goes diff --git a/telemetry-worker/migrations/0022_app_keys.sql b/telemetry-worker/migrations/0022_app_keys.sql new file mode 100644 index 00000000..019e6811 --- /dev/null +++ b/telemetry-worker/migrations/0022_app_keys.sql @@ -0,0 +1,11 @@ +-- An app's extra keys (owner request, 2026-11-13): when two apps of one account are merged into one (an onboarding that +-- made two), the key of the app that goes away keeps reporting, counted as the remaining app's own key. A batch that +-- carries an extra key is aggregated, indexed for export and forwarded under the app's own key (src/collector.ts, +-- src/dashboard.ts canonicalToken); the raw batch in R2 keeps the key the page sent. Deleting the app deletes its +-- extra keys. Added by hand for now (telemetry-worker/README.md "Extra keys"): no route writes this table. +CREATE TABLE IF NOT EXISTS app_keys ( + token TEXT PRIMARY KEY, -- the extra key a page may send (gc_ + 22 base62) + project_token TEXT NOT NULL, -- the app's own key (projects.token) + created TEXT NOT NULL +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS app_keys_project ON app_keys (project_token); diff --git a/telemetry-worker/src/auth.ts b/telemetry-worker/src/auth.ts index 2a30eece..a6ed82ae 100644 --- a/telemetry-worker/src/auth.ts +++ b/telemetry-worker/src/auth.ts @@ -1715,6 +1715,8 @@ async function deleteProject(c: C, id: string): Promise { ...["daily", "hourly", "recent", "sessions", "visits", "agent_reports", "agent_log"].map((t) => c.db.prepare(`DELETE FROM ${t} WHERE token = ?`).bind(p.token)), c.db.prepare("DELETE FROM projects WHERE id = ? AND account_id = ?").bind(id, a.account.id), ]); + // its extra keys (migration 0022) go with it; before that migration the table is missing and there is nothing to delete + await c.db.prepare("DELETE FROM app_keys WHERE project_token = ?").bind(p.token).run().catch(() => undefined); forgetToken(p.token); await forgetExports(c.db, c.env.BUCKET, [p.token]).catch(() => undefined); // a repository linked to it by the GitHub App is unlinked (on its own: its tables come with migration 0013) diff --git a/telemetry-worker/src/collector.ts b/telemetry-worker/src/collector.ts index 886fb4f6..de654483 100644 --- a/telemetry-worker/src/collector.ts +++ b/telemetry-worker/src/collector.ts @@ -13,7 +13,8 @@ // Per-app tokens and dashboards (src/dashboard.ts, src/pages.ts; also routed on genclass.dev/{api/*,start*,dashboard*}): // POST /api/projects, GET /api/projects/, POST and GET /start, GET /start/, GET /start/open-source (no account; no token to paste, init makes one unless the quiet choice is ticked), GET // /dashboard/. A batch with a known top-level `token` is also aggregated into D1 for that app's dashboard; -// without one it is stored in R2 only, exactly as before. +// without one it is stored in R2 only, exactly as before. A token that is one of an app's extra keys (app_keys, +// migration 0022: the key of an app merged into it) is counted under that app's own key (dashboard.ts canonicalToken). // Every plan includes the dashboard (billing/plans.json), so a private link opens it, also for an app with no account // (Free). A plan file that left the dashboard out would open the locked page instead (src/locked.ts) and its stats // would answer 402. Collection never depends on it: every known token is aggregated. @@ -32,7 +33,7 @@ import { agentReport, aggregate, createProject, - knownToken, + canonicalToken, projectBySecret, projectsCreatedSince, randomBase62, @@ -336,12 +337,12 @@ async function ingest(request: Request, env: Env, opts: HandleOptions): Promise< // Past its plan's live fixes for the month, the runtime is told to observe only (src/usage.ts): read before this // batch is counted (cached a minute), so the batch itself is stored and counted the same either way. const db = env.DB; - const observe = batch.token && db ? await observeOnly(db, env, batch.token, now).catch(() => false) : false; + // the app this batch belongs to, by its own key: an app's extra key (app_keys, migration 0022) counts as that key + const token = batch.token && db ? await canonicalToken(db, batch.token, now.getTime()).catch(() => null) : null; + const observe = token && db ? await observeOnly(db, env, token, now).catch(() => false) : false; // Dashboard aggregation for a known token. Never affects the response (failures are swallowed). - if (batch.token && db) { - const token = batch.token; + if (token && db) { const task = (async () => { - if (!(await knownToken(db, token, now.getTime()))) return; await aggregate(db, token, { ...batch, country }, now).catch(() => undefined); // export index, and forwarding for apps on a paid plan (src/paid.ts) const f = opts.fetch ?? ((u: string, i?: RequestInit) => fetch(u, i)); diff --git a/telemetry-worker/src/dashboard.ts b/telemetry-worker/src/dashboard.ts index f3596e7e..35da0fcc 100644 --- a/telemetry-worker/src/dashboard.ts +++ b/telemetry-worker/src/dashboard.ts @@ -147,7 +147,7 @@ export async function createProject( .prepare("INSERT INTO projects (id, token, secret_hash, name, created, account_id, private_link) VALUES (?, ?, ?, ?, ?, ?, ?)") .bind(id, token, await sha256Hex(secret), name, created, accountId, accountId ? 0 : 1) .run(); - tokenCache.set(token, { ok: true, exp: now.getTime() + CACHE_OK_MS }); + tokenCache.set(token, { token, exp: now.getTime() + CACHE_OK_MS }); return { id, token, secret, name, created }; } catch (e) { if (attempt >= 2) throw e; // a collision is astronomically unlikely; anything else is a real error @@ -175,28 +175,47 @@ export async function projectBySecret(db: D1Like, secret: string): Promise(); } -/** Drop a deleted project's token from this isolate's cache, so its batches stop being counted here at once. */ +/** Drop a deleted project's token from this isolate's cache, and its extra keys, so their batches stop being counted here at once. */ export function forgetToken(token: string): void { tokenCache.delete(token); + for (const [k, v] of tokenCache) if (v.token === token) tokenCache.delete(k); } const CACHE_OK_MS = 5 * 60_000; const CACHE_MISS_MS = 30_000; -const tokenCache = new Map(); +const tokenCache = new Map(); export function resetTokenCache(): void { tokenCache.clear(); } -/** Is this a token of a known project? Cached in memory briefly (5 min hits, 30 s misses). */ -export async function knownToken(db: D1Like, token: string, nowMs: number): Promise { - if (!TOKEN_RE.test(token)) return false; +/** + * The app a batch's token belongs to, as that app's own key: the token itself when it is an app's key, the app's key + * when it is one of that app's extra keys (app_keys, migration 0022), else null. Cached in memory briefly (5 min hits, + * 30 s misses). An app's own key always wins over an extra key with the same value. + */ +export async function canonicalToken(db: D1Like, token: string, nowMs: number): Promise { + if (!TOKEN_RE.test(token)) return null; const c = tokenCache.get(token); - if (c && c.exp > nowMs) return c.ok; - const row = await db.prepare("SELECT 1 AS ok FROM projects WHERE token = ?").bind(token).first(); - const ok = !!row; + if (c && c.exp > nowMs) return c.token; + const own = await db.prepare("SELECT token FROM projects WHERE token = ?").bind(token).first<{ token: string }>(); + let found = own?.token ?? null; + if (!found) { + // before migration 0022 the table is missing: that reads as "no extra key" + const extra = await db + .prepare("SELECT k.project_token AS token FROM app_keys k JOIN projects p ON p.token = k.project_token WHERE k.token = ?") + .bind(token) + .first<{ token: string }>() + .catch(() => null); + found = extra?.token ?? null; + } if (tokenCache.size > 5000) tokenCache.clear(); - tokenCache.set(token, { ok, exp: nowMs + (ok ? CACHE_OK_MS : CACHE_MISS_MS) }); - return ok; + tokenCache.set(token, { token: found, exp: nowMs + (found ? CACHE_OK_MS : CACHE_MISS_MS) }); + return found; +} + +/** Is this a token of a known project (its own key or one of its extra keys)? */ +export async function knownToken(db: D1Like, token: string, nowMs: number): Promise { + return (await canonicalToken(db, token, nowMs)) !== null; } // --------------------------------------------------------------------------------------------- aggregation diff --git a/telemetry-worker/test/app-keys.test.ts b/telemetry-worker/test/app-keys.test.ts new file mode 100644 index 00000000..bf7742b0 --- /dev/null +++ b/telemetry-worker/test/app-keys.test.ts @@ -0,0 +1,62 @@ +// An app's extra keys (migration 0022, src/dashboard.ts canonicalToken, src/collector.ts ingest): a batch that carries +// one is counted on the app it belongs to, as if it carried the app's own key; an unknown key is still not counted; an +// app's own key wins over an extra key with the same value; deleting the app deletes its extra keys. +import { beforeEach, describe, expect, it } from "vitest"; +import { canonicalToken, resetTokenCache } from "../src/dashboard.js"; +import { call, createApp, now, resetClock, sendBatch, setup, signedIn } from "./account-helpers.js"; + +const page = (route: string): Record => ({ t: "session", seq: 0, at: 0, host: "shop.example", route, effectiveMode: "observe" }); + +const OLD = "gc_OLDKEYOLDKEYOLDKEYOLDKEY0".slice(0, 25); + +const loads = (sql: ReturnType["sql"], token: string): number => + Number((sql.db.prepare("SELECT COALESCE(SUM(count), 0) AS n FROM daily WHERE token = ? AND metric = 'sessions' AND key = ''").get(token) as { n: number }).n); + +beforeEach(() => { + resetClock(); + resetTokenCache(); +}); + +describe("extra keys of an app (migration 0022)", () => { + it("a batch with an app's extra key is counted on that app; an unknown key is not", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const app = await createApp(env, cookie, "Glyco site"); + expect(OLD).toMatch(/^gc_[A-Za-z0-9]{22}$/); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, app.token, now().toISOString()); + await sendBatch(env, app.token, [page("/")], { sid: "aaaaaaaaaaaa" }); + await sendBatch(env, OLD, [page("/pricing")], { sid: "bbbbbbbbbbbb" }); + await sendBatch(env, "gc_UNKNOWNUNKNOWNUNKNOWN0".slice(0, 25), [page("/")], { sid: "cccccccccccc" }); + expect(loads(sql, app.token)).toBe(2); + expect(loads(sql, OLD)).toBe(0); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBe(app.token); + // the dashboard shows both pages under the one app + const routes = sql.db.prepare("SELECT key FROM daily WHERE token = ? AND metric = 'page_route' ORDER BY key").all(app.token) as { key: string }[]; + expect(routes.map((r) => r.key)).toEqual(["/", "/pricing"]); + }); + + it("an app's own key wins over an extra key with the same value, and a key whose app is gone counts nowhere", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const a = await createApp(env, cookie, "A"); + const b = await createApp(env, cookie, "B"); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(b.token, a.token, now().toISOString()); + expect(await canonicalToken(env.DB!, b.token, now().getTime())).toBe(b.token); + resetTokenCache(); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, "gc_NOSUCHAPPNOSUCHAPPNOSU", now().toISOString()); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBeNull(); + }); + + it("deleting the app deletes its extra keys, and the key stops counting at once", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const app = await createApp(env, cookie, "Shop"); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, app.token, now().toISOString()); + await sendBatch(env, OLD, [page("/")]); + expect(loads(sql, app.token)).toBe(1); + const del = await call(env, "DELETE", `/api/account/projects/${app.id}`, { body: { confirm: "delete" }, cookie }); + expect(del.status).toBe(204); + expect(sql.db.prepare("SELECT COUNT(*) AS n FROM app_keys").get()).toEqual({ n: 0 }); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBeNull(); + }); +});