From 0e530e01f42461e3c5187e5ce7f4baf65ae4a09f Mon Sep 17 00:00:00 2001 From: Mehar Khanna Date: Fri, 13 Nov 2026 12:37:51 -0500 Subject: [PATCH] Collector: an app's extra keys (migration 0022): a merged app's old key keeps reporting to it The owner merged Glyco's two apps (the one with the live site's data and the one the plan was bought on) and asked that the key of the app that went away keep working. app_keys maps an extra key to an app's own key; ingest resolves the batch's token to the app's own key (canonicalToken, cached like knownToken was) before observe-only, aggregation, export and forwarding; the raw R2 batch keeps the key the page sent. An app's own key always wins; deleting the app deletes its extra keys. No route writes the table yet (README: Extra keys). Also fixes createProject's cache entry to the new shape. Co-Authored-By: Claude Opus 5.5 --- telemetry-worker/README.md | 16 +++++ telemetry-worker/migrations/0022_app_keys.sql | 11 ++++ telemetry-worker/src/auth.ts | 2 + telemetry-worker/src/collector.ts | 13 ++-- telemetry-worker/src/dashboard.ts | 41 ++++++++---- telemetry-worker/test/app-keys.test.ts | 62 +++++++++++++++++++ 6 files changed, 128 insertions(+), 17 deletions(-) create mode 100644 telemetry-worker/migrations/0022_app_keys.sql create mode 100644 telemetry-worker/test/app-keys.test.ts diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index b5e48032..49ea4d66 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -1249,6 +1249,22 @@ One new table, `suggest_usage`. Additive; apply it before or after the deploy: `npx --yes wrangler@4 d1 migrations apply genclass-dashboard --remote`. Then set the key, as a secret, from your own terminal: `npx --yes wrangler@4 secret put OPENAI_API_KEY`. +## Extra keys (two apps merged into one) + +When an onboarding left one person with two apps for one site (one with the data, one with the plan), they are merged +into one app and the key of the app that goes away keeps working: `app_keys` (migration 0022) maps that key to the +remaining app's own key, and a batch that carries it is aggregated, exported and forwarded under the app's key +(`src/dashboard.ts` `canonicalToken`; the raw batch in R2 keeps the key the page sent). An app's own key always wins; +deleting the app deletes its extra keys. No route writes the table; add a key by hand: + +```bash +CLOUDFLARE_ACCOUNT_ID=144bd5f5270b51dbe7faf46227a154f0 npx --yes wrangler@4 d1 execute genclass-dashboard --remote \ + --command "INSERT INTO app_keys (token, project_token, created) VALUES ('', (SELECT token FROM projects WHERE id = ''), strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" +``` + +A worker keeps a key's answer in memory for up to 5 minutes (30 seconds for an unknown key), so a new extra key counts +within 30 seconds. + ## Reports by email A GenClass report about an account's apps goes to the account's email on the schedule it chooses, and an alert goes diff --git a/telemetry-worker/migrations/0022_app_keys.sql b/telemetry-worker/migrations/0022_app_keys.sql new file mode 100644 index 00000000..019e6811 --- /dev/null +++ b/telemetry-worker/migrations/0022_app_keys.sql @@ -0,0 +1,11 @@ +-- An app's extra keys (owner request, 2026-11-13): when two apps of one account are merged into one (an onboarding that +-- made two), the key of the app that goes away keeps reporting, counted as the remaining app's own key. A batch that +-- carries an extra key is aggregated, indexed for export and forwarded under the app's own key (src/collector.ts, +-- src/dashboard.ts canonicalToken); the raw batch in R2 keeps the key the page sent. Deleting the app deletes its +-- extra keys. Added by hand for now (telemetry-worker/README.md "Extra keys"): no route writes this table. +CREATE TABLE IF NOT EXISTS app_keys ( + token TEXT PRIMARY KEY, -- the extra key a page may send (gc_ + 22 base62) + project_token TEXT NOT NULL, -- the app's own key (projects.token) + created TEXT NOT NULL +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS app_keys_project ON app_keys (project_token); diff --git a/telemetry-worker/src/auth.ts b/telemetry-worker/src/auth.ts index 2a30eece..a6ed82ae 100644 --- a/telemetry-worker/src/auth.ts +++ b/telemetry-worker/src/auth.ts @@ -1715,6 +1715,8 @@ async function deleteProject(c: C, id: string): Promise { ...["daily", "hourly", "recent", "sessions", "visits", "agent_reports", "agent_log"].map((t) => c.db.prepare(`DELETE FROM ${t} WHERE token = ?`).bind(p.token)), c.db.prepare("DELETE FROM projects WHERE id = ? AND account_id = ?").bind(id, a.account.id), ]); + // its extra keys (migration 0022) go with it; before that migration the table is missing and there is nothing to delete + await c.db.prepare("DELETE FROM app_keys WHERE project_token = ?").bind(p.token).run().catch(() => undefined); forgetToken(p.token); await forgetExports(c.db, c.env.BUCKET, [p.token]).catch(() => undefined); // a repository linked to it by the GitHub App is unlinked (on its own: its tables come with migration 0013) diff --git a/telemetry-worker/src/collector.ts b/telemetry-worker/src/collector.ts index 886fb4f6..de654483 100644 --- a/telemetry-worker/src/collector.ts +++ b/telemetry-worker/src/collector.ts @@ -13,7 +13,8 @@ // Per-app tokens and dashboards (src/dashboard.ts, src/pages.ts; also routed on genclass.dev/{api/*,start*,dashboard*}): // POST /api/projects, GET /api/projects/, POST and GET /start, GET /start/, GET /start/open-source (no account; no token to paste, init makes one unless the quiet choice is ticked), GET // /dashboard/. A batch with a known top-level `token` is also aggregated into D1 for that app's dashboard; -// without one it is stored in R2 only, exactly as before. +// without one it is stored in R2 only, exactly as before. A token that is one of an app's extra keys (app_keys, +// migration 0022: the key of an app merged into it) is counted under that app's own key (dashboard.ts canonicalToken). // Every plan includes the dashboard (billing/plans.json), so a private link opens it, also for an app with no account // (Free). A plan file that left the dashboard out would open the locked page instead (src/locked.ts) and its stats // would answer 402. Collection never depends on it: every known token is aggregated. @@ -32,7 +33,7 @@ import { agentReport, aggregate, createProject, - knownToken, + canonicalToken, projectBySecret, projectsCreatedSince, randomBase62, @@ -336,12 +337,12 @@ async function ingest(request: Request, env: Env, opts: HandleOptions): Promise< // Past its plan's live fixes for the month, the runtime is told to observe only (src/usage.ts): read before this // batch is counted (cached a minute), so the batch itself is stored and counted the same either way. const db = env.DB; - const observe = batch.token && db ? await observeOnly(db, env, batch.token, now).catch(() => false) : false; + // the app this batch belongs to, by its own key: an app's extra key (app_keys, migration 0022) counts as that key + const token = batch.token && db ? await canonicalToken(db, batch.token, now.getTime()).catch(() => null) : null; + const observe = token && db ? await observeOnly(db, env, token, now).catch(() => false) : false; // Dashboard aggregation for a known token. Never affects the response (failures are swallowed). - if (batch.token && db) { - const token = batch.token; + if (token && db) { const task = (async () => { - if (!(await knownToken(db, token, now.getTime()))) return; await aggregate(db, token, { ...batch, country }, now).catch(() => undefined); // export index, and forwarding for apps on a paid plan (src/paid.ts) const f = opts.fetch ?? ((u: string, i?: RequestInit) => fetch(u, i)); diff --git a/telemetry-worker/src/dashboard.ts b/telemetry-worker/src/dashboard.ts index f3596e7e..35da0fcc 100644 --- a/telemetry-worker/src/dashboard.ts +++ b/telemetry-worker/src/dashboard.ts @@ -147,7 +147,7 @@ export async function createProject( .prepare("INSERT INTO projects (id, token, secret_hash, name, created, account_id, private_link) VALUES (?, ?, ?, ?, ?, ?, ?)") .bind(id, token, await sha256Hex(secret), name, created, accountId, accountId ? 0 : 1) .run(); - tokenCache.set(token, { ok: true, exp: now.getTime() + CACHE_OK_MS }); + tokenCache.set(token, { token, exp: now.getTime() + CACHE_OK_MS }); return { id, token, secret, name, created }; } catch (e) { if (attempt >= 2) throw e; // a collision is astronomically unlikely; anything else is a real error @@ -175,28 +175,47 @@ export async function projectBySecret(db: D1Like, secret: string): Promise(); } -/** Drop a deleted project's token from this isolate's cache, so its batches stop being counted here at once. */ +/** Drop a deleted project's token from this isolate's cache, and its extra keys, so their batches stop being counted here at once. */ export function forgetToken(token: string): void { tokenCache.delete(token); + for (const [k, v] of tokenCache) if (v.token === token) tokenCache.delete(k); } const CACHE_OK_MS = 5 * 60_000; const CACHE_MISS_MS = 30_000; -const tokenCache = new Map(); +const tokenCache = new Map(); export function resetTokenCache(): void { tokenCache.clear(); } -/** Is this a token of a known project? Cached in memory briefly (5 min hits, 30 s misses). */ -export async function knownToken(db: D1Like, token: string, nowMs: number): Promise { - if (!TOKEN_RE.test(token)) return false; +/** + * The app a batch's token belongs to, as that app's own key: the token itself when it is an app's key, the app's key + * when it is one of that app's extra keys (app_keys, migration 0022), else null. Cached in memory briefly (5 min hits, + * 30 s misses). An app's own key always wins over an extra key with the same value. + */ +export async function canonicalToken(db: D1Like, token: string, nowMs: number): Promise { + if (!TOKEN_RE.test(token)) return null; const c = tokenCache.get(token); - if (c && c.exp > nowMs) return c.ok; - const row = await db.prepare("SELECT 1 AS ok FROM projects WHERE token = ?").bind(token).first(); - const ok = !!row; + if (c && c.exp > nowMs) return c.token; + const own = await db.prepare("SELECT token FROM projects WHERE token = ?").bind(token).first<{ token: string }>(); + let found = own?.token ?? null; + if (!found) { + // before migration 0022 the table is missing: that reads as "no extra key" + const extra = await db + .prepare("SELECT k.project_token AS token FROM app_keys k JOIN projects p ON p.token = k.project_token WHERE k.token = ?") + .bind(token) + .first<{ token: string }>() + .catch(() => null); + found = extra?.token ?? null; + } if (tokenCache.size > 5000) tokenCache.clear(); - tokenCache.set(token, { ok, exp: nowMs + (ok ? CACHE_OK_MS : CACHE_MISS_MS) }); - return ok; + tokenCache.set(token, { token: found, exp: nowMs + (found ? CACHE_OK_MS : CACHE_MISS_MS) }); + return found; +} + +/** Is this a token of a known project (its own key or one of its extra keys)? */ +export async function knownToken(db: D1Like, token: string, nowMs: number): Promise { + return (await canonicalToken(db, token, nowMs)) !== null; } // --------------------------------------------------------------------------------------------- aggregation diff --git a/telemetry-worker/test/app-keys.test.ts b/telemetry-worker/test/app-keys.test.ts new file mode 100644 index 00000000..bf7742b0 --- /dev/null +++ b/telemetry-worker/test/app-keys.test.ts @@ -0,0 +1,62 @@ +// An app's extra keys (migration 0022, src/dashboard.ts canonicalToken, src/collector.ts ingest): a batch that carries +// one is counted on the app it belongs to, as if it carried the app's own key; an unknown key is still not counted; an +// app's own key wins over an extra key with the same value; deleting the app deletes its extra keys. +import { beforeEach, describe, expect, it } from "vitest"; +import { canonicalToken, resetTokenCache } from "../src/dashboard.js"; +import { call, createApp, now, resetClock, sendBatch, setup, signedIn } from "./account-helpers.js"; + +const page = (route: string): Record => ({ t: "session", seq: 0, at: 0, host: "shop.example", route, effectiveMode: "observe" }); + +const OLD = "gc_OLDKEYOLDKEYOLDKEYOLDKEY0".slice(0, 25); + +const loads = (sql: ReturnType["sql"], token: string): number => + Number((sql.db.prepare("SELECT COALESCE(SUM(count), 0) AS n FROM daily WHERE token = ? AND metric = 'sessions' AND key = ''").get(token) as { n: number }).n); + +beforeEach(() => { + resetClock(); + resetTokenCache(); +}); + +describe("extra keys of an app (migration 0022)", () => { + it("a batch with an app's extra key is counted on that app; an unknown key is not", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const app = await createApp(env, cookie, "Glyco site"); + expect(OLD).toMatch(/^gc_[A-Za-z0-9]{22}$/); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, app.token, now().toISOString()); + await sendBatch(env, app.token, [page("/")], { sid: "aaaaaaaaaaaa" }); + await sendBatch(env, OLD, [page("/pricing")], { sid: "bbbbbbbbbbbb" }); + await sendBatch(env, "gc_UNKNOWNUNKNOWNUNKNOWN0".slice(0, 25), [page("/")], { sid: "cccccccccccc" }); + expect(loads(sql, app.token)).toBe(2); + expect(loads(sql, OLD)).toBe(0); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBe(app.token); + // the dashboard shows both pages under the one app + const routes = sql.db.prepare("SELECT key FROM daily WHERE token = ? AND metric = 'page_route' ORDER BY key").all(app.token) as { key: string }[]; + expect(routes.map((r) => r.key)).toEqual(["/", "/pricing"]); + }); + + it("an app's own key wins over an extra key with the same value, and a key whose app is gone counts nowhere", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const a = await createApp(env, cookie, "A"); + const b = await createApp(env, cookie, "B"); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(b.token, a.token, now().toISOString()); + expect(await canonicalToken(env.DB!, b.token, now().getTime())).toBe(b.token); + resetTokenCache(); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, "gc_NOSUCHAPPNOSUCHAPPNOSU", now().toISOString()); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBeNull(); + }); + + it("deleting the app deletes its extra keys, and the key stops counting at once", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + const app = await createApp(env, cookie, "Shop"); + sql.db.prepare("INSERT INTO app_keys (token, project_token, created) VALUES (?, ?, ?)").run(OLD, app.token, now().toISOString()); + await sendBatch(env, OLD, [page("/")]); + expect(loads(sql, app.token)).toBe(1); + const del = await call(env, "DELETE", `/api/account/projects/${app.id}`, { body: { confirm: "delete" }, cookie }); + expect(del.status).toBe(204); + expect(sql.db.prepare("SELECT COUNT(*) AS n FROM app_keys").get()).toEqual({ n: 0 }); + expect(await canonicalToken(env.DB!, OLD, now().getTime())).toBeNull(); + }); +});