From c244d647eb58bf6ba2bd496978ecad31df9e26c0 Mon Sep 17 00:00:00 2001 From: Miao Yang Date: Thu, 6 Aug 2026 17:58:30 +0800 Subject: [PATCH 1/2] feat(cli): classify identity persistence on every telemetry event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install-grain metrics currently trust every anonymousId equally, but ephemeral/isolated-HOME workloads mint a fresh id per run — one machine produced 2,956 rotating render identities since Jul 30 (94.4% seen on a single render command), inflating acquisition and diluting per-install penetration while looking like real product usage. Every event now carries: - identity_persistence: durable (id loaded from a preexisting config — proven to survive a process boundary) | unknown (minted+persisted this run; an ephemeral HOME is indistinguishable from a genuine first run from inside one process) | process_only (persist failed). Sticky per process so a fresh install re-reading its own write cannot self-promote. - config_write_outcome: ok | ok_unmirrored | failed for the identity- establishing write; absent when the id came from disk. - invocation_id: random uuid per CLI process, so one invocation's events group even when the install identity is untrustworthy (unlike run_id, which needs an orchestrator to set HYPERFRAMES_RUN_ID). Install metrics can then count only durable identities, and a daily churn monitor can alert on the unknown share. --- .../telemetry/client.postureRefresh.test.ts | 2 + packages/cli/src/telemetry/client.test.ts | 2 + packages/cli/src/telemetry/client.ts | 21 +++++- packages/cli/src/telemetry/config.test.ts | 65 +++++++++++++++++++ packages/cli/src/telemetry/config.ts | 63 +++++++++++++++++- packages/cli/src/telemetry/runId.ts | 18 +++++ 6 files changed, 169 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/telemetry/client.postureRefresh.test.ts b/packages/cli/src/telemetry/client.postureRefresh.test.ts index 25f86c10c2..4d6630b0b9 100644 --- a/packages/cli/src/telemetry/client.postureRefresh.test.ts +++ b/packages/cli/src/telemetry/client.postureRefresh.test.ts @@ -16,6 +16,8 @@ const configState = { telemetryEnabled: true }; vi.mock("./config.js", () => ({ readConfig: () => ({ anonymousId: "anon-1", telemetryEnabled: configState.telemetryEnabled }), writeConfig: () => {}, + getIdentityPersistence: () => "durable", + getIdentityWriteOutcome: () => undefined, })); vi.mock("../utils/env.js", () => ({ isDevMode: () => false })); vi.mock("./canary.js", () => ({ canaryEventProperties: () => ({}) })); diff --git a/packages/cli/src/telemetry/client.test.ts b/packages/cli/src/telemetry/client.test.ts index 149bf14840..bc194dc397 100644 --- a/packages/cli/src/telemetry/client.test.ts +++ b/packages/cli/src/telemetry/client.test.ts @@ -10,6 +10,8 @@ vi.stubEnv("DO_NOT_TRACK", ""); vi.mock("./config.js", () => ({ readConfig: () => ({ anonymousId: "anon-test-123", telemetryEnabled: true }), writeConfig: () => {}, + getIdentityPersistence: () => "durable", + getIdentityWriteOutcome: () => undefined, })); // shouldTrack() short-circuits in dev mode — force production behavior. diff --git a/packages/cli/src/telemetry/client.ts b/packages/cli/src/telemetry/client.ts index 0cb1d34aad..54ed5cfe9c 100644 --- a/packages/cli/src/telemetry/client.ts +++ b/packages/cli/src/telemetry/client.ts @@ -1,4 +1,10 @@ -import { readConfig, writeConfig } from "./config.js"; +import { + getIdentityPersistence, + getIdentityWriteOutcome, + readConfig, + writeConfig, +} from "./config.js"; +import { getInvocationId } from "./runId.js"; import { VERSION } from "../version.js"; import { c } from "../ui/colors.js"; import { diag } from "../ui/diagnostics.js"; @@ -101,6 +107,19 @@ export function trackEvent( // could not read. Without it a partial disk write is indistinguishable // from a genuinely fresh install. Absent in the normal case. install_state_file_corrupt: readConfig().stateFileCorrupt, + // Whether this process's anonymousId can be trusted to survive to the + // next run: `durable` (loaded from a preexisting config), `unknown` + // (minted+persisted this run — an ephemeral HOME is indistinguishable + // from a genuine first run), `process_only` (persist failed). Install- + // grain metrics should count only durable identities; the identity- + // churn workloads (fresh id per run) are never durable. + identity_persistence: getIdentityPersistence(), + // Outcome of the identity-establishing config write; absent when the + // identity came from disk and nothing needed writing. + config_write_outcome: getIdentityWriteOutcome(), + // Groups one invocation's events even when the install identity is + // untrustworthy. Always present, unlike the orchestrator-set run_id. + invocation_id: getInvocationId(), // Canary assignments as `$feature/canary-` — PostHog's native flag // property shape, so breakdowns and experiment analysis work on a canary // with nothing configured server-side. On EVERY event, not just renders: diff --git a/packages/cli/src/telemetry/config.test.ts b/packages/cli/src/telemetry/config.test.ts index d714d5ffcb..ddf2d6ca95 100644 --- a/packages/cli/src/telemetry/config.test.ts +++ b/packages/cli/src/telemetry/config.test.ts @@ -699,3 +699,68 @@ describe("an unwritable config dir must not re-roll the seed", () => { }); }); }); + +describe("identity-persistence classification (sticky per process)", () => { + let readConfig: typeof import("./config.js").readConfig; + let readConfigFresh: typeof import("./config.js").readConfigFresh; + let getIdentityPersistence: typeof import("./config.js").getIdentityPersistence; + let getIdentityWriteOutcome: typeof import("./config.js").getIdentityWriteOutcome; + let CONFIG_PATH: typeof import("./config.js").CONFIG_PATH; + + beforeEach(async () => { + fsState.files.clear(); + policyState.runtimeOverride = null; + vi.resetModules(); + ({ readConfig, readConfigFresh, getIdentityPersistence, getIdentityWriteOutcome, CONFIG_PATH } = + await import("./config.js")); + }); + + it("classifies a fresh mint whose write landed as unknown, never durable", () => { + readConfig(); + expect(getIdentityPersistence()).toBe("unknown"); + expect(getIdentityWriteOutcome()).toBe("ok"); + }); + + it("classifies an id loaded from a preexisting config as durable, with no write outcome", () => { + fsState.files.set( + CONFIG_PATH, + JSON.stringify({ telemetryEnabled: true, anonymousId: "prior-id", bucketSeed: "seed" }), + ); + readConfig(); + expect(getIdentityPersistence()).toBe("durable"); + expect(getIdentityWriteOutcome()).toBeUndefined(); + }); + + it("classifies a fresh mint whose write failed as process_only", async () => { + const fs = await import("node:fs"); + vi.mocked(fs.writeFileSync).mockImplementation(() => { + throw new Error("EACCES: permission denied"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + readConfig(); + expect(getIdentityPersistence()).toBe("process_only"); + expect(getIdentityWriteOutcome()).toBe("failed"); + + warn.mockRestore(); + vi.mocked(fs.writeFileSync).mockImplementation((path, content) => { + fsState.files.set(String(path), String(content)); + }); + }); + + it("does not self-promote to durable when a fresh-install process re-reads its own write", () => { + readConfig(); + expect(fsState.files.has(CONFIG_PATH)).toBe(true); + // The file now exists on disk; a cache-bypassing re-read hits the + // existing-file path — the ephemeral-HOME churn signature. + readConfigFresh(); + expect(getIdentityPersistence()).toBe("unknown"); + }); + + it("classifies a corrupt-config recovery mint by its write outcome, not as durable", () => { + fsState.files.set(CONFIG_PATH, "{not json"); + readConfig(); + expect(getIdentityPersistence()).toBe("unknown"); + expect(getIdentityWriteOutcome()).toBe("ok"); + }); +}); diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index 6048b67985..b186dec318 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -350,6 +350,7 @@ function mintAndCacheConfig(): HyperframesConfig { const config = mintConfig(); const write = writeConfigWithResult(config); if (!write.ok) warnSeedBackfillFailed(write.error); + classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write)); cachedConfig = { ...config }; return { ...config }; } @@ -535,6 +536,60 @@ const DEFAULT_CONFIG: HyperframesConfig = { let cachedConfig: HyperframesConfig | null = null; +// --------------------------------------------------------------------------- +// Identity-persistence classification — one sticky verdict per process. +// +// Install-grain metrics need to know whether this process's anonymousId can +// be trusted to survive to the next run. Three-way, because from inside a +// single process durability is not always provable: +// +// durable — the id was LOADED from a preexisting config file: it has +// already survived at least one process boundary. +// unknown — the id was minted this run and the write landed. An +// ephemeral/isolated HOME (the identity-churn workloads: +// fresh id per run, install_predecessor_found=false every +// time) looks IDENTICAL to a genuine first run from in +// here, so this cannot be promoted to durable. +// process_only — minted this run and the write failed (read-only mount, +// full disk): the id dies with this process, guaranteed. +// +// The verdict is sticky: a fresh-install process that later re-reads its own +// just-written file must not upgrade itself to durable. +// --------------------------------------------------------------------------- + +export type IdentityPersistence = "durable" | "process_only" | "unknown"; +/** `ok_unmirrored`: config.json landed but the install-state mirror did not. */ +export type IdentityWriteOutcome = "ok" | "ok_unmirrored" | "failed"; + +let identityPersistence: IdentityPersistence | undefined; +let identityWriteOutcome: IdentityWriteOutcome | undefined; + +function classifyIdentity(persistence: IdentityPersistence, outcome?: IdentityWriteOutcome): void { + if (identityPersistence !== undefined) return; + identityPersistence = persistence; + identityWriteOutcome = outcome; +} + +function writeOutcomeOf(write: ConfigWriteResult): IdentityWriteOutcome { + if (!write.ok) return "failed"; + return write.mirrored === false ? "ok_unmirrored" : "ok"; +} + +/** The process's sticky identity-persistence verdict (classifies on demand). */ +export function getIdentityPersistence(): IdentityPersistence { + readConfig(); + return identityPersistence ?? "unknown"; +} + +/** + * Outcome of the identity-establishing config write. Absent when the identity + * came from disk and nothing needed writing (the `durable` case). + */ +export function getIdentityWriteOutcome(): IdentityWriteOutcome | undefined { + readConfig(); + return identityWriteOutcome; +} + /** A non-empty string, or undefined — hand-edited configs can carry anything. */ function parseNonEmptyString(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined; @@ -637,6 +692,11 @@ export function readConfig(): HyperframesConfig { const config = materializeConfig(parsed); + // The id was loaded from a file that predates this process — the one + // case where cross-run persistence is already proven. Sticky, so a + // fresh-install process re-reading its own write cannot self-promote. + classifyIdentity("durable"); + // One-time backfill for configs predating the bucket seed: prefer the // recorded seed if a previous install already wrote one, else mint. // Persisted immediately — an unpersisted seed would re-roll every process. @@ -658,7 +718,8 @@ export function readConfig(): HyperframesConfig { // breaker survives config corruption too — but fail closed for the // privacy control: recovery must never silently turn telemetry back on. const config = { ...mintConfig(), telemetryEnabled: false }; - writeConfig(config); + const write = writeConfigWithResult(config); + classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write)); return config; } } diff --git a/packages/cli/src/telemetry/runId.ts b/packages/cli/src/telemetry/runId.ts index 5c5e307fb1..944def5a6f 100644 --- a/packages/cli/src/telemetry/runId.ts +++ b/packages/cli/src/telemetry/runId.ts @@ -1,3 +1,5 @@ +import { randomUUID } from "node:crypto"; + let resolved = false; let runId: string | undefined; @@ -10,3 +12,19 @@ export function getRunId(): string | undefined { return runId; } + +// --------------------------------------------------------------------------- +// Invocation id — a random uuid minted once per CLI process, present on every +// event that process emits. Unlike run_id (set only when an orchestrator +// exports HYPERFRAMES_RUN_ID), it needs no environment plumbing: it exists so +// the events of ONE invocation can be grouped even when the install identity +// is untrustworthy (identity_persistence != durable, e.g. an ephemeral HOME +// minting a fresh anonymousId per run). +// --------------------------------------------------------------------------- + +let invocationId: string | undefined; + +export function getInvocationId(): string { + invocationId ??= randomUUID(); + return invocationId; +} From c6fdd20738a2b6d90eaf06f98fb33a400c7781d8 Mon Sep 17 00:00:00 2001 From: Miao Yang Date: Thu, 6 Aug 2026 21:48:47 +0800 Subject: [PATCH 2/2] fix(cli): require the anonymousId to come off disk before classifying durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding: materializeConfig mints a replacement anonymousId when a hand-edited/image-baked config lacks one. That replacement only reaches disk when the bucket-seed backfill happens to write; with a seed present the read path performs no write at all, so the install re-mints a fresh id every run while the unconditional durable branch stamped each of them with the one label durable-only counting is allowed to trust. durable now requires parseNonEmptyString(parsed.anonymousId): a minted replacement classifies like a fresh mint — by the backfill write outcome when that path runs (unknown/process_only), and process_only on the no-write path where the id provably dies with the process. Two tests pin both shapes. --- packages/cli/src/telemetry/config.test.ts | 24 +++++++++++++++++++++++ packages/cli/src/telemetry/config.ts | 24 ++++++++++++++++++----- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/telemetry/config.test.ts b/packages/cli/src/telemetry/config.test.ts index ddf2d6ca95..f7e987f4a3 100644 --- a/packages/cli/src/telemetry/config.test.ts +++ b/packages/cli/src/telemetry/config.test.ts @@ -757,6 +757,30 @@ describe("identity-persistence classification (sticky per process)", () => { expect(getIdentityPersistence()).toBe("unknown"); }); + it("does not label a replacement id minted at read time as durable (seed present, no write)", () => { + // Hand-edited / image-baked config: file exists with a seed but NO + // anonymousId. materializeConfig mints a replacement, nothing persists it + // (the seed suppresses the backfill write) — so the install re-mints + // every run. Labelling that durable would dress the churn signature in + // the one trustworthy label (review finding). + fsState.files.set( + CONFIG_PATH, + JSON.stringify({ telemetryEnabled: true, bucketSeed: "baked-seed" }), + ); + readConfig(); + expect(getIdentityPersistence()).toBe("process_only"); + }); + + it("classifies a replacement id carried to disk by the seed backfill by its write outcome", () => { + // Same hand-edited shape but ALSO missing the seed: the backfill write + // persists the whole config, replacement id included — a fresh mint in + // all but name, so it classifies like one (unknown, never durable). + fsState.files.set(CONFIG_PATH, JSON.stringify({ telemetryEnabled: true })); + readConfig(); + expect(getIdentityPersistence()).toBe("unknown"); + expect(getIdentityWriteOutcome()).toBe("ok"); + }); + it("classifies a corrupt-config recovery mint by its write outcome, not as durable", () => { fsState.files.set(CONFIG_PATH, "{not json"); readConfig(); diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index b186dec318..f317a6a8a8 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -147,11 +147,12 @@ function warnSeedBackfillFailed(error: string | undefined): void { * is exactly when that happens, so it says so once rather than failing * invisibly. */ -function backfillBucketSeed(config: HyperframesConfig): void { +function backfillBucketSeed(config: HyperframesConfig): ConfigWriteResult { const recorded = readInstallState(); config.bucketSeed = (isInstallState(recorded) ? recorded.bucketSeed : undefined) ?? randomUUID(); const write = writeConfigWithResult(config); if (!write.ok) warnSeedBackfillFailed(write.error); + return write; } // ONLY the positive is cached. The latch is monotonic across processes in one @@ -692,16 +693,25 @@ export function readConfig(): HyperframesConfig { const config = materializeConfig(parsed); - // The id was loaded from a file that predates this process — the one - // case where cross-run persistence is already proven. Sticky, so a + // `durable` requires the id to have actually COME OFF DISK — a file that + // predates this process proves cross-run persistence. materializeConfig + // mints a REPLACEMENT id when the parsed file lacks one (hand-edited / + // image-baked configs), and that replacement only reaches disk if the + // bucket-seed backfill below happens to write; labelling it durable would + // dress the exact churn signature this field exists to catch in the one + // trustworthy label (review finding). Sticky either way, so a // fresh-install process re-reading its own write cannot self-promote. - classifyIdentity("durable"); + const idFromDisk = parseNonEmptyString(parsed.anonymousId) !== undefined; // One-time backfill for configs predating the bucket seed: prefer the // recorded seed if a previous install already wrote one, else mint. // Persisted immediately — an unpersisted seed would re-roll every process. if (config.bucketSeed === undefined) { - backfillBucketSeed(config); + const write = backfillBucketSeed(config); + // The backfill write carries any replacement id to disk, so a minted id + // classifies exactly like a fresh mint: by whether the write landed. + if (idFromDisk) classifyIdentity("durable"); + else classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write)); // Cache even if the write failed, so the seed is at least stable for // the life of this process (a re-roll per readConfigFresh would flip // cohorts mid-session). @@ -709,6 +719,10 @@ export function readConfig(): HyperframesConfig { return { ...config }; } + // No write happens on this path: a replacement id lives only in this + // process, guaranteed — the definition of process_only. + classifyIdentity(idFromDisk ? "durable" : "process_only"); + cachedConfig = config; return { ...config }; } catch {