Merge pull request #314 from modelstudioai/feat/telemetry-device-id-persistence

Persist anonymous telemetry install id as AEM uid

Author: hopelynd

docs/agents/telemetry-change.md

@@ -116,12 +116,16 @@ source-config 只用于百炼 / DashScope API 侧消费，不发送到通用网
 
 `page` 仅记录有限数值页码；Webhook 等命令使用的不透明字符串游标不得进入本地日志或远端 AEM，即使字符串内容看起来是数字。
 
-事件同时写入两处：
+事件写入两处：
 
-1. 本地 `~/.bailian/telemetry.jsonl`：权限 `0600`，超过 5 MB 后重建
+1. 本地 `~/.bailian/telemetry.jsonl`：权限 `0600`，超过 5 MB 后重建；每行附带 `uid`（只读已有文件，不创建）
 2. AEM：`pid=bailian-cli-node`，源码运行自动使用 `env=dev`，npm 安装或编译二进制使用 `env=prod`
 
-底层 Node tracker 还会附加公共设备字段：OS 类型/版本、Node 应用名与版本、平台，以及由本机网络标识计算的 MD5 `device_id`。
+旁路状态：`~/.bailian/telemetry-uid`（遵循 `BAILIAN_CONFIG_DIR`）由 `ensureUid` 在首次远端上报时生成，不是事件落盘目标。
+
+底层 Node tracker 还会附加公共设备字段：OS 类型/版本、Node 应用名与版本、平台，以及内置的 `device_id`（本机网卡 MAC 的 MD5，保持不变）。稳定匿名 install id 写入 AEM 的 `uid`：来自 `~/.bailian/telemetry-uid`，首次远端上报时生成随机 UUID v4，以 0600 权限原子发布；生成失败时不上报 `uid`。遥测关闭时不生成该文件。本地 `telemetry.jsonl` 每行附带同名 `uid`。注意：这不是账号用户 ID，只是上报到 AEM `uid` 字段的匿名标识；与 Tracker 内置 `device_id` 无关。
+
+`ensureUid` 无锁（对齐 qwen-code `InstallationManager`）：稳态只读；缺失或损坏时同目录 temp 写满后 `rename` 发布（内容原子替换；并发首次创建时后写覆盖先写，磁盘最终为一个 UUID）。失败则省略 `uid`，不阻塞业务退出。
 
 从 2.8.2 起，每条命令事件通过 `commandPrefix` 显式记录调用入口，三种入口仍共用 `pid=bailian-cli-node`：
 
@@ -180,6 +184,8 @@ AEM 映射：
 - [ ] 成功与失败均覆盖；遥测异常必须静默且不改变业务退出码
 - [ ] runtime 本地语义错误应记录 `exitCode`；新增字段默认随 AEM `ext` 上报，无需占用新的 `c1`—`c5`
 - [ ] 检查 `DO_NOT_TRACK=1` 与 `telemetry: false` 两个关闭入口
+- [ ] `remoteSink` 从调用到 `remoteSendEvent` 入队之间不得出现 `await`（flush 只排空已入队事件）
+- [ ] 测试中调用 `remoteSink` 的用例必须使用 `enterIsolatedHome()`，避免写入真实配置目录
 - [ ] 遥测发送的 DNS/HTTP/响应体必须受可取消预算控制；flush 超时必须取消在途操作，不能只停止等待，也不能修改全局 DNS/dispatcher 或强制退出宿主
 - [ ] 退出生命周期变更保留真实子进程回归：挂起响应头/响应体、DNS 预查询和关闭遥测；检查实际退出及进程仍存活时的连接释放，不以 flush Promise 返回代替验收
 - [ ] 错误字段不得额外拼接 token、请求体、prompt 或本地路径
@@ -190,7 +196,7 @@ AEM 映射：
 rg -n "trackingHeaders|x-dashscope-source-config|User-Agent" packages --glob '*.ts'
 rg -n "trackCommandExecution|PARAM_ALLOWLIST|buildRemoteAemOptions|readCaller" packages/core packages/runtime --glob '*.ts'
 vp check
-vp test packages/core/tests packages/runtime/tests/telemetry-stage.test.ts packages/commands/tests/e2e/auth.e2e.test.ts
+vp test packages/core/tests/telemetry-uid.test.ts packages/core/tests/telemetry-sink.test.ts packages/core/tests/telemetry-tracker.test.ts packages/core/tests/telemetry-send.test.ts packages/runtime/tests/telemetry-stage.test.ts packages/commands/tests/e2e/auth.e2e.test.ts
 ```
 
 ## 常见漏点

packages/core/src/telemetry/sink.ts

@@ -3,6 +3,7 @@ import { join } from "path";
 import { getConfigDir, ensureConfigDir } from "../config/paths.ts";
 import { buildRemoteAemOptions, type TrackingEvent } from "./event.ts";
 import { detectEnv } from "./env.ts";
+import { ensureUid, loadUid } from "./uid.ts";
 import Tracker from "../../lib/remote-telemetry/tracker.cjs";
 import EventPlugin from "../../lib/remote-telemetry/event-plugin.cjs";
 
@@ -12,74 +13,90 @@ const MAX_FILE_SIZE = 5 * 1024 * 1024; // 5 MB
 
 let remoteSendEvent: ((name: string, opts: Record<string, unknown>) => void) | undefined;
 
-// 追踪在途（in-flight）的远端发送请求，以便在进程退出前等待它们完成。
-// 底层 tracker 是 fire-and-forget 模式（发起 fetch 后不再持有 promise）；
-// 如果不追踪，CLI 在短命令或 SIGINT 场景下会在请求真正落网络前就退出。
-// 配合 tracker.js 中的 `keepalive: true`，await 这些 promise 可以在不阻塞
-// 主流程的前提下，尽最大努力把埋点送达。
+// Track in-flight remote sends so we can wait for them before process exit.
+// The underlying tracker is fire-and-forget (fetch starts without holding the promise);
+// without tracking, short commands or SIGINT can exit before the request hits the network.
+// With `keepalive: true` in tracker.js, awaiting these promises best-effort delivers
+// telemetry without blocking the main command path.
 const SEND_TIMEOUT_MS = 1000;
 interface PendingSend {
   controller: AbortController;
   promise: Promise<void>;
 }
 const inflightSends = new Set<PendingSend>();
 let remoteClient: any = undefined;
+let remoteInitialized = false;
 
-try {
-  // env 传给 Tracker 顶层 config 后会进 URL 查询参数, AEM 后台 "环境" 列
-  // (原始字段 env) 直接读它, 值域 prod/pre/daily/dev。我们目前只用 prod/dev。
-  const client = new Tracker({ pid: "bailian-cli-node", env: detectEnv() });
-  // 仅对当前实例的 `send` 做猴补丁，把每次 fetch 的 promise 收集起来。
-  // 不修改 prototype，避免污染其他可能存在的 Tracker 使用方。
-  const originalSend = client.send.bind(client);
-  client.send = function (payload: string) {
-    const controller = new AbortController();
-    const deadline = setTimeout(() => controller.abort(), SEND_TIMEOUT_MS);
-    deadline.unref?.();
-    const pending: PendingSend = {
-      controller,
-      promise: Promise.resolve()
-        .then(() => originalSend(payload, { signal: controller.signal }))
-        .catch(() => {})
-        .finally(() => {
-          // Bun may resolve body.cancel() before stopping its underlying fetch.
-          // Finish this send's private scope before removing its deadline.
-          controller.abort();
-          clearTimeout(deadline);
-          inflightSends.delete(pending);
-        }),
+/**
+ * Construct the tracker on first remote report so disabled telemetry neither
+ * constructs the tracker nor creates the uid file. Fully synchronous; see ensureUid.
+ */
+function initRemote(): void {
+  if (remoteInitialized) return;
+  remoteInitialized = true;
+  try {
+    const uid = ensureUid();
+    // `env` on the Tracker top-level config becomes a URL query param; AEM's
+    // "environment" column (raw field `env`) reads it. Allowed values:
+    // prod/pre/daily/dev. We currently use only prod/dev.
+    // anonymous install id as `uid`, which AEM exposes as a dashboard field.
+    const client = new Tracker({
+      pid: "bailian-cli-node",
+      env: detectEnv(),
+      ...(uid ? { uid } : {}),
+    });
+    // Monkey-patch only this instance's `send` to collect each fetch promise.
+    // Do not touch the prototype so other Tracker consumers stay unaffected.
+    const originalSend = client.send.bind(client);
+    client.send = function (payload: string) {
+      const controller = new AbortController();
+      const deadline = setTimeout(() => controller.abort(), SEND_TIMEOUT_MS);
+      deadline.unref?.();
+      const pending: PendingSend = {
+        controller,
+        promise: Promise.resolve()
+          .then(() => originalSend(payload, { signal: controller.signal }))
+          .catch(() => {})
+          .finally(() => {
+            // Bun may resolve body.cancel() before stopping its underlying fetch.
+            // Finish this send's private scope before removing its deadline.
+            controller.abort();
+            clearTimeout(deadline);
+            inflightSends.delete(pending);
+          }),
+      };
+      inflightSends.add(pending);
+      return pending.promise;
     };
-    inflightSends.add(pending);
-    return pending.promise;
-  };
-  remoteClient = client;
-  remoteSendEvent = client.use(EventPlugin);
-} catch {
-  // 埋点逻辑任何异常都不能影响 CLI 主流程
+    remoteClient = client;
+    remoteSendEvent = client.use(EventPlugin);
+  } catch {
+    // Telemetry must never affect the CLI main flow
+  }
 }
 
 /**
- * 尽力等待所有在途的埋点发送完成（best-effort）。
+ * Best-effort wait for all in-flight telemetry sends.
  *
- * 1. 先调用 `_sendAll` 排空 tracker 内部的去抖队列，把还卡在 500ms 合并窗口里
- *    的事件立刻推上网络。
- * 2. 在预算内等待发送，预算耗尽时真正取消 DNS/HTTP/body 工作。
+ * 1. Call `_sendAll` to drain the tracker's debounce queue so events still in
+ *    the 500ms merge window are pushed to the network immediately.
+ * 2. Wait for sends within the budget; on timeout, cancel DNS/HTTP/body work.
  *
- * 埋点永远不应阻塞 CLI：调用方应传入较短的超时（例如 1000ms），并始终与超时
- * race。错误与超时一律静默吞掉。
+ * Telemetry must never block the CLI: callers should pass a short timeout
+ * (e.g. 1000ms) and always race against it. Errors and timeouts are swallowed.
  */
 export async function flushTelemetry(timeoutMs = 1000): Promise<void> {
   let deadline: ReturnType<typeof setTimeout> | undefined;
   try {
     if (remoteClient) {
       try {
-        // 排空去抖队列：把卡在 500ms 合并窗口里的事件立刻发出去
-        // （每次发出都会被追加到 inflightSends 里）。
+        // Drain the debounce queue so events stuck in the 500ms merge window
+        // are sent immediately (each send is appended to inflightSends).
         if (typeof remoteClient._sendAll === "function") {
           remoteClient._sendAll();
         }
       } catch {
-        // flush 自身绝不抛出
+        // flush itself must never throw
       }
     }
     if (inflightSends.size === 0) return;
@@ -94,7 +111,7 @@ export async function flushTelemetry(timeoutMs = 1000): Promise<void> {
       }),
     ]);
   } catch {
-    // 埋点逻辑任何异常都不能影响 CLI 主流程
+    // Telemetry must never affect the CLI main flow
   } finally {
     clearTimeout(deadline);
   }
@@ -111,20 +128,25 @@ export async function localSink(event: TrackingEvent): Promise<void> {
         unlinkSync(path);
       }
     } catch {
-      // 文件还不存在，忽略
+      // File does not exist yet; ignore
     }
 
-    appendFileSync(path, JSON.stringify(event) + "\n", { mode: 0o600 });
+    // Read-only; remoteSink creates the file on first report.
+    const uid = loadUid();
+    appendFileSync(path, JSON.stringify(uid ? { ...event, uid } : event) + "\n", {
+      mode: 0o600,
+    });
   } catch {
-    // 埋点逻辑任何异常都不能影响 CLI 主流程
+    // Telemetry must never affect the CLI main flow
   }
 }
 
 export async function remoteSink(event: TrackingEvent): Promise<void> {
   try {
+    initRemote();
     if (!remoteSendEvent) return;
     remoteSendEvent(event.command, buildRemoteAemOptions(event));
   } catch {
-    // 埋点逻辑任何异常都不能影响 CLI 主流程
+    // Telemetry must never affect the CLI main flow
   }
 }

packages/core/src/telemetry/uid.ts

@@ -0,0 +1,96 @@
+import { randomBytes, randomUUID } from "crypto";
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "fs";
+import { join } from "path";
+import { getConfigDir } from "../config/paths.ts";
+
+/** On-disk name for the anonymous install id reported to AEM as `uid`. */
+export const UID_FILE_NAME = "telemetry-uid";
+
+const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+
+type ReadResult =
+  | { kind: "ok"; id: string }
+  | { kind: "missing" }
+  | { kind: "invalid" }
+  | { kind: "unreadable" };
+
+export function getUidPath(): string {
+  return join(getConfigDir(), UID_FILE_NAME);
+}
+
+function readUidFile(path: string): ReadResult {
+  let raw: string;
+  try {
+    raw = readFileSync(path, "utf8");
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === "ENOENT"
+      ? { kind: "missing" }
+      : { kind: "unreadable" };
+  }
+  const id = raw.trim().toLowerCase();
+  return UUID_V4.test(id) ? { kind: "ok", id } : { kind: "invalid" };
+}
+
+/** Read-only: returns undefined when missing or invalid; never creates the file. */
+export function loadUid(): string | undefined {
+  const result = readUidFile(getUidPath());
+  return result.kind === "ok" ? result.id : undefined;
+}
+
+/** Same-directory temp + rename publish (atomic content swap; may overwrite). */
+function publishUid(path: string, id: string): void {
+  const tmp = `${path}.${randomBytes(6).toString("hex")}.tmp`;
+  try {
+    const fd = openSync(tmp, "wx", 0o600);
+    try {
+      // writeFileSync(fd) loops until the full buffer is written; writeSync can short-write.
+      writeFileSync(fd, `${id}\n`);
+    } finally {
+      closeSync(fd);
+    }
+    renameSync(tmp, path);
+  } finally {
+    try {
+      unlinkSync(tmp);
+    } catch {
+      /* rename removed tmp; ignore ENOENT */
+    }
+  }
+}
+
+/**
+ * Read or create the persistent anonymous install id (AEM `uid`).
+ *
+ * Must stay synchronous: remoteSink relies on no await between the call and
+ * event enqueue, otherwise flushTelemetry may return before the event is queued.
+ *
+ * Lock-free like qwen-code InstallationManager: steady-state is a plain read;
+ * missing/invalid content is published with temp+rename (last writer wins under
+ * concurrent first runs). Any failure returns undefined so remoteSink omits
+ * `uid`; built-in Tracker `device_id` is unchanged.
+ */
+export function ensureUid(): string | undefined {
+  const path = getUidPath();
+  const current = readUidFile(path);
+  if (current.kind === "ok") return current.id;
+  // Do not overwrite unreadable paths (permission errors, a directory in the way).
+  if (current.kind === "unreadable") return undefined;
+
+  try {
+    mkdirSync(getConfigDir(), { recursive: true, mode: 0o700 });
+    const id = randomUUID();
+    publishUid(path, id);
+    return id;
+  } catch {
+    // Read-only FS, disk full, etc.: stay silent and try reading once more.
+    return loadUid();
+  }
+}

packages/core/tests/telemetry-send.test.ts

@@ -18,6 +18,18 @@ beforeEach(() => {
 });
 afterEach(() => vi.restoreAllMocks());
 
+test("constructor uid reaches send config without replacing built-in device_id", () => {
+  const id = "0f8fad5b-d9cb-469f-a165-70867728950e";
+  const tracker = new Tracker({ pid: "fixture-pid", uid: id }) as unknown as {
+    getConfig(key: string): unknown;
+    _getSendConfig(): Record<string, unknown>;
+  };
+  expect(tracker.getConfig("uid")).toBe(id);
+  expect(tracker._getSendConfig().uid).toBe(id);
+  expect(tracker.getConfig("device_id")).toMatch(/^[0-9a-f]{32}$/);
+  expect(tracker._getSendConfig().device_id).toBe(tracker.getConfig("device_id"));
+});
+
 test("telemetry uses the bounded transport without a separate DNS preflight", async () => {
   const controller = new AbortController();
   const tracker = new Tracker({ pid: "fixture-pid", env: "dev" });