diff --git a/README.md b/README.md index b93a305..700773b 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ _See code: [src/commands/telemetry/index.js](https://github.com/adobe/aio-cli-pl ## Configuration The following values need to be set when this plugin is hosted by different CLIs - `aioTelemetry`: defined object in root cli package.json with values: - - `postUrl` : Where to post telemetry data - - `postHeaders`: Any specific headers that need to be posted with telemetry data (ex. x-api-key) + - `postUrl` : Where to post telemetry data (overrides the default New Relic ingest endpoint) + - `fetchHeaders`: Headers to send with the telemetry POST request (overrides the default New Relic ingest headers) - `productPrivacyPolicyLink`: A link to display to users when prompting to optIn - `productName`: How to refer to the cli when user is prompted to enable telemetry - this value is read from `displayName` or `name` of the cli's package.json @@ -48,22 +48,67 @@ The following values need to be set when this plugin is hosted by different CLIs - ex. To turn telemetry on run `${productBin} telemetry on` - this value is read from 'bin' of the cli's package.json, if the package exports more than 1 bin the first is used -## POST data +## Opting out via environment variable + +Set `AIO_TELEMETRY_DISABLED=1` (or any truthy value) to suppress all telemetry without modifying the persisted opt-in state. Useful for CI pipelines and scripted environments. -Here is an example of the event data as posted: +```sh +AIO_TELEMETRY_DISABLED=1 aio app deploy ``` -{ "id": 656915165813, - "timestamp": 1673404918437, - "_adobeio": { - "eventType": "telemetry-prompt", - "eventData": "accepted", - "cliVersion": "@adobe/aio-cli@9.1.1", - "clientId": 264421030538, - "commandDuration": 5661, - "commandFlags": "", - "commandSuccess": true, - "nodeVersion": "v14.20.0", - "osNameVersion": "macOS" - } -} + +## Flush architecture + +Telemetry events are sent via a **fire-and-forget detached subprocess** (`src/flush-worker.js`). The parent CLI process spawns the worker and immediately unrefs it, so the CLI exits at its normal time without waiting for the HTTP POST to complete. The worker owns the ingest endpoint and credentials. + +## Agent detection + +The plugin detects whether the CLI is being invoked by an AI agent or a human by inspecting environment variables at the time of the event. The detected context is included in every event as `invocation_context` (`"agent"` or `"human"`) and `agent_name`. + +Supported agents detected automatically: + +| Environment variable | Detected agent name | +|---|---| +| `AGENT` | value of the variable (or `"generic"` if `1`/`true`) | +| `AI_AGENT` | value of the variable (or `"generic"` if `1`/`true`) | +| `AIO_AGENT` | `aio-opt-in` | +| `AIO_INVOCATION_CONTEXT=agent` | `aio-opt-in` | +| `CURSOR_AGENT` | `cursor` | +| `CLAUDECODE` / `CLAUDE_CODE` | `claude` | +| `GEMINI_CLI` | `gemini` | +| `CODEX_SANDBOX` | `codex` | +| `AUGMENT_AGENT` | `augment` | +| `CLINE_ACTIVE` | `cline` | +| `OPENCODE_CLIENT` | `opencode` | +| `REPL_ID` | `replit` | +| `PATH` containing `github.copilot-chat` | `github-copilot` | + +To opt into agent tracking without setting a tool-specific variable, set `AIO_INVOCATION_CONTEXT=agent`. + +## POST data + +Events are posted to the New Relic Metric API. Here is an example of the payload: + +```json +[{ + "metrics": [{ + "name": "aio.cli.telemetry", + "type": "gauge", + "value": 1, + "timestamp": 1673404918437, + "attributes": { + "eventType": "postrun", + "eventData": "{}", + "cliVersion": "@adobe/aio-cli@11.0.2", + "clientId": 264421030538, + "command": "app:deploy", + "commandDuration": 5661, + "commandFlags": "", + "commandSuccess": true, + "nodeVersion": "v22.21.1", + "osNameVersion": "macOS Sequoia 15.4", + "invocation_context": "human", + "agent_name": "unknown" + } + }] +}] ``` diff --git a/package.json b/package.json index 730529a..a773fd3 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,7 @@ "@oclif/core": "^4", "ci-info": "^4.0.0", "debug": "^4.1.1", - "inquirer": "^8.2.1", - "os-name": "^4.0.1", - "splunk-logging": "^0.11.1" + "inquirer": "^8.2.1" }, "devDependencies": { "@adobe/eslint-config-aio-lib-config": "5.0.0", diff --git a/src/flush-worker.js b/src/flush-worker.js new file mode 100644 index 0000000..06de68f --- /dev/null +++ b/src/flush-worker.js @@ -0,0 +1,79 @@ +/* +Copyright 2026 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + +/** + * Telemetry flush worker — spawned as a detached subprocess by trackEvent so + * the parent process can exit immediately without waiting on the HTTP POST. + * + * Accepts a single CLI argument: a JSON-encoded object with shape { body: string } + * where body is a serialised New Relic metric payload (array of metric batches). + * + * On each run the worker merges any previously-failed events from the persistent + * queue (src/queue-store.js) with the current event before POSTing. On success + * the queue is cleared; on failure the merged set is written back so the next + * invocation can retry. + */ + +'use strict' + +const { createFetch } = require('@adobe/aio-lib-core-networking') +const { readQueue, writeQueue, clearQueue } = require('./queue-store') + +const fetch = createFetch() + +const POST_URL = 'https://metric-api.newrelic.com/metric/v1' +const FETCH_HEADERS = { + 'Content-Type': 'application/json', + // New Relic ingest key — write-only, cannot read data or access any other system. + 'Api-Key': 'd6b73f9c1859dc462e6de8dee3de1eb2FFFFNRAL' +} + +/** + * Reads the persistent queue, merges it with the current event, POSTs the batch, + * and either clears the queue on success or writes back on failure for retry. + * @returns {Promise} + */ +async function main () { + // Parse the current event payload passed by the parent process. + let currentMetrics + try { + const { body } = JSON.parse(process.argv[2]) + currentMetrics = JSON.parse(body)[0].metrics + } catch { + // Malformed argument — nothing useful to do. + return + } + + // Merge previously-queued metrics (if any) with the current event so they + // are all retried in a single POST. + const queuedMetrics = readQueue() + const allMetrics = [...queuedMetrics, ...currentMetrics] + + try { + await fetch(POST_URL, { + method: 'POST', + headers: FETCH_HEADERS, + body: JSON.stringify([{ metrics: allMetrics }]) + }) + // Successful delivery — the queue is no longer needed. + clearQueue() + } catch { + // Network failure — persist all metrics so the next invocation can retry. + writeQueue(allMetrics) + } +} + +/* istanbul ignore next */ +if (require.main === module) { + main() +} + +module.exports = { main } diff --git a/src/queue-store.js b/src/queue-store.js new file mode 100644 index 0000000..fdf9310 --- /dev/null +++ b/src/queue-store.js @@ -0,0 +1,81 @@ +/* +Copyright 2026 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + +/** + * Persistent queue store for telemetry events that failed to POST. + * + * The queue is kept in a dedicated JSON file that lives alongside the main aio + * config directory but is completely separate from user-visible aio configuration: + * + * ${XDG_CONFIG_HOME:-~/.config}/aio/.telemetry-queue.json + * + * On the next CLI invocation the flush worker picks up any queued events, merges + * them with the new event, and retries the batch. On success the file is removed. + */ + +'use strict' + +const fs = require('fs') +const path = require('path') +const os = require('os') + +/** + * Resolves the absolute path to the queue file, honouring XDG_CONFIG_HOME when set. + * @returns {string} Absolute path to .telemetry-queue.json. + */ +function getQueuePath () { + const base = process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config') + return path.join(base, 'aio', '.telemetry-queue.json') +} + +/** + * Reads the current queue from disk. + * Returns an empty array when the file does not exist or is unreadable. + * @returns {Array} Flat array of New Relic metric objects. + */ +function readQueue () { + try { + const data = fs.readFileSync(getQueuePath(), 'utf8') + const parsed = JSON.parse(data) + return Array.isArray(parsed) ? parsed : [] + } catch { + return [] + } +} + +/** + * Persists the given metrics array to the queue file, creating directories as needed. + * Silently ignores write errors — telemetry must never crash the CLI. + * @param {Array} items Flat array of New Relic metric objects. + */ +function writeQueue (items) { + const file = getQueuePath() + try { + fs.mkdirSync(path.dirname(file), { recursive: true }) + fs.writeFileSync(file, JSON.stringify(items), 'utf8') + } catch { + // silently ignore — failing to persist the queue must not affect the CLI + } +} + +/** + * Removes the queue file. + * Silently ignores errors (e.g. the file does not exist). + */ +function clearQueue () { + try { + fs.unlinkSync(getQueuePath()) + } catch { + // silently ignore — queue file may not exist + } +} + +module.exports = { getQueuePath, readQueue, writeQueue, clearQueue } diff --git a/src/telemetry-lib.js b/src/telemetry-lib.js index f40bfac..27d6141 100644 --- a/src/telemetry-lib.js +++ b/src/telemetry-lib.js @@ -9,28 +9,81 @@ OF ANY KIND, either express or implied. See the License for the specific languag governing permissions and limitations under the License. */ -const { createFetch } = require('@adobe/aio-lib-core-networking') +const { spawn } = require('child_process') +const path = require('path') +const os = require('os') const config = require('@adobe/aio-lib-core-config') -const fetch = createFetch() -const osName = require('os-name') const inquirer = require('inquirer') const debug = require('debug')('aio-telemetry:telemetry-lib') let isDisabledForCommand = false -const osNameVersion = osName() +/** + * Detects GitHub Copilot Chat command shims injected into PATH. + * + * @param {string} pathValue - PATH environment variable value. + * @returns {string|null} Agent name when detected, otherwise null. + */ +function detectCopilotAgent (pathValue) { + if (pathValue.includes('github.copilot-chat/debugCommand') || pathValue.includes('github.copilot-chat/copilotCli')) { + return 'github-copilot' + } + return null +} -// this is set by the init hook, ex. @adobe/aio-cli@8.2.0 +// TODO: detect VSCODE run as an agent +/** + * Environment variables checked for agent detection (proposed standard first, then tool-specific). + * Used for metrics only. See aio-cli README "Agent detection" for full list. + */ +const AGENT_ENV_VARS = [ + { env: 'AGENT', name: (v) => (v && v !== '1' && v !== 'true' ? String(v).toLowerCase() : 'generic') }, + { env: 'AI_AGENT', name: (v) => (v && v !== '1' && v !== 'true' ? String(v).toLowerCase() : 'generic') }, + { env: 'AIO_AGENT', name: () => 'aio-opt-in' }, + { env: 'AIO_INVOCATION_CONTEXT', name: (v) => (v === 'agent' ? 'aio-opt-in' : null) }, + { env: 'CURSOR_AGENT', name: () => 'cursor' }, + { env: 'CLAUDECODE', name: () => 'claude' }, + { env: 'CLAUDE_CODE', name: () => 'claude' }, + { env: 'GEMINI_CLI', name: () => 'gemini' }, + { env: 'CODEX_SANDBOX', name: () => 'codex' }, + { env: 'AUGMENT_AGENT', name: () => 'augment' }, + { env: 'CLINE_ACTIVE', name: () => 'cline' }, + { env: 'OPENCODE_CLIENT', name: () => 'opencode' }, + { env: 'PATH', name: detectCopilotAgent }, + { env: 'REPL_ID', name: () => 'replit' } +] + +/** + * Detects whether the CLI is being invoked by an AI agent (vs a human) using env vars. + * Used for metrics only. + * + * @param {object} [env] - Environment object to read (defaults to process.env when omitted). + * @returns {{ isAgent: boolean, agentName: string|null }} Invocation context metadata. + */ +function getInvocationContext (env) { + const envToUse = env !== undefined ? env : process.env + for (const { env: key, name } of AGENT_ENV_VARS) { + const value = envToUse[key] + if (value !== undefined && value !== '') { + const agentName = name(value) + if (agentName) { + return { isAgent: true, agentName } + } + } + } + return { isAgent: false, agentName: null } +} + +const osNameVersion = `${os.type()} ${os.release()}` + +// this is set by the init hook, ex. @adobe/aio-cli@8.2.0q let rootCliVersion = '?' let prerunEvent = { flags: [] } -// postUrl and fetchHeaders are set by the init hook if these values are set in the root cli package.json -let postUrl = 'https://dcs.adobedc.net/collection/ffb5bdcefe744485c5c968662012f91293eee10f5dac4ca009beb14d7c028424?asynchronous=true' + let fetchHeaders = { 'Content-Type': 'application/json', - 'x-adobe-flow-id': '18dce8db-f523-4ff1-8806-0719de3fd367', - 'x-api-key': 'adobe_io', - 'sandbox-name': 'developer-lifecycle-dev1' + 'Api-Key': 'd6b73f9c1859dc462e6de8dee3de1eb2FFFFNRAL' } let configKey = 'aio-cli-telemetry' const defaultPrivacyPolicyLink = 'https://developer.adobe.com/app-builder/docs/guides/telemetry/' @@ -60,42 +113,50 @@ const getOffMessage = (binName) => { * @param {string} eventData additional data, like the error message, or custom telemetry payload * @returns {undefined} */ -async function trackEvent (eventType, eventData = '') { +async function trackEvent (eventType, eventData = {}) { // prerunEvent will be null when telemetry-prompt event fires, this happens before // any command is actually run, so we want to ignore the command+flags in this case - if (isDisabledForCommand || config.get(`${configKey}.optOut`, 'global') === true) { + if (isDisabledForCommand || process.env.AIO_TELEMETRY_DISABLED || config.get(`${configKey}.optOut`, 'global') === true) { debug('Telemetry is off. Not logging telemetry event', eventType) } else { const clientId = getClientId() const timestamp = Date.now() + const invocationContext = getInvocationContext() const fetchConfig = { method: 'POST', headers: fetchHeaders, - body: JSON.stringify({ - id: Math.floor(timestamp * Math.random()), - timestamp, - _adobeio: { - eventType, - eventData, - cliVersion: rootCliVersion, - clientId, - command: prerunEvent.command, - commandDuration: timestamp - prerunEvent.start, - commandFlags: prerunEvent.flags.toString(), - commandSuccess: eventType !== 'command-error', - nodeVersion: process.version, - osNameVersion - } - }) - } - try { - debug('posting telemetry event', fetchConfig) - const response = await fetch(postUrl, fetchConfig) - debug('response.ok = ', response.ok) - } catch (exc) { - debug('error reaching telemetry server : ', exc) + body: JSON.stringify([{ + metrics: [{ + name: 'aio.cli.telemetry', + type: 'gauge', + value: 1, + // id: Math.floor(timestamp * Math.random()), + timestamp, + attributes: { + eventType, + eventData: JSON.stringify(eventData), + cliVersion: rootCliVersion, + clientId, + command: prerunEvent.command, + commandDuration: timestamp - prerunEvent.start, + commandFlags: prerunEvent.flags.toString(), + commandSuccess: eventType !== 'command-error', + nodeVersion: process.version, + osNameVersion, + invocation_context: /* istanbul ignore next */ invocationContext.isAgent ? 'agent' : 'human', + agent_name: /* istanbul ignore next */ invocationContext.agentName || 'unknown' + } + }] + }]) } + const flushPayload = JSON.stringify({ body: fetchConfig.body }) + const child = spawn(process.execPath, [path.join(__dirname, 'flush-worker.js'), flushPayload], { + env: { ...process.env, AIO_TELEMETRY_DISABLED: '1' }, + detached: true, + stdio: 'ignore' + }) + child.unref() } } @@ -109,15 +170,13 @@ function trackPrerun (command, flags, start) { } module.exports = { + getInvocationContext, init: (versionString, binName, remoteConf = {}) => { global.commandHookStartTime = Date.now() rootCliVersion = versionString if (remoteConf.fetchHeaders) { fetchHeaders = remoteConf.fetchHeaders } - if (remoteConf.postUrl) { - postUrl = remoteConf.postUrl - } configKey = binName + '-cli-telemetry' }, getClientId, @@ -128,13 +187,13 @@ module.exports = { config.set(`${configKey}.optOut`, true) }, isEnabled: () => { - return !isDisabledForCommand && config.get(`${configKey}.optOut`, 'global') === false + return !isDisabledForCommand && !process.env.AIO_TELEMETRY_DISABLED && config.get(`${configKey}.optOut`, 'global') === false }, disableForCommand: () => { isDisabledForCommand = true }, isNull: () => { - return config.get(`${configKey}.optOut`, 'global') === undefined + return !process.env.AIO_TELEMETRY_DISABLED && config.get(`${configKey}.optOut`, 'global') === undefined }, trackEvent, trackPrerun, diff --git a/test/flush-worker.test.js b/test/flush-worker.test.js new file mode 100644 index 0000000..f23320b --- /dev/null +++ b/test/flush-worker.test.js @@ -0,0 +1,104 @@ +/* + * Copyright 2026 Adobe Inc. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +const { createFetch } = require('@adobe/aio-lib-core-networking') + +jest.mock('../src/queue-store', () => ({ + readQueue: jest.fn(() => []), + writeQueue: jest.fn(), + clearQueue: jest.fn() +})) + +const fetch = createFetch() +const { readQueue, writeQueue, clearQueue } = require('../src/queue-store') +const { main } = require('../src/flush-worker') + +const METRIC = { name: 'aio.cli.telemetry', type: 'gauge', value: 1, timestamp: 1000, attributes: { eventType: 'postrun' } } +const BODY = JSON.stringify([{ metrics: [METRIC] }]) + +describe('flush-worker main()', () => { + let origArgv + + beforeEach(() => { + origArgv = process.argv + fetch.mockReset() + readQueue.mockClear() + writeQueue.mockClear() + clearQueue.mockClear() + }) + + afterEach(() => { + process.argv = origArgv + }) + + test('POSTs merged metrics and clears queue on success', async () => { + const queued = [{ name: 'aio.cli.telemetry', value: 1, attributes: { eventType: 'prerun' } }] + readQueue.mockReturnValue(queued) + fetch.mockResolvedValue({ ok: true }) + + process.argv = ['node', 'flush-worker.js', JSON.stringify({ body: BODY })] + await main() + + expect(fetch).toHaveBeenCalledTimes(1) + const [url, opts] = fetch.mock.calls[0] + expect(url).toBe('https://metric-api.newrelic.com/metric/v1') + expect(opts.method).toBe('POST') + expect(opts.headers['Api-Key']).toBeTruthy() + + const posted = JSON.parse(opts.body) + expect(posted[0].metrics).toHaveLength(2) + expect(posted[0].metrics[0]).toEqual(queued[0]) + expect(posted[0].metrics[1]).toEqual(METRIC) + + expect(clearQueue).toHaveBeenCalledTimes(1) + expect(writeQueue).not.toHaveBeenCalled() + }) + + test('writes merged metrics to queue on fetch failure', async () => { + readQueue.mockReturnValue([]) + fetch.mockRejectedValue(new Error('network error')) + + process.argv = ['node', 'flush-worker.js', JSON.stringify({ body: BODY })] + await main() + + expect(writeQueue).toHaveBeenCalledTimes(1) + expect(writeQueue).toHaveBeenCalledWith([METRIC]) + expect(clearQueue).not.toHaveBeenCalled() + }) + + test('merges empty queue with current event', async () => { + readQueue.mockReturnValue([]) + fetch.mockResolvedValue({ ok: true }) + + process.argv = ['node', 'flush-worker.js', JSON.stringify({ body: BODY })] + await main() + + const posted = JSON.parse(fetch.mock.calls[0][1].body) + expect(posted[0].metrics).toHaveLength(1) + expect(posted[0].metrics[0]).toEqual(METRIC) + expect(clearQueue).toHaveBeenCalledTimes(1) + }) + + test('returns silently when argv[2] is missing', async () => { + process.argv = ['node', 'flush-worker.js'] + await main() + expect(fetch).not.toHaveBeenCalled() + expect(writeQueue).not.toHaveBeenCalled() + }) + + test('returns silently when argv[2] is malformed JSON', async () => { + process.argv = ['node', 'flush-worker.js', 'not-json{{{'] + await main() + expect(fetch).not.toHaveBeenCalled() + expect(writeQueue).not.toHaveBeenCalled() + }) +}) diff --git a/test/hooks.test.js b/test/hooks.test.js index ff23cd6..87fced3 100644 --- a/test/hooks.test.js +++ b/test/hooks.test.js @@ -16,8 +16,12 @@ const config = require('@adobe/aio-lib-core-config') jest.mock('inquirer') jest.mock('@adobe/aio-lib-core-config') +jest.mock('child_process', () => ({ + spawn: jest.fn(() => ({ unref: jest.fn() })) +})) const fetch = createFetch() +const { spawn } = require('child_process') const mockPackageJson = { bin: { aio: '' }, @@ -31,24 +35,25 @@ const mockPackageJson = { describe('hook interfaces', () => { beforeEach(() => { fetch.mockReset() + spawn.mockClear() }) test('command-error', async () => { const hook = require('../src/hooks/command-error') expect(typeof hook).toBe('function') await hook({ message: 'msg' }) - expect(fetch).toHaveBeenCalledTimes(1) - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"command-error"') })) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"command-error"') }) test('command-not-found', async () => { const hook = require('../src/hooks/command-not-found') expect(typeof hook).toBe('function') await hook({ id: 'id' }) - expect(fetch).toHaveBeenCalledTimes(1) - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"command-not-found"') })) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"command-not-found"') }) /** @@ -64,9 +69,10 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ config: { name: 'name', version: '0.0.1', pjson: mockPackageJson }, argv: [] }) expect(inquirer.prompt).toHaveBeenCalled() - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"telemetry-prompt","eventData":"accepted"') })) - expect(fetch).toHaveBeenCalledTimes(1) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"telemetry-prompt"') + expect(flushPayload.body).toContain('accepted') process.env = preEnv }) @@ -79,7 +85,7 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ id: 'telemetry', config: { name: 'name', version: '0.0.1' }, argv: [] }) expect(inquirer.prompt).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() process.env = preEnv }) @@ -90,7 +96,7 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ id: 'telemetry', config: { name: 'name', version: '0.0.1' }, argv: [] }) expect(inquirer.prompt).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() }) test('init prompt - dont run when oclif is generating readme', async () => { @@ -100,7 +106,7 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ id: 'readme', config: { name: 'name', version: '0.0.1' }, argv: [] }) expect(inquirer.prompt).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() }) test('init prompt - dont run when oclif is generating readme and CI is off', async () => { @@ -112,7 +118,7 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ id: 'readme', config: { name: 'name', version: '0.0.1' }, argv: [] }) expect(inquirer.prompt).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() process.env = preEnv }) @@ -129,7 +135,7 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) expect(inquirer.prompt).not.toHaveBeenCalled() await hook({ config: { name: 'name', version: '0.0.1' }, argv: ['--verbose'] }) - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() expect(inquirer.prompt).not.toHaveBeenCalled() process.env = preEnv }) @@ -147,9 +153,10 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ config: { name: 'name', version: '0.0.1' }, argv: ['--verbose'] }) expect(inquirer.prompt).toHaveBeenCalled() - expect(fetch).toHaveBeenCalledTimes(1) - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"telemetry-prompt","eventData":"declined"') })) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"telemetry-prompt"') + expect(flushPayload.body).toContain('declined') process.env = preEnv }) @@ -162,18 +169,18 @@ describe('hook interfaces', () => { .mockReturnValueOnce(false) await hook({ message: 'msg' }) - expect(fetch).toHaveBeenCalledTimes(1) - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"telemetry-custom-event"') })) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"telemetry-custom-event"') }) test('postrun', async () => { const hook = require('../src/hooks/postrun') expect(typeof hook).toBe('function') await hook({ Command: { id: 'id' }, argv: ['--hello'] }) - expect(fetch).toHaveBeenCalledTimes(1) - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"_adobeio":{"eventType":"postrun"') })) + expect(spawn).toHaveBeenCalledTimes(1) + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"eventType":"postrun"') }) /** @@ -187,16 +194,16 @@ describe('hook interfaces', () => { config.get = jest.fn().mockReturnValue(undefined) await hook({ config: { name: 'name', version: '0.0.1' }, argv: ['--no-telemetry'] }) expect(inquirer.prompt).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() }) test('prerun', async () => { const hook = require('../src/hooks/prerun') expect(typeof hook).toBe('function') await hook({ Command: { id: 'id' }, argv: ['--hello'] }) - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() await hook({ Command: { id: 'id' }, argv: ['--hello', '--no-telemetry'] }) - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() }) test('prerun disables telemetry for postrun', async () => { @@ -205,6 +212,6 @@ describe('hook interfaces', () => { config.get.mockResolvedValue('clientidxyz') await preHook({ Command: { id: 'id' }, argv: ['--hello', '--no-telemetry'] }) await postHook({ Command: { id: 'id' }, argv: ['--hello', '--no-telemetry'] }) - expect(fetch).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() }) }) diff --git a/test/queue-store.test.js b/test/queue-store.test.js new file mode 100644 index 0000000..f73a22e --- /dev/null +++ b/test/queue-store.test.js @@ -0,0 +1,90 @@ +/* + * Copyright 2026 Adobe Inc. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +const path = require('path') +const os = require('os') + +jest.mock('fs') +const fs = require('fs') + +const queueStore = require('../src/queue-store') + +const DEFAULT_QUEUE_PATH = path.join(os.homedir(), '.config', 'aio', '.telemetry-queue.json') +const XDG_QUEUE_PATH = path.join('/xdg-home', 'aio', '.telemetry-queue.json') + +describe('queue-store', () => { + beforeEach(() => { + jest.resetAllMocks() + delete process.env.XDG_CONFIG_HOME + }) + + describe('getQueuePath', () => { + test('returns path under ~/.config/aio when XDG_CONFIG_HOME is not set', () => { + expect(queueStore.getQueuePath()).toBe(DEFAULT_QUEUE_PATH) + }) + + test('honours XDG_CONFIG_HOME when set', () => { + process.env.XDG_CONFIG_HOME = '/xdg-home' + expect(queueStore.getQueuePath()).toBe(XDG_QUEUE_PATH) + delete process.env.XDG_CONFIG_HOME + }) + }) + + describe('readQueue', () => { + test('returns parsed array when file contains valid JSON', () => { + const items = [{ name: 'aio.cli.telemetry', value: 1 }] + fs.readFileSync.mockReturnValue(JSON.stringify(items)) + expect(queueStore.readQueue()).toEqual(items) + }) + + test('returns empty array when file does not exist', () => { + fs.readFileSync.mockImplementation(() => { throw new Error('ENOENT') }) + expect(queueStore.readQueue()).toEqual([]) + }) + + test('returns empty array when file contains invalid JSON', () => { + fs.readFileSync.mockReturnValue('not-json{{{') + expect(queueStore.readQueue()).toEqual([]) + }) + + test('returns empty array when parsed value is not an array', () => { + fs.readFileSync.mockReturnValue(JSON.stringify({ foo: 'bar' })) + expect(queueStore.readQueue()).toEqual([]) + }) + }) + + describe('writeQueue', () => { + test('creates directory and writes items as JSON', () => { + const items = [{ name: 'aio.cli.telemetry', value: 1 }] + queueStore.writeQueue(items) + expect(fs.mkdirSync).toHaveBeenCalledWith(path.dirname(DEFAULT_QUEUE_PATH), { recursive: true }) + expect(fs.writeFileSync).toHaveBeenCalledWith(DEFAULT_QUEUE_PATH, JSON.stringify(items), 'utf8') + }) + + test('silently ignores write errors', () => { + fs.mkdirSync.mockImplementation(() => { throw new Error('EACCES') }) + expect(() => queueStore.writeQueue([])).not.toThrow() + }) + }) + + describe('clearQueue', () => { + test('deletes the queue file', () => { + queueStore.clearQueue() + expect(fs.unlinkSync).toHaveBeenCalledWith(DEFAULT_QUEUE_PATH) + }) + + test('silently ignores errors when file does not exist', () => { + fs.unlinkSync.mockImplementation(() => { throw new Error('ENOENT') }) + expect(() => queueStore.clearQueue()).not.toThrow() + }) + }) +}) diff --git a/test/telemetry-lib.test.js b/test/telemetry-lib.test.js index 45f4255..2c11e50 100644 --- a/test/telemetry-lib.test.js +++ b/test/telemetry-lib.test.js @@ -15,13 +15,18 @@ const telemetryLib = require('../src/telemetry-lib') const config = require('@adobe/aio-lib-core-config') jest.mock('@adobe/aio-lib-core-config') +jest.mock('child_process', () => ({ + spawn: jest.fn(() => ({ unref: jest.fn() })) +})) const fetch = createFetch() +const { spawn } = require('child_process') describe('telemetry-lib', () => { beforeEach(() => { jest.resetModules() fetch.mockReset() + spawn.mockClear() }) test('exports messages', async () => { @@ -48,7 +53,182 @@ describe('telemetry-lib', () => { await telemetryLib.trackEvent('test-event') expect(config.get).toHaveBeenCalledWith('binTest2-cli-telemetry.clientId') expect(config.get).toHaveBeenCalledWith('binTest2-cli-telemetry.optOut', 'global') - expect(fetch).toHaveBeenCalledWith(expect.any(String), - expect.objectContaining({ body: expect.stringContaining('"clientId":"clientidxyz"') })) + expect(spawn).toHaveBeenCalled() + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + expect(flushPayload.body).toContain('"clientId":"clientidxyz"') + }) + + test('trackEvent includes invocation_context and agent_name in payload', async () => { + config.get.mockReturnValue('clientidxyz') + telemetryLib.init('a@4', 'binTest') + await telemetryLib.trackEvent('postrun') + expect(spawn).toHaveBeenCalled() + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + const body = JSON.parse(flushPayload.body) + const attributes = body[0].metrics[0].attributes + expect(attributes).toHaveProperty('invocation_context') + expect(attributes).toHaveProperty('agent_name') + expect(['agent', 'human']).toContain(attributes.invocation_context) + }) + + test('trackEvent does not post when AIO_TELEMETRY_DISABLED is set', async () => { + const orig = process.env.AIO_TELEMETRY_DISABLED + process.env.AIO_TELEMETRY_DISABLED = '1' + config.get.mockReturnValue('clientidxyz') + telemetryLib.init('a@4', 'binTest') + await telemetryLib.trackEvent('postrun') + expect(spawn).not.toHaveBeenCalled() + if (orig !== undefined) process.env.AIO_TELEMETRY_DISABLED = orig + else delete process.env.AIO_TELEMETRY_DISABLED + }) + + test('trackEvent sends agent context when CURSOR_AGENT env is set', async () => { + const orig = process.env.CURSOR_AGENT + process.env.CURSOR_AGENT = '1' + config.get.mockReturnValue('clientidxyz') + telemetryLib.init('a@4', 'binTest') + await telemetryLib.trackEvent('postrun') + expect(spawn).toHaveBeenCalled() + const flushPayload = JSON.parse(spawn.mock.calls[0][1][1]) + const body = JSON.parse(flushPayload.body) + const attributes = body[0].metrics[0].attributes + expect(attributes.invocation_context).toBe('agent') + expect(attributes.agent_name).toBe('cursor') + if (orig !== undefined) process.env.CURSOR_AGENT = orig + else delete process.env.CURSOR_AGENT + }) +}) + +describe('getInvocationContext', () => { + test('returns human when no agent env vars are set', () => { + const result = telemetryLib.getInvocationContext({}) + expect(result).toEqual({ isAgent: false, agentName: null }) + }) + + test('returns agent cursor when CURSOR_AGENT is set', () => { + const result = telemetryLib.getInvocationContext({ CURSOR_AGENT: '1' }) + expect(result).toEqual({ isAgent: true, agentName: 'cursor' }) + }) + + test('returns agent with name when AGENT is set to a value', () => { + const result = telemetryLib.getInvocationContext({ AGENT: 'goose' }) + expect(result).toEqual({ isAgent: true, agentName: 'goose' }) + }) + + test('returns agent generic when AGENT=1', () => { + const result = telemetryLib.getInvocationContext({ AGENT: '1' }) + expect(result).toEqual({ isAgent: true, agentName: 'generic' }) + }) + + test('returns aio-opt-in when AIO_AGENT is set', () => { + const result = telemetryLib.getInvocationContext({ AIO_AGENT: '1' }) + expect(result).toEqual({ isAgent: true, agentName: 'aio-opt-in' }) + }) + + test('returns aio-opt-in when AIO_INVOCATION_CONTEXT=agent', () => { + const result = telemetryLib.getInvocationContext({ AIO_INVOCATION_CONTEXT: 'agent' }) + expect(result).toEqual({ isAgent: true, agentName: 'aio-opt-in' }) + }) + + test('returns human when AIO_INVOCATION_CONTEXT is not agent', () => { + const result = telemetryLib.getInvocationContext({ AIO_INVOCATION_CONTEXT: 'human' }) + expect(result).toEqual({ isAgent: false, agentName: null }) + }) + + test('returns github-copilot when Copilot Chat PATH markers are present', () => { + const result = telemetryLib.getInvocationContext({ + PATH: '/usr/local/bin:/Users/test/Library/Application Support/Code/User/globalStorage/github.copilot-chat/debugCommand:/Users/test/Library/Application Support/Code/User/globalStorage/github.copilot-chat/copilotCli' + }) + expect(result).toEqual({ isAgent: true, agentName: 'github-copilot' }) + }) + + test('returns human when PATH does not contain Copilot Chat markers', () => { + const result = telemetryLib.getInvocationContext({ PATH: '/usr/local/bin:/usr/bin:/bin' }) + expect(result).toEqual({ isAgent: false, agentName: null }) + }) + + test('AGENT takes precedence over tool-specific when both set', () => { + const result = telemetryLib.getInvocationContext({ AGENT: 'goose', CURSOR_AGENT: '1' }) + expect(result).toEqual({ isAgent: true, agentName: 'goose' }) + }) + + test('ignores empty string env values', () => { + const result = telemetryLib.getInvocationContext({ CURSOR_AGENT: '' }) + expect(result).toEqual({ isAgent: false, agentName: null }) + }) + + test('returns agent generic when AGENT=true', () => { + const result = telemetryLib.getInvocationContext({ AGENT: 'true' }) + expect(result).toEqual({ isAgent: true, agentName: 'generic' }) + }) + + test('returns aio-opt-in when AI_AGENT is set', () => { + const result = telemetryLib.getInvocationContext({ AI_AGENT: 'my-agent' }) + expect(result).toEqual({ isAgent: true, agentName: 'my-agent' }) + }) + + test('returns generic when AI_AGENT=1', () => { + const result = telemetryLib.getInvocationContext({ AI_AGENT: '1' }) + expect(result).toEqual({ isAgent: true, agentName: 'generic' }) + }) + + test('returns claude when CLAUDECODE is set', () => { + expect(telemetryLib.getInvocationContext({ CLAUDECODE: '1' })).toEqual({ isAgent: true, agentName: 'claude' }) + }) + + test('returns claude when CLAUDE_CODE is set', () => { + expect(telemetryLib.getInvocationContext({ CLAUDE_CODE: '1' })).toEqual({ isAgent: true, agentName: 'claude' }) + }) + + test('returns gemini when GEMINI_CLI is set', () => { + expect(telemetryLib.getInvocationContext({ GEMINI_CLI: '1' })).toEqual({ isAgent: true, agentName: 'gemini' }) + }) + + test('returns codex when CODEX_SANDBOX is set', () => { + expect(telemetryLib.getInvocationContext({ CODEX_SANDBOX: '1' })).toEqual({ isAgent: true, agentName: 'codex' }) + }) + + test('returns augment when AUGMENT_AGENT is set', () => { + expect(telemetryLib.getInvocationContext({ AUGMENT_AGENT: '1' })).toEqual({ isAgent: true, agentName: 'augment' }) + }) + + test('returns cline when CLINE_ACTIVE is set', () => { + expect(telemetryLib.getInvocationContext({ CLINE_ACTIVE: '1' })).toEqual({ isAgent: true, agentName: 'cline' }) + }) + + test('returns opencode when OPENCODE_CLIENT is set', () => { + expect(telemetryLib.getInvocationContext({ OPENCODE_CLIENT: '1' })).toEqual({ isAgent: true, agentName: 'opencode' }) + }) + + test('returns replit when REPL_ID is set', () => { + expect(telemetryLib.getInvocationContext({ REPL_ID: 'abc123' })).toEqual({ isAgent: true, agentName: 'replit' }) + }) +}) + +describe('AIO_TELEMETRY_DISABLED', () => { + let orig + + beforeEach(() => { + orig = process.env.AIO_TELEMETRY_DISABLED + process.env.AIO_TELEMETRY_DISABLED = '1' + telemetryLib.init('a@4', 'binTest') + }) + + afterEach(() => { + if (orig !== undefined) process.env.AIO_TELEMETRY_DISABLED = orig + else delete process.env.AIO_TELEMETRY_DISABLED + }) + + test('isEnabled returns false when AIO_TELEMETRY_DISABLED is set', () => { + // config.get would return false (opted in), but the env var should override + const config = require('@adobe/aio-lib-core-config') + config.get.mockReturnValue(false) + expect(telemetryLib.isEnabled()).toBe(false) + }) + + test('isNull returns false when AIO_TELEMETRY_DISABLED is set', () => { + const config = require('@adobe/aio-lib-core-config') + config.get.mockReturnValue(undefined) + expect(telemetryLib.isNull()).toBe(false) }) })