diff --git a/.eslintrc.js b/.eslintrc.js index c9566a73..c1cb4a1e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,6 +17,18 @@ module.exports = { extends: ['@metamask/eslint-config-nodejs'], }, + { + files: ['*.mjs'], + parserOptions: { + sourceType: 'module', + ecmaVersion: 2022, + }, + extends: ['@metamask/eslint-config-nodejs'], + rules: { + 'import/extensions': 'off', + }, + }, + { files: ['*.test.ts', '*.test.js'], extends: ['@metamask/eslint-config-jest'], diff --git a/.github/actions/draft-segment-schema-pr/action.yml b/.github/actions/draft-segment-schema-pr/action.yml new file mode 100644 index 00000000..d23f5860 --- /dev/null +++ b/.github/actions/draft-segment-schema-pr/action.yml @@ -0,0 +1,381 @@ +name: Draft Segment schema PR +description: > + Detects analytics event/property changes on a Mobile or Extension pull request, + comments a proposal, and opens a draft Consensys/segment-schema PR after the + author agrees. Helper, not a merge gate. + +inputs: + platform: + description: 'mobile or extension' + required: true + mode: + description: 'propose (pull_request), create (issue_comment), or close (pull_request closed)' + required: true + github-token: + description: 'Token used to read the client PR and post comments on it' + required: false + default: ${{ github.token }} + segment-schema-token: + description: 'GitHub App token with contents:write and pull_requests:write on the schema repo' + required: true + segment-schema-repository: + description: 'Schema repository to write (owner/name)' + required: false + default: Consensys/segment-schema + segment-schema-base: + description: 'Schema branch to generate from and target with the draft PR' + required: false + default: main + default-library: + description: 'Override the unsorted events library id (platform default is used when empty)' + required: false + default: '' + github-tools-repository: + description: 'Repository containing this action. Defaults to the action repository.' + required: false + default: ${{ github.action_repository }} + github-tools-ref: + description: 'Ref of github-tools to check out. Defaults to the current action ref.' + required: false + default: ${{ github.action_ref }} + dry-run: + description: 'Generate and print a summary; skip git push and GitHub writes' + required: false + default: 'false' + +runs: + using: composite + steps: + - name: Gate + id: gate + uses: actions/github-script@v9 + env: + MODE: ${{ inputs.mode }} + PLATFORM: ${{ inputs.platform }} + SEGMENT_SCHEMA_TOKEN: ${{ inputs.segment-schema-token }} + SEGMENT_SCHEMA_REPOSITORY: ${{ inputs.segment-schema-repository }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_ACTION: ${{ github.event.action }} + COMMENT_BODY: ${{ github.event.comment.body }} + COMMENT_USER: ${{ github.event.comment.user.login }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + CLIENT_REPOSITORY: ${{ github.repository }} + with: + github-token: ${{ inputs.github-token }} + script: | + const path = require('path'); + const { + decideGate, + eventSkipReason, + collectTsDiff, + hasAnalyticsDiff, + botBranchName, + CATALOG_FILES, + OPT_OUT_LABEL, + PROPOSAL_MARKER, + } = await import(path.join(process.env.GITHUB_ACTION_PATH, 'gate.mjs')); + + const mode = process.env.MODE; + const platform = process.env.PLATFORM; + const eventName = process.env.GITHUB_EVENT_NAME ?? ''; + const eventAction = process.env.GITHUB_EVENT_ACTION ?? ''; + const prNumber = Number(process.env.PR_NUMBER ?? '0'); + + const writeOutputs = (decision, pr, schemaPrExists) => { + core.setOutput('skip', decision.skip ? 'true' : 'false'); + core.setOutput('skip_reason', decision.skipReason); + core.setOutput('pr_number', String(pr?.number ?? prNumber)); + core.setOutput('base_sha', pr?.baseSha ?? ''); + core.setOutput('head_sha', pr?.headSha ?? ''); + core.setOutput('merged', pr?.merged ? 'true' : 'false'); + core.setOutput('branch', decision.branch); + core.setOutput('schema_pr_exists', schemaPrExists ? 'true' : 'false'); + core.setOutput('should_push', decision.shouldPush ? 'true' : 'false'); + core.setOutput('has_analytics_diff', decision.hasAnalyticsDiff ? 'true' : 'false'); + core.setOutput('too_many_files', decision.tooManyFiles ? 'true' : 'false'); + }; + + const eventSkip = eventSkipReason(mode, eventName, eventAction); + if (eventSkip) { + writeOutputs( + { + skip: true, + skipReason: eventSkip, + shouldPush: false, + hasAnalyticsDiff: false, + tooManyFiles: false, + branch: botBranchName(platform, prNumber), + }, + null, + false, + ); + return; + } + + const [clientOwner, clientRepo] = process.env.CLIENT_REPOSITORY.split('/'); + const { data } = await github.rest.pulls.get({ + owner: clientOwner, + repo: clientRepo, + pull_number: prNumber, + }); + const labels = data.labels.map((label) => + typeof label === 'string' ? label : (label.name ?? ''), + ); + const headRepoFullName = data.head.repo?.full_name ?? ''; + const pr = { + number: data.number, + baseSha: data.base.sha, + headSha: data.head.sha, + authorLogin: data.user?.login ?? '', + isOpen: data.state === 'open', + merged: Boolean(data.merged), + isFork: headRepoFullName !== process.env.CLIENT_REPOSITORY, + hasOptOutLabel: labels.includes(OPT_OUT_LABEL), + }; + + const [schemaOwner, schemaRepo] = process.env.SEGMENT_SCHEMA_REPOSITORY.split('/'); + const branch = botBranchName(platform, pr.number); + const schemaPulls = await github.request('GET /repos/{owner}/{repo}/pulls', { + owner: schemaOwner, + repo: schemaRepo, + head: `${schemaOwner}:${branch}`, + state: 'open', + per_page: 5, + headers: { authorization: `token ${process.env.SEGMENT_SCHEMA_TOKEN}` }, + }); + const schemaPrExists = Boolean(schemaPulls.data[0]); + + let pullFiles = []; + let hasProposalComment = false; + if (mode !== 'close') { + pullFiles = await github.paginate(github.rest.pulls.listFiles, { + owner: clientOwner, + repo: clientRepo, + pull_number: pr.number, + per_page: 100, + }); + const diff = collectTsDiff(pullFiles); + const analyticsDiff = hasAnalyticsDiff(diff, platform); + if (!analyticsDiff && !schemaPrExists) { + const comments = await github.paginate(github.rest.issues.listComments, { + owner: clientOwner, + repo: clientRepo, + issue_number: pr.number, + per_page: 100, + }); + hasProposalComment = comments.some((comment) => + comment.body?.includes(PROPOSAL_MARKER), + ); + } + } + + const decision = decideGate({ + mode, + eventName, + eventAction, + pr, + commentBody: process.env.COMMENT_BODY ?? '', + commenterLogin: process.env.COMMENT_USER ?? '', + schemaPrExists, + hasProposalComment, + pullFiles, + platform, + }); + writeOutputs(decision, pr, schemaPrExists); + + - name: Checkout GitHub tools + if: ${{ steps.gate.outputs.skip != 'true' }} + uses: actions/checkout@v6 + with: + repository: ${{ inputs.github-tools-repository }} + ref: ${{ inputs.github-tools-ref }} + path: ./.github-tools + + - name: Enable Corepack + if: ${{ steps.gate.outputs.skip != 'true' }} + run: corepack enable + shell: bash + working-directory: ./.github-tools + + - name: Set up Node.js + if: ${{ steps.gate.outputs.skip != 'true' }} + uses: actions/setup-node@v6 + with: + node-version-file: ./.github-tools/.nvmrc + cache-dependency-path: ./.github-tools/yarn.lock + cache: yarn + + - name: Install dependencies + if: ${{ steps.gate.outputs.skip != 'true' }} + run: yarn --immutable + shell: bash + working-directory: ./.github-tools + + - name: Publish close + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode == 'close' }} + shell: bash + working-directory: ./.github-tools + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + SEGMENT_SCHEMA_TOKEN: ${{ inputs.segment-schema-token }} + PR_NUMBER: ${{ steps.gate.outputs.pr_number }} + run: | + extra=() + if [ "${{ inputs.dry-run }}" = "true" ]; then + extra+=(--dry-run) + fi + yarn run segment-schema:draft-pr \ + --phase publish \ + --mode close \ + --platform "${{ inputs.platform }}" \ + --pr-number "$PR_NUMBER" \ + --client-repository "${{ github.repository }}" \ + --segment-schema-repository "${{ inputs.segment-schema-repository }}" \ + --segment-schema-base "${{ inputs.segment-schema-base }}" \ + "${extra[@]}" + + - name: Checkout segment-schema + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' && steps.gate.outputs.has_analytics_diff == 'true' && steps.gate.outputs.too_many_files != 'true' }} + uses: actions/checkout@v6 + with: + repository: ${{ inputs.segment-schema-repository }} + ref: ${{ inputs.segment-schema-base }} + path: ./segment-schema + fetch-depth: 1 + token: ${{ inputs.segment-schema-token }} + persist-credentials: true + + - name: Probe previous bot branch + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' && steps.gate.outputs.has_analytics_diff == 'true' && steps.gate.outputs.too_many_files != 'true' }} + id: previous-ref + uses: actions/github-script@v9 + env: + SCHEMA_REPOSITORY: ${{ inputs.segment-schema-repository }} + BOT_BRANCH: ${{ steps.gate.outputs.branch }} + with: + github-token: ${{ inputs.segment-schema-token }} + script: | + const repository = process.env.SCHEMA_REPOSITORY; + const branch = process.env.BOT_BRANCH; + if (!repository || !branch) { + core.setOutput('exists', 'false'); + return; + } + const [owner, repo] = repository.split('/'); + try { + await github.rest.git.getRef({ + owner, + repo, + ref: `heads/${branch}`, + }); + core.setOutput('exists', 'true'); + } catch (error) { + if (error.status === 404) { + core.setOutput('exists', 'false'); + return; + } + throw error; + } + + - name: Checkout previous bot branch + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' && steps.gate.outputs.has_analytics_diff == 'true' && steps.gate.outputs.too_many_files != 'true' && steps.previous-ref.outputs.exists == 'true' }} + uses: actions/checkout@v6 + with: + repository: ${{ inputs.segment-schema-repository }} + ref: ${{ steps.gate.outputs.branch }} + path: ./segment-schema-previous + token: ${{ inputs.segment-schema-token }} + persist-credentials: false + + - name: Generate schema YAML + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' && steps.gate.outputs.has_analytics_diff == 'true' && steps.gate.outputs.too_many_files != 'true' }} + id: generate + shell: bash + working-directory: ./.github-tools + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + SEGMENT_SCHEMA_TOKEN: ${{ inputs.segment-schema-token }} + PR_NUMBER: ${{ steps.gate.outputs.pr_number }} + run: | + extra=() + if [ -d ../segment-schema-previous ]; then + extra+=(--previous-dir ../segment-schema-previous) + fi + if [ -n "${{ inputs.default-library }}" ]; then + extra+=(--default-library "${{ inputs.default-library }}") + fi + yarn run segment-schema:draft-pr \ + --phase generate \ + --mode "${{ inputs.mode }}" \ + --platform "${{ inputs.platform }}" \ + --schema ../segment-schema \ + --base-sha "${{ steps.gate.outputs.base_sha }}" \ + --head-sha "${{ steps.gate.outputs.head_sha }}" \ + --pr-number "$PR_NUMBER" \ + --client-repository "${{ github.repository }}" \ + --segment-schema-repository "${{ inputs.segment-schema-repository }}" \ + "${extra[@]}" + + - name: Push schema branch + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' && inputs.dry-run != 'true' && steps.gate.outputs.should_push == 'true' && steps.generate.outputs.has_writable_changes == 'true' }} + id: push + shell: bash + working-directory: ./segment-schema + env: + BRANCH: ${{ steps.gate.outputs.branch }} + PLATFORM: ${{ inputs.platform }} + PR_NUMBER: ${{ steps.gate.outputs.pr_number }} + SCHEMA_PR_EXISTS: ${{ steps.gate.outputs.schema_pr_exists }} + run: | + mkdir -p .git/info + echo '.segment-schema-draft-pr-summary.json' >> .git/info/exclude + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$BRANCH" + git fetch origin "$BRANCH" || true + git add -A + git reset -- .segment-schema-draft-pr-summary.json || true + compare_ref=HEAD + if [ "$SCHEMA_PR_EXISTS" = "true" ] && git rev-parse --verify --quiet "refs/remotes/origin/$BRANCH"; then + compare_ref="refs/remotes/origin/$BRANCH" + fi + if git diff --cached --quiet "$compare_ref"; then + echo "pushed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git commit -m "Draft schema for ${PLATFORM} PR #${PR_NUMBER}" + git push --force-with-lease origin "HEAD:${BRANCH}" + echo "pushed=true" >> "$GITHUB_OUTPUT" + + - name: Publish comments and schema PR + if: ${{ steps.gate.outputs.skip != 'true' && inputs.mode != 'close' }} + shell: bash + working-directory: ./.github-tools + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + SEGMENT_SCHEMA_TOKEN: ${{ inputs.segment-schema-token }} + PR_NUMBER: ${{ steps.gate.outputs.pr_number }} + run: | + extra=() + if [ "${{ inputs.dry-run }}" = "true" ]; then + extra+=(--dry-run) + fi + if [ "${{ steps.push.outputs.pushed }}" = "true" ]; then + extra+=(--pushed) + fi + if [ "${{ steps.gate.outputs.too_many_files }}" = "true" ]; then + extra+=(--too-many-files) + fi + if [ "${{ steps.gate.outputs.has_analytics_diff }}" != "true" ]; then + extra+=(--no-analytics-diff) + fi + yarn run segment-schema:draft-pr \ + --phase publish \ + --mode "${{ inputs.mode }}" \ + --platform "${{ inputs.platform }}" \ + --schema ../segment-schema \ + --pr-number "$PR_NUMBER" \ + --client-repository "${{ github.repository }}" \ + --segment-schema-repository "${{ inputs.segment-schema-repository }}" \ + --segment-schema-base "${{ inputs.segment-schema-base }}" \ + "${extra[@]}" diff --git a/.github/actions/draft-segment-schema-pr/gate.mjs b/.github/actions/draft-segment-schema-pr/gate.mjs new file mode 100644 index 00000000..a1187247 --- /dev/null +++ b/.github/actions/draft-segment-schema-pr/gate.mjs @@ -0,0 +1,265 @@ +/** + * Pre-install gate for draft-segment-schema-pr. + * + * Why: this runs in `actions/github-script` before github-tools is checked + * out and installed, so the constants are duplicated from + * `src/segment-schema-draft-pr/constants.ts` rather than imported. + */ + +export const AGREEMENT_PHRASE = 'I agree to open a draft Segment schema PR'; + +export const OPT_OUT_LABEL = 'no-schema-pr'; + +export const PROPOSAL_MARKER = ''; + +export const DIFF_PREFILTER = + /EVENT_NAME|MetaMetricsEventName|trackEvent|addProperties|createEventBuilder/u; + +export const MAX_CHANGED_TS_FILES = 150; + +export const CATALOG_FILES = { + mobile: 'app/core/Analytics/MetaMetrics.events.ts', + extension: 'shared/constants/metametrics.ts', +}; + +const TEST_PATH = + /(?:^|\/)(?:__tests__\/|(?:[^/]+\.)?(?:test|spec)\.[jt]sx?$)/u; + +/** + * Builds the deterministic bot branch for one client PR. + * + * @param {string} platform - Mobile or extension. + * @param {number} prNumber - Client pull request number. + * @returns {string} Branch name `metamaskbot/-pr-`. + */ +export function botBranchName(platform, prNumber) { + return `metamaskbot/${platform}-pr-${prNumber}`; +} + +/** + * True for production TypeScript paths the analytics extractor walks. + * + * @param {string} filePath - Path relative to the repository root. + * @returns {boolean} Whether the path is a non-test `.ts` / `.tsx` file. + */ +export function isNonTestTsFile(filePath) { + return ( + (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) && + !TEST_PATH.test(filePath) + ); +} + +/** + * Returns a skip reason when the GitHub event does not match the requested mode. + * + * @param {string} mode - Workflow mode: propose, create, or close. + * @param {string} eventName - Value of github.event_name. + * @param {string} eventAction - Value of github.event.action. + * @returns {string} Skip reason, or empty when the event matches. + */ +export function eventSkipReason(mode, eventName, eventAction) { + if (mode === 'propose' || mode === 'close') { + if (eventName !== 'pull_request') { + return 'wrong-event'; + } + if (mode === 'propose' && eventAction === 'closed') { + return 'wrong-event'; + } + if (mode === 'close' && eventAction !== 'closed') { + return 'wrong-event'; + } + } + if (mode === 'create' && eventName !== 'issue_comment') { + return 'wrong-event'; + } + return ''; +} + +/** + * True when the comment is the author's exact agreement sentence. + * + * @param {string} body - Comment body. + * @param {string} commenterLogin - Comment author. + * @param {string} prAuthorLogin - Pull request author. + * @returns {boolean} Whether create should proceed past the comment check. + */ +export function isAgreementComment(body, commenterLogin, prAuthorLogin) { + if (commenterLogin !== prAuthorLogin) { + return false; + } + const firstLine = (body.split('\n')[0] ?? '').trim(); + return firstLine === AGREEMENT_PHRASE; +} + +/** + * Lists non-test TypeScript paths and whether their patches mention analytics APIs. + * + * @param {{ filename: string, patch?: string | null, previous_filename?: string }[]} files - PR file list. + * @returns {{ files: string[], mentionsAnalytics: boolean }} Changed TS paths and pre-filter. + */ +export function collectTsDiff(files) { + const tsFiles = new Set(); + let mentionsAnalytics = false; + + for (const item of files) { + const candidates = [item.filename]; + if (item.previous_filename) { + candidates.push(item.previous_filename); + } + + const tsPaths = candidates.filter(isNonTestTsFile); + if (tsPaths.length === 0) { + continue; + } + + for (const filePath of tsPaths) { + tsFiles.add(filePath); + } + + if (item.patch === undefined || item.patch === null) { + mentionsAnalytics = true; + continue; + } + if (DIFF_PREFILTER.test(item.patch)) { + mentionsAnalytics = true; + } + } + + return { files: [...tsFiles], mentionsAnalytics }; +} + +/** + * True when the PR file list looks like an analytics change. + * + * @param {{ files: string[], mentionsAnalytics: boolean }} diff - Result of collectTsDiff. + * @param {string} platform - Mobile or extension. + * @returns {boolean} Whether generate should run. + */ +export function hasAnalyticsDiff(diff, platform) { + const catalog = CATALOG_FILES[platform]; + return diff.mentionsAnalytics || diff.files.includes(catalog); +} + +/** + * Whether the later generate step is allowed to git-push. + * + * @param {string} mode - Workflow mode. + * @param {boolean} schemaPrExists - Open schema PR on the bot branch. + * @param {boolean} clientPrOpen - Client PR still open (create only). + * @param {boolean} canWriteYaml - Analytics diff within the file cap. + * @returns {boolean} Whether a later generate and push is allowed. + */ +export function shouldPushSchema( + mode, + schemaPrExists, + clientPrOpen, + canWriteYaml, +) { + if (!canWriteYaml) { + return false; + } + if (mode === 'create' && clientPrOpen) { + return true; + } + return mode === 'propose' && schemaPrExists; +} + +/** + * Decides skip / push / analytics outputs before the toolchain install. + * + * @param {object} input - Gate inputs. + * @param {string} input.mode - Propose, create, or close. + * @param {string} input.eventName - GitHub event name. + * @param {string} input.eventAction - GitHub event action. + * @param {object} input.pr - Resolved client pull request. + * @param {number} input.pr.number - Pull request number. + * @param {boolean} input.pr.isFork - Head repo is not github.repository. + * @param {boolean} input.pr.hasOptOutLabel - Presence of the `no-schema-pr` label. + * @param {boolean} input.pr.isOpen - PR state is open. + * @param {boolean} input.pr.merged - PR was merged. + * @param {string} input.pr.authorLogin - PR author. + * @param {string} input.commentBody - Issue comment body. + * @param {string} input.commenterLogin - Issue comment author. + * @param {boolean} input.schemaPrExists - Open schema PR on the bot branch. + * @param {boolean} input.hasProposalComment - Sticky proposal comment exists. + * @param {{ filename: string, patch?: string | null, previous_filename?: string }[]} input.pullFiles - Pull request files. + * @param {string} input.platform - Mobile or extension. + * @returns {{ skip: boolean, skipReason: string, shouldPush: boolean, hasAnalyticsDiff: boolean, tooManyFiles: boolean, branch: string }} Gate decision. + */ +export function decideGate(input) { + const branch = botBranchName(input.platform, input.pr.number); + const skipped = (skipReason) => ({ + skip: true, + skipReason, + shouldPush: false, + hasAnalyticsDiff: false, + tooManyFiles: false, + branch, + }); + + const eventSkip = eventSkipReason( + input.mode, + input.eventName, + input.eventAction, + ); + if (eventSkip) { + return skipped(eventSkip); + } + + if (input.pr.isFork) { + return skipped('fork'); + } + if ( + (input.mode === 'propose' || input.mode === 'create') && + input.pr.hasOptOutLabel + ) { + return skipped('opt-out'); + } + if (input.mode === 'create' && !input.pr.isOpen) { + return skipped('client-pr-closed'); + } + if ( + input.mode === 'create' && + !isAgreementComment( + input.commentBody, + input.commenterLogin, + input.pr.authorLogin, + ) + ) { + return skipped('agreement-mismatch'); + } + if (input.mode === 'close' && !input.schemaPrExists) { + return skipped('no-schema-pr'); + } + + let analyticsDiff = true; + let tooManyFiles = false; + if (input.mode !== 'close') { + const diff = collectTsDiff(input.pullFiles ?? []); + tooManyFiles = diff.files.length > MAX_CHANGED_TS_FILES; + analyticsDiff = hasAnalyticsDiff(diff, input.platform); + + if ( + !tooManyFiles && + !analyticsDiff && + !input.schemaPrExists && + !input.hasProposalComment + ) { + return skipped('no-analytics-diff'); + } + } + + return { + skip: false, + skipReason: '', + shouldPush: shouldPushSchema( + input.mode, + input.schemaPrExists, + input.pr.isOpen, + analyticsDiff && !tooManyFiles, + ), + hasAnalyticsDiff: analyticsDiff, + tooManyFiles, + branch, + }; +} diff --git a/.github/actions/draft-segment-schema-pr/gate.test.mjs b/.github/actions/draft-segment-schema-pr/gate.test.mjs new file mode 100644 index 00000000..41c97d99 --- /dev/null +++ b/.github/actions/draft-segment-schema-pr/gate.test.mjs @@ -0,0 +1,225 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + AGREEMENT_PHRASE, + CATALOG_FILES, + MAX_CHANGED_TS_FILES, + botBranchName, + collectTsDiff, + decideGate, + eventSkipReason, + hasAnalyticsDiff, + isAgreementComment, + shouldPushSchema, +} from './gate.mjs'; + +const PR = { + number: 12, + isFork: false, + hasOptOutLabel: false, + isOpen: true, + merged: false, + authorLogin: 'alice', +}; + +/** + * Builds decideGate input with overrides. + * + * @param {object} overrides - Partial input. + * @returns {object} Full input. + */ +function input(overrides = {}) { + return { + mode: 'propose', + eventName: 'pull_request', + eventAction: 'synchronize', + pr: PR, + commentBody: '', + commenterLogin: '', + schemaPrExists: false, + hasProposalComment: false, + pullFiles: [ + { + filename: 'app/Home.ts', + patch: '+createEventBuilder(MetaMetricsEvents.APP_OPENED)', + }, + ], + platform: 'mobile', + ...overrides, + }; +} + +describe('eventSkipReason', () => { + it('skips propose on issue_comment and close unless the PR closed', () => { + assert.equal( + eventSkipReason('propose', 'issue_comment', 'created'), + 'wrong-event', + ); + assert.equal( + eventSkipReason('propose', 'pull_request', 'closed'), + 'wrong-event', + ); + assert.equal( + eventSkipReason('close', 'pull_request', 'synchronize'), + 'wrong-event', + ); + assert.equal(eventSkipReason('create', 'issue_comment', 'created'), ''); + assert.equal(eventSkipReason('propose', 'pull_request', 'synchronize'), ''); + assert.equal(eventSkipReason('close', 'pull_request', 'closed'), ''); + }); +}); + +describe('isAgreementComment', () => { + it('treats trailing whitespace on the agreement sentence as a match', () => { + assert.equal( + isAgreementComment(`${AGREEMENT_PHRASE} \nThanks`, 'alice', 'alice'), + true, + ); + assert.equal(isAgreementComment(AGREEMENT_PHRASE, 'bob', 'alice'), false); + assert.equal(isAgreementComment('please open it', 'alice', 'alice'), false); + }); +}); + +describe('collectTsDiff and hasAnalyticsDiff', () => { + it('keeps non-test TypeScript paths and detects analytics patches', () => { + const diff = collectTsDiff([ + { + filename: 'app/Home.ts', + patch: '+createEventBuilder(MetaMetricsEvents.APP_OPENED)', + }, + { filename: 'README.md', patch: '+docs' }, + { filename: 'app/foo.test.ts', patch: '+createEventBuilder(x)' }, + ]); + assert.deepEqual(diff.files, ['app/Home.ts']); + assert.equal(diff.mentionsAnalytics, true); + assert.equal(hasAnalyticsDiff(diff, 'mobile'), true); + }); + + it('treats a catalog-only change as an analytics hit', () => { + const diff = collectTsDiff([ + { + filename: CATALOG_FILES.mobile, + patch: "+NEW_EVENT = 'New Event'", + }, + ]); + assert.equal(diff.mentionsAnalytics, false); + assert.equal(hasAnalyticsDiff(diff, 'mobile'), true); + }); + + it('does not treat unrelated TypeScript patches as analytics hits', () => { + const diff = collectTsDiff([ + { filename: 'app/Home.ts', patch: '+const x = 1;\n' }, + ]); + assert.equal(diff.mentionsAnalytics, false); + assert.equal(hasAnalyticsDiff(diff, 'mobile'), false); + }); +}); + +describe('shouldPushSchema', () => { + it('pushes on create when open, and on propose only after a schema PR exists', () => { + assert.equal(shouldPushSchema('create', false, true, true), true); + assert.equal(shouldPushSchema('propose', false, true, true), false); + assert.equal(shouldPushSchema('propose', true, true, true), true); + assert.equal(shouldPushSchema('close', true, false, true), false); + assert.equal(shouldPushSchema('create', false, true, false), false); + }); +}); + +describe('decideGate', () => { + it('skips forks, opt-out, closed PRs, and non-agreement comments', () => { + assert.equal( + decideGate(input({ pr: { ...PR, isFork: true } })).skipReason, + 'fork', + ); + assert.equal( + decideGate(input({ pr: { ...PR, hasOptOutLabel: true } })).skipReason, + 'opt-out', + ); + assert.equal( + decideGate( + input({ + mode: 'create', + eventName: 'issue_comment', + eventAction: 'created', + pr: { ...PR, isOpen: false }, + commentBody: AGREEMENT_PHRASE, + commenterLogin: 'alice', + }), + ).skipReason, + 'client-pr-closed', + ); + assert.equal( + decideGate( + input({ + mode: 'create', + eventName: 'issue_comment', + eventAction: 'created', + }), + ).skipReason, + 'agreement-mismatch', + ); + }); + + it('skips close when no schema PR exists', () => { + const result = decideGate( + input({ + mode: 'close', + eventAction: 'closed', + schemaPrExists: false, + }), + ); + assert.equal(result.skip, true); + assert.equal(result.skipReason, 'no-schema-pr'); + }); + + it('skips propose when there is no analytics diff, schema PR, or sticky comment', () => { + const result = decideGate( + input({ + pullFiles: [{ filename: 'app/Home.ts', patch: '+const x = 1;\n' }], + }), + ); + assert.equal(result.skipReason, 'no-analytics-diff'); + }); + + it('continues propose so publish can edit a sticky comment when analytics drop off', () => { + const result = decideGate( + input({ + pullFiles: [{ filename: 'app/Home.ts', patch: '+const x = 1;\n' }], + hasProposalComment: true, + }), + ); + assert.equal(result.skip, false); + assert.equal(result.hasAnalyticsDiff, false); + assert.equal(result.shouldPush, false); + }); + + it('sets tooManyFiles and does not push when the TS file list exceeds the cap', () => { + const pullFiles = Array.from( + { length: MAX_CHANGED_TS_FILES + 1 }, + (_, i) => ({ + filename: `app/file-${i}.ts`, + patch: '+createEventBuilder(MetaMetricsEvents.APP_OPENED)', + }), + ); + const result = decideGate(input({ pullFiles, schemaPrExists: true })); + assert.equal(result.skip, false); + assert.equal(result.tooManyFiles, true); + assert.equal(result.shouldPush, false); + }); + + it('pushes on create when agreement matches and YAML can be written', () => { + const result = decideGate( + input({ + mode: 'create', + eventName: 'issue_comment', + eventAction: 'created', + commentBody: AGREEMENT_PHRASE, + commenterLogin: 'alice', + }), + ); + assert.equal(result.skip, false); + assert.equal(result.shouldPush, true); + assert.equal(result.branch, botBranchName('mobile', 12)); + }); +}); diff --git a/CHANGELOG.md b/CHANGELOG.md index 371bddbc..edfea9d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `draft-segment-schema-pr` composite action that proposes Segment schema YAML from Mobile or Extension analytics diffs (read through the GitHub API so the diff matches the pull request file list) and opens a draft schema PR after the author comments `I agree to open a draft Segment schema PR` + ## [1.19.0] ### Changed diff --git a/README.md b/README.md index 4c8c4ead..4089129c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ This repository holds a collection of scripts which are intended to be run local - `yarn run slack:release-testing`: Publishes a notification to slack for active releases regarding the release testing statuses. +- `yarn segment-schema:draft-pr`: Generates Segment schema YAML from a Mobile or Extension analytics diff (see [Draft Segment schema PR](#draft-segment-schema-pr)). + ### Authentication Some scripts require a GitHub token in order to run fully. @@ -60,3 +62,173 @@ DEBUG="metamask:*" Run `yarn test` to run the tests once. To run tests on file changes, run `yarn test:watch`. Run `yarn lint` to run the linter, or run `yarn lint:fix` to run the linter and fix any automatically fixable issues. + +## Draft Segment schema PR + +Composite action: `.github/actions/draft-segment-schema-pr`. It is a helper, not a merge gate. One Mobile or Extension pull request maps to one draft PR on `Consensys/segment-schema`. + +**Opt-in phrase** (first line of the PR author's comment, exact match, optional trailing whitespace): + +``` +I agree to open a draft Segment schema PR +``` + +### Modes + +A `github-script` gate runs before github-tools is installed. Forks, opt-out labels, comments that are not the agreement sentence, `close` when no schema PR exists, and PRs with no analytics-looking file list (and no sticky proposal to edit) end the job there. + +Pull requests that change more than 150 non-test TypeScript files skip YAML generation. The sticky comment says so; the author opens the schema PR manually. + +| Mode | Event | What it does | +| --------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `propose` | `pull_request` opened / synchronize / reopened / ready_for_review | After the gate, generate YAML and post or edit one proposal comment. Git-push only if an **open** schema PR already exists. | +| `create` | `issue_comment` created | First git-push and `pulls.create` (`draft: true`) when the commenter is the PR author, the client PR is still open, it is not a fork, and there are additive YAML changes. | +| `close` | `pull_request` closed | If a schema PR exists and the client PR was **not** merged, close the schema PR. If merged, leave it open. | + +Forks: `propose` / `close` skip when `head.repo.full_name != github.repository`. `create` matches Mobile/Extension `update-attributions.yml`: a first job runs `gh pr view -R` `--json isCrossRepository` and mints the GitHub App token only when that is `false`. + +Auth: jobs use `environment: segment-schema` and `actions/create-github-app-token@v3` (no `owner` / `repositories`). Pass `steps.app-token.outputs.token` as `segment-schema-token`. Every schema write names `Consensys/segment-schema` explicitly. + +### Local CLI + +From this repo, generate needs `GITHUB_TOKEN` (writes YAML into `--schema`; use a throwaway copy of `segment-schema`): + +``` +GITHUB_TOKEN=ghp_xxx yarn segment-schema:draft-pr \ + --phase generate \ + --mode propose \ + --platform mobile \ + --schema /path/to/segment-schema-copy \ + --client-repository MetaMask/metamask-mobile \ + --pr-number 12 \ + --base-sha \ + --head-sha +``` + +`--phase publish --dry-run` prints the summary path and skips GitHub writes. + +### Consumer workflow contract + +Land this in `metamask-mobile` and `metamask-extension` **after** this action ships. Pin a github-tools commit SHA until `@v1` exists. The workflow must not be a required status check. `issue_comment` workflows must live on the default branch. The `segment-schema` environment must have no required reviewers or wait timer. + +`paths-ignore` skips the workflow only when **every** changed file matches. Draft PRs and bot-authored PRs skip `propose`. Two concurrency groups keep a `propose` from cancelling a `create` between `git push` and `pulls.create`. + +```yaml +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, closed] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'locales/**' + - '**/*.test.*' + - '**/*.spec.*' + - '**/__snapshots__/**' + - 'e2e/**' + - 'ios/**' + - 'android/**' + - '**/*.json' + - '**/*.png' + - '**/*.svg' + - '.github/**' + issue_comment: + types: [created] +jobs: + propose: + if: > + github.event_name == 'pull_request' && + github.event.action != 'closed' && + github.event.pull_request.draft == false && + github.event.pull_request.user.type != 'Bot' && + github.event.pull_request.head.repo.full_name == github.repository && + !contains(github.event.pull_request.labels.*.name, 'no-schema-pr') + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: segment-schema + concurrency: + group: draft-segment-schema-propose-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: MetaMask/github-tools/.github/actions/draft-segment-schema-pr@ + with: + platform: mobile + mode: propose + segment-schema-token: ${{ steps.app-token.outputs.token }} + segment-schema-repository: Consensys/segment-schema + is-cross-repo-pr: + if: > + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.user.login == github.event.issue.user.login && + startsWith(github.event.comment.body, 'I agree to open a draft Segment schema PR') + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + IS_CROSS_REPO_PR: ${{ steps.is-cross-repo.outputs.IS_CROSS_REPO_PR }} + steps: + - id: is-cross-repo + run: echo "IS_CROSS_REPO_PR=$(gh pr view -R "$GITHUB_REPOSITORY" --json isCrossRepository --jq '.isCrossRepository' "${PR_NUMBER}")" >> "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + create: + needs: is-cross-repo-pr + if: ${{ needs.is-cross-repo-pr.outputs.IS_CROSS_REPO_PR == 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: segment-schema + concurrency: + group: draft-segment-schema-write-${{ github.event.issue.number }} + cancel-in-progress: false + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: MetaMask/github-tools/.github/actions/draft-segment-schema-pr@ + with: + platform: mobile + mode: create + segment-schema-token: ${{ steps.app-token.outputs.token }} + segment-schema-repository: Consensys/segment-schema + close-segment-schema-pr: + if: > + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: segment-schema + concurrency: + group: draft-segment-schema-write-${{ github.event.pull_request.number }} + cancel-in-progress: false + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: MetaMask/github-tools/.github/actions/draft-segment-schema-pr@ + with: + platform: mobile + mode: close + segment-schema-token: ${{ steps.app-token.outputs.token }} + segment-schema-repository: Consensys/segment-schema +``` + +Use `platform: extension` in the Extension repo. Do not land these workflows until a pin-able github-tools SHA exists. diff --git a/jest.config.js b/jest.config.js index 1a4bf0e7..d5a90829 100644 --- a/jest.config.js +++ b/jest.config.js @@ -41,10 +41,10 @@ module.exports = { // An object that configures minimum threshold enforcement for coverage results coverageThreshold: { global: { - branches: 0, - functions: 0, - lines: 0, - statements: 0, + branches: 47.69, + functions: 53.12, + lines: 53.65, + statements: 53.74, }, }, diff --git a/package.json b/package.json index 4f117eaa..3dfb1e10 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,13 @@ "lint:constraints": "yarn constraints", "lint:changelog": "auto-changelog validate --prettier", "lint:dependencies": "depcheck && yarn dedupe", - "lint:eslint": "eslint . --cache --ext js,ts", + "lint:eslint": "eslint . --cache --ext js,ts && eslint --no-ignore --cache --ext mjs .github/actions/draft-segment-schema-pr", "lint:fix": "yarn lint:eslint --fix && yarn lint:constraints --fix && yarn lint:misc --write && yarn lint:dependencies", - "lint:misc": "prettier '**/*.json' '**/*.md' '**/*.yml' '!.yarnrc.yml' --ignore-path .gitignore --no-error-on-unmatched-pattern", + "lint:misc": "prettier '**/*.json' '**/*.md' '**/*.yml' '!.yarnrc.yml' '!.cursor/**' --ignore-path .gitignore --no-error-on-unmatched-pattern", "lint:tsc": "tsc", + "segment-schema:draft-pr": "ts-node src/segment-schema-draft-pr/cli.ts", "slack:release-testing": "node .github/scripts/slack-release-testing.mjs", - "test": "jest && jest-it-up", + "test": "jest && node --test .github/actions/draft-segment-schema-pr/gate.test.mjs && jest-it-up", "test:watch": "jest --watch", "update-release-sheet": "node .github/scripts/update-release-sheet.mjs" }, @@ -42,7 +43,9 @@ "ora": "^5.4.1", "semver": "^7.7.2", "simple-git": "3.27.0", - "unzipper": "^0.12.3" + "typescript": "^5.1.3", + "unzipper": "^0.12.3", + "yaml": "^2.9.0" }, "devDependencies": { "@lavamoat/allow-scripts": "^2.3.1", @@ -73,8 +76,7 @@ "prettier": "^3.6.2", "prettier-plugin-packagejson": "^2.5.19", "ts-jest": "^28.0.7", - "ts-node": "^10.9.1", - "typescript": "^5.1.3" + "ts-node": "^10.9.1" }, "packageManager": "yarn@4.14.1", "engines": { diff --git a/src/env-utils.test.ts b/src/env-utils.test.ts new file mode 100644 index 00000000..9a0a44d8 --- /dev/null +++ b/src/env-utils.test.ts @@ -0,0 +1,14 @@ +import { getProcessEnv, getRequiredEnvironmentVariable } from './env-utils'; + +describe('env-utils', () => { + it('returns the process environment map', () => { + expect(typeof getProcessEnv()).toBe('object'); + expect(getProcessEnv().PATH).toBeDefined(); + }); + + it('throws when a required variable is missing', () => { + expect(() => + getRequiredEnvironmentVariable('GITHUB_TOOLS_MISSING_ENV_VAR'), + ).toThrow('Must set GITHUB_TOOLS_MISSING_ENV_VAR'); + }); +}); diff --git a/src/env-utils.ts b/src/env-utils.ts index 954432f3..d5615b1c 100644 --- a/src/env-utils.ts +++ b/src/env-utils.ts @@ -1,3 +1,14 @@ +/** + * Returns the process environment map. + * + * @returns The current process environment. + */ +export function getProcessEnv(): NodeJS.ProcessEnv { + // This function is designed to access `process.env`. + // eslint-disable-next-line n/no-process-env + return process.env; +} + /** * Retrieves the value of an environment variable, throwing if it doesn't exist. * @@ -6,9 +17,7 @@ * @returns The value of the environment variable. */ export function getRequiredEnvironmentVariable(name: string): string { - // This function is designed to access `process.env`. - // eslint-disable-next-line n/no-process-env - const value = process.env[name]; + const value = getProcessEnv()[name]; if (value === undefined) { throw new Error(`Must set ${name}`); diff --git a/src/segment-schema-draft-pr/analytics-model.test.ts b/src/segment-schema-draft-pr/analytics-model.test.ts new file mode 100644 index 00000000..328f71ee --- /dev/null +++ b/src/segment-schema-draft-pr/analytics-model.test.ts @@ -0,0 +1,124 @@ +import { extractAnalyticsModel } from './analytics-model'; +import { getPlatformConfig } from './config'; + +const MOBILE = getPlatformConfig('mobile'); +const EXTENSION = getPlatformConfig('extension'); + +describe('extractAnalyticsModel', () => { + it('reads EVENT_NAME catalog members', () => { + const source = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + BUTTON_CLICKED = 'Button Clicked', + } + `; + const model = extractAnalyticsModel('catalog.ts', source, MOBILE); + expect(model.catalog.get('APP_OPENED')).toBe('App Opened'); + expect(model.catalog.get('BUTTON_CLICKED')).toBe('Button Clicked'); + }); + + it('extracts createEventBuilder addProperties and addSensitiveProperties literals', () => { + const source = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + } + const MetaMetricsEvents = { APP_OPENED: EVENT_NAME.APP_OPENED }; + createEventBuilder(MetaMetricsEvents.APP_OPENED) + .addProperties({ usdValue: '1', count: 2, enabled: true, tags: ['a'] }) + .addSensitiveProperties({ secret: 'x' }) + .build(); + `; + const model = extractAnalyticsModel('call.ts', source, MOBILE); + const event = model.events.get('App Opened'); + expect(event?.properties.get('usd_value')).toBe('string'); + expect(event?.properties.get('count')).toBe('number'); + expect(event?.properties.get('enabled')).toBe('boolean'); + expect(event?.properties.get('tags')).toBe('array'); + expect(event?.properties.get('secret')).toBe('string'); + }); + + it('ignores generateOpt action and name properties', () => { + const source = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + } + createEventBuilder(MetaMetricsEvents.APP_OPENED).addProperties({ + action: 'click', + name: 'App Opened', + location: 'Home', + }); + `; + const model = extractAnalyticsModel('call.ts', source, MOBILE); + const event = model.events.get('App Opened'); + expect(event?.properties.has('action')).toBe(false); + expect(event?.properties.has('name')).toBe(false); + expect(event?.properties.get('location')).toBe('string'); + }); + + it('extracts extension trackEvent property bags', () => { + const source = ` + enum MetaMetricsEventName { + AppOpened = 'App Opened', + } + trackEvent({ + event: MetaMetricsEventName.AppOpened, + properties: { chainId: '0x1', nested: true }, + }); + `; + const model = extractAnalyticsModel('ui.ts', source, EXTENSION); + const event = model.events.get('App Opened'); + expect(event?.properties.get('chain_id')).toBe('string'); + expect(event?.properties.get('nested')).toBe('boolean'); + }); + + it('marks spreads, helpers, and unknown identifiers as unresolved', () => { + const source = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + } + createEventBuilder(MetaMetricsEvents.APP_OPENED).addProperties({ + ...extra, + helper: getProps(), + }); + `; + const model = extractAnalyticsModel('call.ts', source, MOBILE); + const event = model.events.get('App Opened'); + expect(event?.unresolved).toStrictEqual([ + { eventName: 'App Opened', key: '...', file: 'call.ts' }, + { eventName: 'App Opened', key: 'helper', file: 'call.ts' }, + ]); + }); + + it('resolves same-file as const object property types', () => { + const source = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + } + const Props = { usdValue: '1' } as const; + createEventBuilder(MetaMetricsEvents.APP_OPENED).addProperties({ + amount: Props.usdValue, + }); + `; + const model = extractAnalyticsModel('call.ts', source, MOBILE); + expect(model.events.get('App Opened')?.properties.get('amount')).toBe( + 'string', + ); + }); + + it('resolves event names using a shared catalog from another file', () => { + const { catalog } = extractAnalyticsModel( + 'catalog.ts', + `enum EVENT_NAME { NEW_EVENT = 'New Event' }`, + MOBILE, + ); + const model = extractAnalyticsModel( + 'call.ts', + `createEventBuilder(MetaMetricsEvents.NEW_EVENT).addProperties({ source: 'banner' });`, + MOBILE, + catalog, + ); + expect(model.events.get('New Event')?.properties.get('source')).toBe( + 'string', + ); + }); +}); diff --git a/src/segment-schema-draft-pr/analytics-model.ts b/src/segment-schema-draft-pr/analytics-model.ts new file mode 100644 index 00000000..a46e02fe --- /dev/null +++ b/src/segment-schema-draft-pr/analytics-model.ts @@ -0,0 +1,536 @@ +import ts from 'typescript'; + +import { IGNORED_GENERATE_OPT_PROPS } from './constants'; +import { toSnakeCase } from './names'; +import type { + AnalyticsModel, + EventCatalog, + EventModel, + PlatformConfig, + PropertyType, +} from './types'; + +type ConstObject = Map; + +/** + * Extracts catalog enum members and literal tracking call-site property bags. + * + * @param filePath - Path used in unresolved listings. + * @param sourceText - TypeScript source at one SHA. + * @param config - Platform catalog/enum names. + * @param sharedCatalog - Enum members from the platform catalog file, so call sites in other files can resolve event names. + * @returns Catalog map and per-event property models. + */ +export function extractAnalyticsModel( + filePath: string, + sourceText: string, + config: PlatformConfig, + sharedCatalog: EventCatalog | undefined = undefined, +): AnalyticsModel { + const sourceFile = ts.createSourceFile( + filePath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TSX, + ); + + const catalog: EventCatalog = new Map(sharedCatalog); + for (const [key, name] of extractEnumCatalog(sourceFile, config.enumName)) { + catalog.set(key, name); + } + const constObjects = collectConstObjects(sourceFile); + const events = new Map(); + + visit(sourceFile, (node) => { + if (!ts.isCallExpression(node)) { + return; + } + + const callName = getCalledName(node); + if (callName === 'createEventBuilder') { + collectCreateEventBuilder( + node, + catalog, + config, + constObjects, + filePath, + events, + ); + return; + } + + if (callName === 'trackEvent') { + collectTrackEvent(node, catalog, config, constObjects, filePath, events); + } + }); + + return { catalog, events }; +} + +/** + * Walks a node tree depth-first. + * + * @param node - Current node. + * @param onNode - Visitor. + */ +function visit(node: ts.Node, onNode: (node: ts.Node) => void): void { + onNode(node); + ts.forEachChild(node, (child) => visit(child, onNode)); +} + +/** + * Reads KEY -> display name from a string enum. + * + * @param sourceFile - Parsed source. + * @param enumName - `EVENT_NAME` or `MetaMetricsEventName`. + * @returns Catalog map. + */ +export function extractEnumCatalog( + sourceFile: ts.SourceFile, + enumName: string, +): EventCatalog { + const catalog: EventCatalog = new Map(); + + visit(sourceFile, (node) => { + if (!ts.isEnumDeclaration(node) || node.name.text !== enumName) { + return; + } + + for (const member of node.members) { + if (!ts.isIdentifier(member.name) || !member.initializer) { + continue; + } + if (!ts.isStringLiteral(member.initializer)) { + continue; + } + catalog.set(member.name.text, member.initializer.text); + } + }); + + return catalog; +} + +/** + * Collects same-file `as const` object property types for identifier resolution. + * + * @param sourceFile - Parsed source. + * @returns Object name -> property types. + */ +function collectConstObjects( + sourceFile: ts.SourceFile, +): Map { + const objects = new Map(); + + visit(sourceFile, (node) => { + if (!ts.isVariableDeclaration(node) || !ts.isIdentifier(node.name)) { + return; + } + if (!node.initializer) { + return; + } + + let objectLiteral: ts.ObjectLiteralExpression | undefined; + if (ts.isObjectLiteralExpression(node.initializer)) { + objectLiteral = node.initializer; + } else if ( + ts.isAsExpression(node.initializer) && + ts.isObjectLiteralExpression(node.initializer.expression) + ) { + objectLiteral = node.initializer.expression; + } + + if (!objectLiteral) { + return; + } + + const props: ConstObject = new Map(); + for (const property of objectLiteral.properties) { + if (!ts.isPropertyAssignment(property)) { + continue; + } + const key = propertyName(property.name); + if (!key) { + continue; + } + const inferred = inferLiteralType(property.initializer, objects); + if (inferred) { + props.set(key, inferred); + } + } + objects.set(node.name.text, props); + }); + + return objects; +} + +/** + * Records properties from createEventBuilder addProperties chains. + * + * @param node - The createEventBuilder call. + * @param catalog - Enum catalog. + * @param config - Platform config. + * @param constObjects - Same-file const objects. + * @param filePath - Source path. + * @param events - Accumulator. + */ +function collectCreateEventBuilder( + node: ts.CallExpression, + catalog: EventCatalog, + config: PlatformConfig, + constObjects: Map, + filePath: string, + events: Map, +): void { + const eventName = resolveEventName(node.arguments[0], catalog, config); + if (!eventName) { + return; + } + + const model = getOrCreateEvent(events, eventName); + const chain = chainedCalls(node); + + for (const call of chain) { + if ( + call.name !== 'addProperties' && + call.name !== 'addSensitiveProperties' + ) { + continue; + } + const bag = call.args[0]; + if (!bag) { + continue; + } + mergeBag(bag, eventName, filePath, constObjects, model); + } +} + +/** + * Records properties from trackEvent event/properties bags. + * + * @param node - The trackEvent call. + * @param catalog - Enum catalog. + * @param config - Platform config. + * @param constObjects - Same-file const objects. + * @param filePath - Source path. + * @param events - Accumulator. + */ +function collectTrackEvent( + node: ts.CallExpression, + catalog: EventCatalog, + config: PlatformConfig, + constObjects: Map, + filePath: string, + events: Map, +): void { + const arg = node.arguments[0]; + if (!arg || !ts.isObjectLiteralExpression(arg)) { + return; + } + + let eventName: string | undefined; + let propertiesNode: ts.Expression | undefined; + + for (const property of arg.properties) { + if (!ts.isPropertyAssignment(property)) { + if (ts.isSpreadAssignment(property) && !eventName) { + return; + } + continue; + } + const key = propertyName(property.name); + if (key === 'event') { + eventName = resolveEventName(property.initializer, catalog, config); + } + if (key === 'properties') { + propertiesNode = property.initializer; + } + } + + if (!eventName) { + return; + } + + const model = getOrCreateEvent(events, eventName); + if (propertiesNode) { + mergeBag(propertiesNode, eventName, filePath, constObjects, model); + } +} + +/** + * Merges a property bag into the event model. + * + * @param bag - Object literal or other expression. + * @param eventName - Event display name. + * @param filePath - Source path. + * @param constObjects - Same-file const objects. + * @param model - Event accumulator. + */ +function mergeBag( + bag: ts.Expression, + eventName: string, + filePath: string, + constObjects: Map, + model: EventModel, +): void { + if (!ts.isObjectLiteralExpression(bag)) { + model.unresolved.push({ + eventName, + key: '(non-literal properties)', + file: filePath, + }); + return; + } + + for (const property of bag.properties) { + if (ts.isSpreadAssignment(property)) { + model.unresolved.push({ + eventName, + key: '...', + file: filePath, + }); + continue; + } + + if (ts.isShorthandPropertyAssignment(property)) { + const snake = toSnakeCase(property.name.text); + if (IGNORED_GENERATE_OPT_PROPS.has(snake)) { + continue; + } + model.unresolved.push({ + eventName, + key: snake, + file: filePath, + }); + continue; + } + + if (!ts.isPropertyAssignment(property)) { + continue; + } + + const rawKey = propertyName(property.name); + if (!rawKey) { + continue; + } + const snake = toSnakeCase(rawKey); + if (IGNORED_GENERATE_OPT_PROPS.has(snake)) { + continue; + } + + const inferred = inferLiteralType(property.initializer, constObjects); + if (inferred) { + const existing = model.properties.get(snake); + if (!existing) { + model.properties.set(snake, inferred); + } + continue; + } + + model.unresolved.push({ + eventName, + key: snake, + file: filePath, + }); + } +} + +/** + * Infers a Segment YAML type from a literal (or same-file const) expression. + * + * @param expression - Property value. + * @param constObjects - Same-file const objects. + * @returns Type, or undefined when the value is not a literal. + */ +function inferLiteralType( + expression: ts.Expression, + constObjects: Map, +): PropertyType | undefined { + if ( + ts.isStringLiteral(expression) || + ts.isNoSubstitutionTemplateLiteral(expression) + ) { + return 'string'; + } + if (ts.isNumericLiteral(expression)) { + return 'number'; + } + if ( + expression.kind === ts.SyntaxKind.TrueKeyword || + expression.kind === ts.SyntaxKind.FalseKeyword + ) { + return 'boolean'; + } + if (ts.isArrayLiteralExpression(expression)) { + return 'array'; + } + if ( + ts.isPrefixUnaryExpression(expression) && + ts.isNumericLiteral(expression.operand) + ) { + return 'number'; + } + + if ( + ts.isPropertyAccessExpression(expression) && + ts.isIdentifier(expression.expression) + ) { + const objectName = expression.expression.text; + const object = constObjects.get(objectName); + return object?.get(expression.name.text); + } + + if (ts.isAsExpression(expression)) { + return inferLiteralType(expression.expression, constObjects); + } + + return undefined; +} + +/** + * Resolves a call argument to a catalog event display name. + * + * @param expression - First argument of createEventBuilder / event field. + * @param catalog - Enum catalog. + * @param config - Platform config. + * @returns Display name, if it is a documented enum ref. + */ +function resolveEventName( + expression: ts.Expression | undefined, + catalog: EventCatalog, + config: PlatformConfig, +): string | undefined { + if (!expression) { + return undefined; + } + + if (ts.isStringLiteral(expression)) { + return expression.text; + } + + if ( + ts.isPropertyAccessExpression(expression) && + ts.isIdentifier(expression.expression) + ) { + const objectName = expression.expression.text; + const member = expression.name.text; + if ( + objectName === config.enumName || + objectName === config.eventRefPrefix + ) { + return catalog.get(member); + } + } + + return undefined; +} + +/** + * Collects `.method()` calls chained off a call expression. + * + * @param start - Innermost call (createEventBuilder). + * @returns Method names and arguments in chain order. + */ +function chainedCalls( + start: ts.CallExpression, +): { name: string; args: readonly ts.Expression[] }[] { + const calls: { name: string; args: readonly ts.Expression[] }[] = []; + let current: ts.Node | undefined = start.parent; + + while (current) { + if ( + ts.isPropertyAccessExpression(current) && + ts.isCallExpression(current.parent) + ) { + calls.push({ + name: current.name.text, + args: current.parent.arguments, + }); + current = current.parent.parent; + continue; + } + break; + } + + return calls; +} + +/** + * Returns the identifier name of a called function. + * + * @param node - Call expression. + * @returns Function or method name. + */ +function getCalledName(node: ts.CallExpression): string | undefined { + if (ts.isIdentifier(node.expression)) { + return node.expression.text; + } + if (ts.isPropertyAccessExpression(node.expression)) { + return node.expression.name.text; + } + return undefined; +} + +/** + * Reads a property name from an identifier or string literal. + * + * @param name - Property name node. + * @returns Text, if static. + */ +function propertyName(name: ts.PropertyName): string | undefined { + if (ts.isIdentifier(name) || ts.isStringLiteral(name)) { + return name.text; + } + return undefined; +} + +/** + * Returns the accumulator for an event name. + * + * @param events - Event map. + * @param eventName - Display name. + * @returns Existing or new model. + */ +function getOrCreateEvent( + events: Map, + eventName: string, +): EventModel { + const existing = events.get(eventName); + if (existing) { + return existing; + } + const created: EventModel = { + properties: new Map(), + unresolved: [], + }; + events.set(eventName, created); + return created; +} + +/** + * Merges models from many files into one (catalog last-write-wins by key). + * + * @param models - Per-file models. + * @returns Combined model. + */ +export function mergeAnalyticsModels(models: AnalyticsModel[]): AnalyticsModel { + const catalog: EventCatalog = new Map(); + const events = new Map(); + + for (const model of models) { + for (const [key, name] of model.catalog) { + catalog.set(key, name); + } + for (const [eventName, event] of model.events) { + const target = getOrCreateEvent(events, eventName); + for (const [key, type] of event.properties) { + if (!target.properties.has(key)) { + target.properties.set(key, type); + } + } + target.unresolved.push(...event.unresolved); + } + } + + return { catalog, events }; +} diff --git a/src/segment-schema-draft-pr/apply-changes.test.ts b/src/segment-schema-draft-pr/apply-changes.test.ts new file mode 100644 index 00000000..9eedfc03 --- /dev/null +++ b/src/segment-schema-draft-pr/apply-changes.test.ts @@ -0,0 +1,191 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { applyChanges } from './apply-changes'; +import { getPlatformConfig } from './config'; +import { buildSchemaIndex } from './schema-index'; +import type { AnalyticsChangeSet } from './types'; + +const MOBILE = getPlatformConfig('mobile'); + +const EXISTING_EVENT = `name: App Opened +description: "TODO: fill description" +type: TRACK +version: 1 +labels: + library: metamask-mobile-perps +default_props: + - metamask-mobile-globals +properties: + existing_prop: + type: string + description: "TODO: fill description" + required: false +`; + +const GLOBALS = `properties: + anonymous: + type: boolean +`; + +const PLAN = `name: metamask-mobile +libraries: + - metamask-mobile-globals + - metamask-mobile-perps +`; + +/** + * Writes a nested file tree for schema fixtures. + * + * @param destDir - Directory to write into. + * @param files - Map of relative paths to file contents. + */ +async function writeTree( + destDir: string, + files: Record, +): Promise { + for (const [relative, contents] of Object.entries(files)) { + const absolute = path.join(destDir, relative); + await fs.mkdir(path.dirname(absolute), { recursive: true }); + await fs.writeFile(absolute, contents); + } +} + +describe('schema-index and apply-changes', () => { + let schemaDir: string; + + beforeEach(async () => { + schemaDir = await fs.mkdtemp(path.join(os.tmpdir(), 'schema-')); + await writeTree(schemaDir, { + 'libraries/events/metamask-mobile-perps/app-opened.yaml': EXISTING_EVENT, + 'libraries/properties/metamask-mobile-globals.yaml': GLOBALS, + 'tracking-plans/metamask-mobile.yaml': PLAN, + }); + }); + + afterEach(async () => { + await fs.rm(schemaDir, { recursive: true, force: true }); + }); + + it('indexes events by name and default_props keys', async () => { + const index = await buildSchemaIndex(schemaDir); + const event = index.eventsByName.get('App Opened'); + expect(event?.library).toBe('metamask-mobile-perps'); + expect(event?.propertyKeys.has('existing_prop')).toBe(true); + expect( + index.propertiesByLibrary + .get('metamask-mobile-globals') + ?.has('anonymous'), + ).toBe(true); + }); + + it('patches existing events in place and creates unsorted events', async () => { + const index = await buildSchemaIndex(schemaDir); + const changeset: AnalyticsChangeSet = { + eventsAdded: [{ enumKey: 'NEW_EVENT', eventName: 'New Event' }], + eventsRemoved: [{ enumKey: 'GONE', eventName: 'Gone' }], + eventsRenamed: [], + propertiesAdded: [ + { eventName: 'App Opened', key: 'usd_value', type: 'number' }, + { eventName: 'App Opened', key: 'anonymous', type: 'boolean' }, + { eventName: 'New Event', key: 'source', type: 'string' }, + ], + propertiesRemoved: [{ eventName: 'App Opened', key: 'old' }], + typeChanges: [ + { eventName: 'App Opened', key: 'existing_prop', type: 'number' }, + ], + unresolved: [{ eventName: 'New Event', key: 'helper', file: 'a.ts' }], + }; + + const result = await applyChanges( + schemaDir, + undefined, + MOBILE, + changeset, + index, + ); + expect(result.intendedFiles.map((file) => file.path)).toStrictEqual( + expect.arrayContaining([ + 'libraries/events/metamask-mobile-perps/app-opened.yaml', + 'libraries/events/metamask-mobile-unsorted/new-event.yaml', + ]), + ); + + const updated = await fs.readFile( + path.join( + schemaDir, + 'libraries/events/metamask-mobile-perps/app-opened.yaml', + ), + 'utf8', + ); + expect(updated).toContain('usd_value'); + expect(updated).toContain('required: false'); + expect(updated).toMatch(/existing_prop:[\s\S]*type: string/u); + expect(updated).not.toContain('anonymous:'); + + const created = await fs.readFile( + path.join( + schemaDir, + 'libraries/events/metamask-mobile-unsorted/new-event.yaml', + ), + 'utf8', + ); + expect(created).toContain('name: New Event'); + expect(created).toContain('source:'); + expect(created).not.toContain('helper:'); + + const plan = await fs.readFile( + path.join(schemaDir, 'tracking-plans/metamask-mobile.yaml'), + 'utf8', + ); + expect(plan).toContain('metamask-mobile-unsorted'); + }); + + it('preserves non-TODO descriptions and labels.kpi from the previous bot branch', async () => { + const previousDir = await fs.mkdtemp(path.join(os.tmpdir(), 'prev-')); + await writeTree(previousDir, { + 'libraries/events/metamask-mobile-unsorted/new-event.yaml': `name: New Event +description: Human written description +type: TRACK +version: 1 +labels: + library: metamask-mobile-unsorted + kpi: true +properties: + source: + type: string + description: Where the event came from + required: false +`, + }); + + const index = await buildSchemaIndex(schemaDir); + const changeset: AnalyticsChangeSet = { + eventsAdded: [{ enumKey: 'NEW_EVENT', eventName: 'New Event' }], + eventsRemoved: [], + eventsRenamed: [], + propertiesAdded: [ + { eventName: 'New Event', key: 'source', type: 'string' }, + { eventName: 'New Event', key: 'extra', type: 'boolean' }, + ], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], + }; + + await applyChanges(schemaDir, previousDir, MOBILE, changeset, index); + const written = await fs.readFile( + path.join( + schemaDir, + 'libraries/events/metamask-mobile-unsorted/new-event.yaml', + ), + 'utf8', + ); + expect(written).toContain('Human written description'); + expect(written).toContain('kpi: true'); + expect(written).toContain('Where the event came from'); + expect(written).toContain('extra:'); + await fs.rm(previousDir, { recursive: true, force: true }); + }); +}); diff --git a/src/segment-schema-draft-pr/apply-changes.ts b/src/segment-schema-draft-pr/apply-changes.ts new file mode 100644 index 00000000..936fb2aa --- /dev/null +++ b/src/segment-schema-draft-pr/apply-changes.ts @@ -0,0 +1,391 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { parseDocument, YAMLMap, Scalar } from 'yaml'; +import type { Document } from 'yaml'; + +import { TODO_DESCRIPTION } from './constants'; +import { toKebabSlug } from './names'; +import { defaultPropKeys, type SchemaIndex } from './schema-index'; +import type { + AnalyticsChangeSet, + IntendedFileChange, + PlatformConfig, + PropertyChange, + PropertyType, +} from './types'; + +export type ApplyResult = { + intendedFiles: IntendedFileChange[]; + wroteLibraryToPlan: boolean; +}; + +/** + * Writes additive YAML into the schema working tree. + * + * @param schemaDir - Schema checkout (usually `main`). + * @param previousDir - Previous bot-branch checkout, if any. + * @param config - Platform config. + * @param changeset - Diff to apply. + * @param index - Current schema index. + * @returns Paths that were created or updated. + */ +export async function applyChanges( + schemaDir: string, + previousDir: string | undefined, + config: PlatformConfig, + changeset: AnalyticsChangeSet, + index: SchemaIndex, +): Promise { + const intendedFiles: IntendedFileChange[] = []; + const addedEventNames = new Set( + changeset.eventsAdded.map((item) => item.eventName), + ); + + let wroteLibraryToPlan = false; + + for (const event of changeset.eventsAdded) { + const existing = index.eventsByName.get(event.eventName); + if (existing) { + continue; + } + + const relative = path.join( + 'libraries', + 'events', + config.defaultLibrary, + `${toKebabSlug(event.eventName)}.yaml`, + ); + const properties = changeset.propertiesAdded.filter( + (item) => item.eventName === event.eventName, + ); + const absolute = path.join(schemaDir, relative); + await fs.mkdir(path.dirname(absolute), { recursive: true }); + const doc = buildNewEventDocument(event.eventName, config, properties); + await preserveAndWrite(absolute, previousDir, relative, doc); + intendedFiles.push({ + path: relative, + kind: 'create', + eventName: event.eventName, + }); + wroteLibraryToPlan = true; + } + + const propertiesByEvent = groupByEvent(changeset.propertiesAdded); + for (const [eventName, properties] of propertiesByEvent) { + if (addedEventNames.has(eventName) && !index.eventsByName.get(eventName)) { + continue; + } + + const indexed = index.eventsByName.get(eventName); + if (!indexed) { + const relative = path.join( + 'libraries', + 'events', + config.defaultLibrary, + `${toKebabSlug(eventName)}.yaml`, + ); + const absolute = path.join(schemaDir, relative); + await fs.mkdir(path.dirname(absolute), { recursive: true }); + const doc = buildNewEventDocument(eventName, config, properties); + await preserveAndWrite(absolute, previousDir, relative, doc); + intendedFiles.push({ path: relative, kind: 'create', eventName }); + wroteLibraryToPlan = true; + continue; + } + + const supplied = defaultPropKeys(index, indexed.defaultProps); + const toAdd = properties.filter((item) => { + return !indexed.propertyKeys.has(item.key) && !supplied.has(item.key); + }); + if (toAdd.length === 0) { + continue; + } + + const text = await fs.readFile(indexed.filePath, 'utf8'); + const doc = parseDocument(text); + appendProperties(doc, toAdd); + const relative = path.relative(schemaDir, indexed.filePath); + await preserveAndWrite(indexed.filePath, previousDir, relative, doc); + intendedFiles.push({ path: relative, kind: 'update', eventName }); + } + + if (wroteLibraryToPlan) { + const attached = await attachLibraryToPlan(schemaDir, config); + wroteLibraryToPlan = attached; + } + + return { intendedFiles, wroteLibraryToPlan }; +} + +/** + * Builds a new unsorted-library event document. + * + * @param eventName - Display name. + * @param config - Platform config. + * @param properties - Additive properties. + * @returns YAML document. + */ +function buildNewEventDocument( + eventName: string, + config: PlatformConfig, + properties: PropertyChange[], +): Document { + const doc = parseDocument( + [ + `name: ${eventName}`, + `description: ${JSON.stringify(TODO_DESCRIPTION)}`, + 'type: TRACK', + 'version: 1', + 'labels:', + ` library: ${config.defaultLibrary}`, + 'default_props:', + ` - ${config.globals}`, + 'properties: {}', + ].join('\n'), + ); + appendProperties(doc, properties); + return doc; +} + +/** + * Appends properties with required: false and TODO descriptions. + * + * @param doc - Event YAML document. + * @param properties - Properties to add. + */ +function appendProperties(doc: Document, properties: PropertyChange[]): void { + const map = ensurePropertiesMap(doc); + for (const property of properties) { + if (map.has(property.key)) { + continue; + } + const prop = new YAMLMap(); + prop.set('type', yamlType(property.type)); + prop.set('description', quotedScalar(TODO_DESCRIPTION)); + prop.set('required', false); + map.set(property.key, prop); + } +} + +/** + * Quotes a scalar so values like `TODO: ...` stay valid YAML. + * + * @param value - String to quote. + * @returns Double-quoted YAML scalar. + */ +function quotedScalar(value: string): Scalar { + const scalar = new Scalar(value); + scalar.type = Scalar.QUOTE_DOUBLE; + return scalar; +} + +/** + * Ensures the event document has a YAML map at `properties`. + * + * @param doc - Event YAML document. + * @returns Properties map. + */ +function ensurePropertiesMap(doc: Document): YAMLMap { + const current = doc.get('properties'); + if (current instanceof YAMLMap) { + current.flow = false; + return current; + } + const map = new YAMLMap(); + map.flow = false; + doc.set('properties', map); + return map; +} + +/** + * Maps inferred types onto Segment YAML type names. + * + * @param type - Syntactic type. + * @returns YAML `type` value. + */ +function yamlType(type: PropertyType): string { + return type; +} + +/** + * Copies human-edited descriptions and labels.kpi from the previous bot branch. + * + * @param absolute - File to write in the main checkout. + * @param previousDir - Previous bot-branch root. + * @param relative - Path from schema root. + * @param doc - Document about to be written. + */ +async function preserveAndWrite( + absolute: string, + previousDir: string | undefined, + relative: string, + doc: Document, +): Promise { + if (previousDir) { + const previousPath = path.join(previousDir, relative); + try { + const previousText = await fs.readFile(previousPath, 'utf8'); + const previous = parseDocument(previousText); + overlayPreservedFields(doc, previous); + } catch (error) { + const { code } = error as { code?: string }; + if (code !== 'ENOENT') { + throw error; + } + } + } + + await fs.writeFile(absolute, String(doc)); +} + +/** + * Keeps non-TODO descriptions and labels.kpi from a previous version of the file. + * + * @param next - Newly generated document. + * @param previous - Document from the last bot branch. + */ +function overlayPreservedFields(next: Document, previous: Document): void { + const previousDescription = previous.get('description'); + if (isNonTodoDescription(previousDescription)) { + next.set('description', previousDescription); + } + + const previousKpi = previous.getIn(['labels', 'kpi']); + if (previousKpi !== undefined && previousKpi !== null) { + next.setIn(['labels', 'kpi'], previousKpi); + } + + const previousProperties = previous.get('properties'); + const nextProperties = next.get('properties'); + if (!isYamlMap(previousProperties) || !isYamlMap(nextProperties)) { + return; + } + + for (const item of previousProperties.items) { + const key = yamlKey(item.key); + if (!key || next.getIn(['properties', key]) === undefined) { + continue; + } + const previousProp = item.value; + if (!isYamlMap(previousProp)) { + continue; + } + const description = previousProp.get('description'); + if (isNonTodoDescription(description)) { + next.setIn(['properties', key, 'description'], description); + } + } +} + +/** + * Appends the unsorted library id to the platform tracking plan if missing. + * + * @param schemaDir - Schema root. + * @param config - Platform config. + * @returns Whether the plan file was updated. + */ +async function attachLibraryToPlan( + schemaDir: string, + config: PlatformConfig, +): Promise { + const planPath = path.join(schemaDir, config.trackingPlan); + const text = await fs.readFile(planPath, 'utf8'); + const doc = parseDocument(text); + const libraries = doc.get('libraries'); + if (!isYamlSeq(libraries)) { + return false; + } + + const existing = libraries.items.map((item) => yamlKey(item)); + if (existing.includes(config.defaultLibrary)) { + return false; + } + + libraries.add(config.defaultLibrary); + await fs.writeFile(planPath, String(doc)); + return true; +} + +/** + * Groups property changes by event name. + * + * @param properties - Flat list. + * @returns Map of event name to properties. + */ +function groupByEvent( + properties: PropertyChange[], +): Map { + const grouped = new Map(); + for (const property of properties) { + const list = grouped.get(property.eventName) ?? []; + list.push(property); + grouped.set(property.eventName, list); + } + return grouped; +} + +/** + * True when a description is a human-written string, not the TODO placeholder. + * + * @param value - YAML node. + * @returns Whether to preserve it. + */ +function isNonTodoDescription(value: unknown): boolean { + const text = yamlKey(value); + return Boolean(text && text !== TODO_DESCRIPTION && !text.startsWith('TODO')); +} + +/** + * Reads a YAML scalar or node as a string. + * + * @param value - YAML node. + * @returns String text. + */ +function yamlKey(value: unknown): string | undefined { + if (typeof value === 'string') { + return value; + } + if (value && typeof value === 'object' && 'value' in value) { + const inner = (value as { value: unknown }).value; + if (typeof inner === 'string') { + return inner; + } + } + return undefined; +} + +/** + * Narrows a YAML map. + * + * @param value - Parsed node. + * @returns Whether it is a map with items. + */ +function isYamlMap(value: unknown): value is { + items: { key: unknown; value: unknown }[]; + get: (key: string) => unknown; +} { + return ( + typeof value === 'object' && + value !== null && + 'items' in value && + Array.isArray((value as { items: unknown }).items) + ); +} + +/** + * Narrows a YAML sequence. + * + * @param value - Parsed node. + * @returns Whether it is a sequence with add(). + */ +function isYamlSeq( + value: unknown, +): value is { items: unknown[]; add: (item: string) => void } { + return ( + typeof value === 'object' && + value !== null && + 'items' in value && + 'add' in value && + typeof (value as { add: unknown }).add === 'function' + ); +} diff --git a/src/segment-schema-draft-pr/cli.ts b/src/segment-schema-draft-pr/cli.ts new file mode 100644 index 00000000..ae8ab9fc --- /dev/null +++ b/src/segment-schema-draft-pr/cli.ts @@ -0,0 +1,71 @@ +import { botBranchName } from './config'; +import { changesetHasWritableChanges } from './diff-models'; +import { generateSchemaDraft, writeSummaryFile } from './generate'; +import { createOctokitClients, parseRepo } from './github-api'; +import { writeGithubOutput } from './github-output'; +import { parseCliArgs, type CliArgs } from './parse-args'; +import { runPublish } from './publish'; +import { getProcessEnv } from '../env-utils'; + +main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = 1; +}); + +/** + * CLI entry for generate / publish phases. + */ +async function main(): Promise { + const args = parseCliArgs(process.argv.slice(2), getProcessEnv()); + + if (args.phase === 'generate') { + await runGenerate(args); + return; + } + if (args.phase === 'publish') { + await runPublish(args); + } +} + +/** + * Writes YAML into the schema working tree from the client PR file list. + * + * @param args - CLI args. + */ +async function runGenerate(args: CliArgs): Promise { + const previousDir = + args.previousDir && args.previousDir !== 'true' + ? args.previousDir + : undefined; + const clientRepo = parseRepo(args.clientRepository); + const { client } = createOctokitClients( + args.githubToken, + args.segmentSchemaToken, + ); + + const summary = await generateSchemaDraft({ + octokit: client, + clientRepo, + prNumber: args.prNumber, + schemaDir: args.schema, + previousDir, + platform: args.platform, + baseSha: args.baseSha, + headSha: args.headSha, + branch: botBranchName(args.platform, args.prNumber), + defaultLibrary: args.defaultLibrary, + }); + + const summaryFile = await writeSummaryFile(args.schema, summary); + + writeGithubOutput(args.githubOutput, [ + ['has_changes', summary.hasChanges ? 'true' : 'false'], + [ + 'has_writable_changes', + changesetHasWritableChanges(summary.changeset) ? 'true' : 'false', + ], + ['branch', summary.branch], + ['summary_file', summaryFile], + ]); +} diff --git a/src/segment-schema-draft-pr/client-diff.test.ts b/src/segment-schema-draft-pr/client-diff.test.ts new file mode 100644 index 00000000..a2b868c3 --- /dev/null +++ b/src/segment-schema-draft-pr/client-diff.test.ts @@ -0,0 +1,143 @@ +import type { Octokit } from '@octokit/rest'; + +import { + getFileAtRef, + hasAnalyticsDiff, + loadPullRequestTsDiff, +} from './client-diff'; + +const PULL_NUMBER = 'pull_number'; +const PER_PAGE = 'per_page'; +const PREVIOUS_FILENAME = 'previous_filename'; + +const REPO = { owner: 'MetaMask', repo: 'metamask-mobile' }; + +describe('loadPullRequestTsDiff', () => { + it('keeps non-test TypeScript paths from the PR file list', async () => { + const paginate = jest.fn().mockResolvedValue([ + { + filename: 'app/Home.ts', + patch: '+createEventBuilder(MetaMetricsEvents.APP_OPENED)', + }, + { filename: 'README.md', patch: '+docs' }, + { filename: 'app/foo.test.ts', patch: '+createEventBuilder(x)' }, + { filename: 'app/__tests__/bar.ts', patch: '+trackEvent' }, + ]); + const octokit = { paginate } as unknown as Octokit; + + const diff = await loadPullRequestTsDiff(octokit, REPO, 12); + + expect(diff.files).toStrictEqual(['app/Home.ts']); + expect(diff.mentionsAnalytics).toBe(true); + expect(paginate).toHaveBeenCalledWith( + 'GET /repos/{owner}/{repo}/pulls/{pull_number}/files', + { + owner: 'MetaMask', + repo: 'metamask-mobile', + [PULL_NUMBER]: 12, + [PER_PAGE]: 100, + }, + ); + }); + + it('includes previous_filename so a rename is visible at both SHAs', async () => { + const paginate = jest.fn().mockResolvedValue([ + { + filename: 'app/track.tsx', + [PREVIOUS_FILENAME]: 'app/old-track.tsx', + patch: '+addProperties({ source: "banner" })', + }, + ]); + const octokit = { paginate } as unknown as Octokit; + + const diff = await loadPullRequestTsDiff(octokit, REPO, 12); + + expect(diff.files).toStrictEqual(['app/track.tsx', 'app/old-track.tsx']); + expect(diff.mentionsAnalytics).toBe(true); + }); + + it('treats an omitted patch on a TypeScript file as an analytics hit', async () => { + const paginate = jest.fn().mockResolvedValue([{ filename: 'app/Home.ts' }]); + const octokit = { paginate } as unknown as Octokit; + + const diff = await loadPullRequestTsDiff(octokit, REPO, 12); + + expect(diff.files).toStrictEqual(['app/Home.ts']); + expect(diff.mentionsAnalytics).toBe(true); + }); + + it('does not treat unrelated TypeScript patches as analytics hits', async () => { + const paginate = jest + .fn() + .mockResolvedValue([ + { filename: 'app/Home.ts', patch: '+const x = 1;\n' }, + ]); + const octokit = { paginate } as unknown as Octokit; + + const diff = await loadPullRequestTsDiff(octokit, REPO, 12); + + expect(diff.mentionsAnalytics).toBe(false); + }); +}); + +describe('hasAnalyticsDiff', () => { + const catalog = 'app/core/Analytics/MetaMetrics.events.ts'; + const config = { + catalogFile: catalog, + enumName: 'EVENT_NAME', + eventRefPrefix: 'MetaMetricsEvents', + trackingPlan: 'tracking-plans/metamask-mobile.yaml', + globals: 'metamask-mobile-globals', + defaultLibrary: 'metamask-mobile-unsorted', + }; + + it('is true when the catalog file changed even without a prefilter hit', () => { + expect( + hasAnalyticsDiff({ files: [catalog], mentionsAnalytics: false }, config), + ).toBe(true); + }); + + it('is false when neither the catalog nor analytics APIs appear', () => { + expect( + hasAnalyticsDiff( + { files: ['app/Home.ts'], mentionsAnalytics: false }, + config, + ), + ).toBe(false); + }); +}); + +describe('getFileAtRef', () => { + it('decodes base64 file contents', async () => { + const getContent = jest.fn().mockResolvedValue({ + data: { + content: Buffer.from('enum EVENT_NAME {}', 'utf8').toString('base64'), + }, + }); + const octokit = { repos: { getContent } } as unknown as Octokit; + + expect( + await getFileAtRef( + octokit, + REPO, + 'app/core/Analytics/MetaMetrics.events.ts', + 'abc', + ), + ).toBe('enum EVENT_NAME {}'); + expect(getContent).toHaveBeenCalledWith({ + owner: 'MetaMask', + repo: 'metamask-mobile', + path: 'app/core/Analytics/MetaMetrics.events.ts', + ref: 'abc', + }); + }); + + it('returns null on 404', async () => { + const error = new Error('Not Found') as Error & { status: number }; + error.status = 404; + const getContent = jest.fn().mockRejectedValue(error); + const octokit = { repos: { getContent } } as unknown as Octokit; + + expect(await getFileAtRef(octokit, REPO, 'app/gone.ts', 'abc')).toBeNull(); + }); +}); diff --git a/src/segment-schema-draft-pr/client-diff.ts b/src/segment-schema-draft-pr/client-diff.ts new file mode 100644 index 00000000..84fc32f0 --- /dev/null +++ b/src/segment-schema-draft-pr/client-diff.ts @@ -0,0 +1,158 @@ +import type { Octokit } from '@octokit/rest'; + +import { DIFF_PREFILTER } from './constants'; +import type { RepoId } from './github-api'; +import type { PlatformConfig } from './types'; + +const PULL_NUMBER = 'pull_number'; +const PER_PAGE = 'per_page'; +const PREVIOUS_FILENAME = 'previous_filename'; + +const TEST_PATH = + /(?:^|\/)(?:__tests__\/|(?:[^/]+\.)?(?:test|spec)\.[jt]sx?$)/u; + +export type PullRequestTsDiff = { + files: string[]; + mentionsAnalytics: boolean; +}; + +type PullsFile = { + filename: string; + patch?: string | null; + [PREVIOUS_FILENAME]?: string; +}; + +/** + * Lists non-test TypeScript paths on a pull request and whether their patches + * mention analytics APIs. + * + * Why: `pulls.listFiles` is GitHub's three-dot PR file list (merge-base → head), + * so a branch that lagged the base tip does not look like it reverted unrelated + * analytics landings. + * + * @param octokit - Client-repo Octokit. + * @param repoId - Mobile or Extension repo. + * @param prNumber - Pull request number. + * @returns Changed TS paths and the analytics pre-filter result. + */ +export async function loadPullRequestTsDiff( + octokit: Octokit, + repoId: RepoId, + prNumber: number, +): Promise { + const items = await octokit.paginate( + 'GET /repos/{owner}/{repo}/pulls/{pull_number}/files', + { + owner: repoId.owner, + repo: repoId.repo, + [PULL_NUMBER]: prNumber, + [PER_PAGE]: 100, + }, + ); + + const files = new Set(); + let mentionsAnalytics = false; + + for (const item of items) { + const candidates = [item.filename]; + const renamedFrom = item[PREVIOUS_FILENAME]; + if (renamedFrom) { + candidates.push(renamedFrom); + } + + const tsPaths = candidates.filter(isNonTestTsFile); + if (tsPaths.length === 0) { + continue; + } + + for (const filePath of tsPaths) { + files.add(filePath); + } + + if (item.patch === undefined || item.patch === null) { + mentionsAnalytics = true; + continue; + } + if (DIFF_PREFILTER.test(item.patch)) { + mentionsAnalytics = true; + } + } + + return { files: [...files], mentionsAnalytics }; +} + +/** + * True when the PR file list looks like an analytics change. + * + * @param diff - Changed TS paths and pre-filter result. + * @param config - Platform config (catalog path). + * @returns Whether generate should walk the files. + */ +export function hasAnalyticsDiff( + diff: PullRequestTsDiff, + config: PlatformConfig, +): boolean { + return diff.mentionsAnalytics || diff.files.includes(config.catalogFile); +} + +/** + * Returns file contents at a git ref, or null when the path is missing. + * + * @param octokit - Client-repo Octokit. + * @param repoId - Mobile or Extension repo. + * @param filePath - Path relative to the repository root. + * @param sha - Commit SHA. + * @returns File text, or null on 404 / non-file. + */ +export async function getFileAtRef( + octokit: Octokit, + repoId: RepoId, + filePath: string, + sha: string, +): Promise { + try { + const { data } = await octokit.repos.getContent({ + owner: repoId.owner, + repo: repoId.repo, + path: filePath, + ref: sha, + }); + if (Array.isArray(data) || !('content' in data) || !data.content) { + return null; + } + return Buffer.from(data.content, 'base64').toString('utf8'); + } catch (error: unknown) { + if (isNotFoundError(error)) { + return null; + } + throw error; + } +} + +/** + * True for production TypeScript paths the analytics extractor walks. + * + * @param filePath - Path relative to the repository root. + * @returns Whether the path is a non-test `.ts` / `.tsx` file. + */ +function isNonTestTsFile(filePath: string): boolean { + return ( + (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) && + !TEST_PATH.test(filePath) + ); +} + +/** + * True when an Octokit error is a missing object. + * + * @param error - Thrown value. + * @returns Whether the error is HTTP 404. + */ +function isNotFoundError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'status' in error && + error.status === 404 + ); +} diff --git a/src/segment-schema-draft-pr/config.test.ts b/src/segment-schema-draft-pr/config.test.ts new file mode 100644 index 00000000..7a92a2bb --- /dev/null +++ b/src/segment-schema-draft-pr/config.test.ts @@ -0,0 +1,29 @@ +import { getPlatformConfig, botBranchName } from './config'; + +describe('config', () => { + it('returns mobile catalog and unsorted library paths', () => { + expect(getPlatformConfig('mobile')).toStrictEqual({ + catalogFile: 'app/core/Analytics/MetaMetrics.events.ts', + enumName: 'EVENT_NAME', + eventRefPrefix: 'MetaMetricsEvents', + trackingPlan: 'tracking-plans/metamask-mobile.yaml', + globals: 'metamask-mobile-globals', + defaultLibrary: 'metamask-mobile-unsorted', + }); + }); + + it('returns extension catalog and unsorted library paths', () => { + expect(getPlatformConfig('extension')).toStrictEqual({ + catalogFile: 'shared/constants/metametrics.ts', + enumName: 'MetaMetricsEventName', + eventRefPrefix: 'MetaMetricsEventName', + trackingPlan: 'tracking-plans/metamask-extension.yaml', + globals: 'metamask-extension-globals', + defaultLibrary: 'metamask-extension-unsorted', + }); + }); + + it('builds the deterministic bot branch name', () => { + expect(botBranchName('mobile', 42)).toBe('metamaskbot/mobile-pr-42'); + }); +}); diff --git a/src/segment-schema-draft-pr/config.ts b/src/segment-schema-draft-pr/config.ts new file mode 100644 index 00000000..0071b472 --- /dev/null +++ b/src/segment-schema-draft-pr/config.ts @@ -0,0 +1,40 @@ +import type { Platform, PlatformConfig } from './types'; + +const MOBILE_CONFIG: PlatformConfig = { + catalogFile: 'app/core/Analytics/MetaMetrics.events.ts', + enumName: 'EVENT_NAME', + eventRefPrefix: 'MetaMetricsEvents', + trackingPlan: 'tracking-plans/metamask-mobile.yaml', + globals: 'metamask-mobile-globals', + defaultLibrary: 'metamask-mobile-unsorted', +}; + +const EXTENSION_CONFIG: PlatformConfig = { + catalogFile: 'shared/constants/metametrics.ts', + enumName: 'MetaMetricsEventName', + eventRefPrefix: 'MetaMetricsEventName', + trackingPlan: 'tracking-plans/metamask-extension.yaml', + globals: 'metamask-extension-globals', + defaultLibrary: 'metamask-extension-unsorted', +}; + +/** + * Returns the per-platform catalog and schema paths. + * + * @param platform - Mobile or extension. + * @returns Catalog file, enum, tracking plan, and unsorted library ids. + */ +export function getPlatformConfig(platform: Platform): PlatformConfig { + return platform === 'mobile' ? MOBILE_CONFIG : EXTENSION_CONFIG; +} + +/** + * Builds the deterministic bot branch for one client PR. + * + * @param platform - Mobile or extension. + * @param prNumber - Client pull request number. + * @returns Branch name `metamaskbot/-pr-`. + */ +export function botBranchName(platform: Platform, prNumber: number): string { + return `metamaskbot/${platform}-pr-${prNumber}`; +} diff --git a/src/segment-schema-draft-pr/constants.ts b/src/segment-schema-draft-pr/constants.ts new file mode 100644 index 00000000..3dcc2600 --- /dev/null +++ b/src/segment-schema-draft-pr/constants.ts @@ -0,0 +1,18 @@ +export const AGREEMENT_PHRASE = 'I agree to open a draft Segment schema PR'; + +export const TODO_DESCRIPTION = 'TODO: fill description'; + +export const OPT_OUT_LABEL = 'no-schema-pr'; + +export const PROPOSAL_MARKER = ''; + +export const BODY_START_MARKER = ''; + +export const BODY_END_MARKER = ''; + +export const DIFF_PREFILTER = + /EVENT_NAME|MetaMetricsEventName|trackEvent|addProperties|createEventBuilder/u; + +export const MAX_CHANGED_TS_FILES = 150; + +export const IGNORED_GENERATE_OPT_PROPS = new Set(['action', 'name']); diff --git a/src/segment-schema-draft-pr/diff-models.test.ts b/src/segment-schema-draft-pr/diff-models.test.ts new file mode 100644 index 00000000..aee4a43a --- /dev/null +++ b/src/segment-schema-draft-pr/diff-models.test.ts @@ -0,0 +1,112 @@ +import { + changesetHasContent, + changesetHasWritableChanges, + diffModels, +} from './diff-models'; +import type { AnalyticsModel } from './types'; + +/** + * Builds a partial analytics model for diff tests. + * + * @param partial - Catalog and/or events to include. + * @returns A complete analytics model. + */ +function model(partial: Partial): AnalyticsModel { + return { + catalog: partial.catalog ?? new Map(), + events: partial.events ?? new Map(), + }; +} + +describe('diffModels', () => { + it('detects added, removed, and renamed catalog events', () => { + const changeset = diffModels( + model({ + catalog: new Map([ + ['KEEP', 'Keep'], + ['GONE', 'Gone'], + ['RENAME', 'Old Name'], + ]), + }), + model({ + catalog: new Map([ + ['KEEP', 'Keep'], + ['NEW', 'New Event'], + ['RENAME', 'New Name'], + ]), + }), + ); + + expect(changeset.eventsAdded).toStrictEqual([ + { enumKey: 'NEW', eventName: 'New Event' }, + ]); + expect(changeset.eventsRemoved).toStrictEqual([ + { enumKey: 'GONE', eventName: 'Gone' }, + ]); + expect(changeset.eventsRenamed).toStrictEqual([ + { enumKey: 'RENAME', fromName: 'Old Name', toName: 'New Name' }, + ]); + }); + + it('detects added, removed, and type-changed properties', () => { + const changeset = diffModels( + model({ + events: new Map([ + [ + 'App Opened', + { + properties: new Map([ + ['keep', 'string' as const], + ['gone', 'string' as const], + ['count', 'string' as const], + ]), + unresolved: [], + }, + ], + ]), + }), + model({ + events: new Map([ + [ + 'App Opened', + { + properties: new Map([ + ['keep', 'string' as const], + ['usd_value', 'number' as const], + ['count', 'number' as const], + ]), + unresolved: [ + { eventName: 'App Opened', key: 'helper', file: 'a.ts' }, + ], + }, + ], + ]), + }), + ); + + expect(changeset.propertiesAdded).toStrictEqual([ + { eventName: 'App Opened', key: 'usd_value', type: 'number' }, + ]); + expect(changeset.propertiesRemoved).toStrictEqual([ + { eventName: 'App Opened', key: 'gone' }, + ]); + expect(changeset.typeChanges).toStrictEqual([ + { eventName: 'App Opened', key: 'count', type: 'number' }, + ]); + expect(changeset.unresolved).toHaveLength(1); + }); + + it('treats listed-only changes as content but not writable', () => { + const listedOnly = { + eventsAdded: [], + eventsRemoved: [{ enumKey: 'GONE', eventName: 'Gone' }], + eventsRenamed: [], + propertiesAdded: [], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], + }; + expect(changesetHasContent(listedOnly)).toBe(true); + expect(changesetHasWritableChanges(listedOnly)).toBe(false); + }); +}); diff --git a/src/segment-schema-draft-pr/diff-models.ts b/src/segment-schema-draft-pr/diff-models.ts new file mode 100644 index 00000000..a5699110 --- /dev/null +++ b/src/segment-schema-draft-pr/diff-models.ts @@ -0,0 +1,122 @@ +import type { + AnalyticsChangeSet, + AnalyticsModel, + EventChange, + PropertyChange, + RemovedProperty, + RenameChange, +} from './types'; + +/** + * Diffs catalog + call-site models between PR base and head. + * + * @param base - Model at the base SHA. + * @param head - Model at the head SHA. + * @returns Additive changes plus listed-only removals/renames/unresolved. + */ +export function diffModels( + base: AnalyticsModel, + head: AnalyticsModel, +): AnalyticsChangeSet { + const eventsAdded: EventChange[] = []; + const eventsRemoved: EventChange[] = []; + const eventsRenamed: RenameChange[] = []; + + for (const [enumKey, eventName] of head.catalog) { + const baseName = base.catalog.get(enumKey); + if (baseName === undefined) { + eventsAdded.push({ enumKey, eventName }); + continue; + } + if (baseName !== eventName) { + eventsRenamed.push({ enumKey, fromName: baseName, toName: eventName }); + } + } + + for (const [enumKey, eventName] of base.catalog) { + if (!head.catalog.has(enumKey)) { + eventsRemoved.push({ enumKey, eventName }); + } + } + + const renamedFrom = new Set(eventsRenamed.map((item) => item.fromName)); + const renamedTo = new Set(eventsRenamed.map((item) => item.toName)); + + const propertiesAdded: PropertyChange[] = []; + const propertiesRemoved: RemovedProperty[] = []; + const typeChanges: PropertyChange[] = []; + + const eventNames = new Set([...base.events.keys(), ...head.events.keys()]); + + for (const eventName of eventNames) { + if (renamedFrom.has(eventName) || renamedTo.has(eventName)) { + continue; + } + + const baseProps = base.events.get(eventName)?.properties ?? new Map(); + const headProps = head.events.get(eventName)?.properties ?? new Map(); + + for (const [key, type] of headProps) { + const previous = baseProps.get(key); + if (previous === undefined) { + propertiesAdded.push({ eventName, key, type }); + continue; + } + if (previous !== type) { + typeChanges.push({ eventName, key, type }); + } + } + + for (const key of baseProps.keys()) { + if (!headProps.has(key)) { + propertiesRemoved.push({ eventName, key }); + } + } + } + + const unresolved = [...head.events.values()].flatMap( + (event) => event.unresolved, + ); + + return { + eventsAdded, + eventsRemoved, + eventsRenamed, + propertiesAdded, + propertiesRemoved, + typeChanges, + unresolved, + }; +} + +/** + * True when the changeset has anything to show on the proposal or schema PR. + * + * @param changeset - Diff result. + * @returns Whether a proposal (or schema write) is warranted. + */ +export function changesetHasContent(changeset: AnalyticsChangeSet): boolean { + return ( + changeset.eventsAdded.length > 0 || + changeset.eventsRemoved.length > 0 || + changeset.eventsRenamed.length > 0 || + changeset.propertiesAdded.length > 0 || + changeset.propertiesRemoved.length > 0 || + changeset.typeChanges.length > 0 || + changeset.unresolved.length > 0 + ); +} + +/** + * True when YAML should be written (additive events/properties only). + * + * @param changeset - Diff result. + * @returns Whether apply-changes will mutate schema files. + */ +export function changesetHasWritableChanges( + changeset: AnalyticsChangeSet, +): boolean { + return ( + changeset.eventsAdded.length > 0 || changeset.propertiesAdded.length > 0 + ); +} diff --git a/src/segment-schema-draft-pr/generate.test.ts b/src/segment-schema-draft-pr/generate.test.ts new file mode 100644 index 00000000..85fcc674 --- /dev/null +++ b/src/segment-schema-draft-pr/generate.test.ts @@ -0,0 +1,131 @@ +import type { Octokit } from '@octokit/rest'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { generateSchemaDraft } from './generate'; + +const CATALOG = 'app/core/Analytics/MetaMetrics.events.ts'; +const HOME = 'app/Home.ts'; + +const BASE_CATALOG = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + } + `; + +const HEAD_CATALOG = ` + enum EVENT_NAME { + APP_OPENED = 'App Opened', + NEW_EVENT = 'New Event', + } + `; + +const BASE_HOME = ` + createEventBuilder(MetaMetricsEvents.APP_OPENED).addProperties({ location: 'Home' }); + `; + +const HEAD_HOME = ` + createEventBuilder(MetaMetricsEvents.APP_OPENED).addProperties({ location: 'Home' }); + createEventBuilder(MetaMetricsEvents.NEW_EVENT).addProperties({ source: 'banner' }); + `; + +/** + * Encodes a GitHub contents API file payload. + * + * @param text - File text. + * @returns Octokit getContent shape. + */ +function encodedFile(text: string): { data: { content: string } } { + return { data: { content: Buffer.from(text, 'utf8').toString('base64') } }; +} + +/** + * Builds an Octokit mock for listFiles + getContent. + * + * @param contents - SHA → path → text. + * @returns Mock Octokit. + */ +function mockClientOctokit( + contents: Record>, +): Octokit { + const paginate = jest.fn().mockResolvedValue([ + { + filename: CATALOG, + patch: "+NEW_EVENT = 'New Event'", + }, + { + filename: HOME, + patch: '+createEventBuilder(MetaMetricsEvents.NEW_EVENT)', + }, + ]); + const getContent = jest + .fn() + .mockImplementation( + async ({ path: filePath, ref }: { path: string; ref: string }) => { + const text = contents[ref]?.[filePath]; + if (text === undefined) { + const error: Error & { status: number } = Object.assign( + new Error('Not Found'), + { status: 404 }, + ); + return Promise.reject(error); + } + return Promise.resolve(encodedFile(text)); + }, + ); + + return { paginate, repos: { getContent } } as unknown as Octokit; +} + +describe('generateSchemaDraft', () => { + it('writes unsorted YAML for a new catalog event and call-site property', async () => { + const schemaDir = await fs.mkdtemp(path.join(os.tmpdir(), 'schema-')); + await fs.mkdir(path.join(schemaDir, 'libraries/properties'), { + recursive: true, + }); + await fs.mkdir(path.join(schemaDir, 'tracking-plans'), { recursive: true }); + await fs.writeFile( + path.join(schemaDir, 'libraries/properties/metamask-mobile-globals.yaml'), + 'properties:\n anonymous:\n type: boolean\n', + ); + await fs.writeFile( + path.join(schemaDir, 'tracking-plans/metamask-mobile.yaml'), + 'name: metamask-mobile\nlibraries:\n - metamask-mobile-globals\n', + ); + + const octokit = mockClientOctokit({ + base: { [CATALOG]: BASE_CATALOG, [HOME]: BASE_HOME }, + head: { [CATALOG]: HEAD_CATALOG, [HOME]: HEAD_HOME }, + }); + + const summary = await generateSchemaDraft({ + octokit, + clientRepo: { owner: 'MetaMask', repo: 'metamask-mobile' }, + prNumber: 1, + schemaDir, + previousDir: undefined, + platform: 'mobile', + baseSha: 'base', + headSha: 'head', + branch: 'metamaskbot/mobile-pr-1', + defaultLibrary: undefined, + }); + + expect(summary.hasChanges).toBe(true); + expect(summary.changeset.eventsAdded).toStrictEqual([ + { enumKey: 'NEW_EVENT', eventName: 'New Event' }, + ]); + const yaml = await fs.readFile( + path.join( + schemaDir, + 'libraries/events/metamask-mobile-unsorted/new-event.yaml', + ), + 'utf8', + ); + expect(yaml).toContain('name: New Event'); + expect(yaml).toContain('source:'); + + await fs.rm(schemaDir, { recursive: true, force: true }); + }); +}); diff --git a/src/segment-schema-draft-pr/generate.ts b/src/segment-schema-draft-pr/generate.ts new file mode 100644 index 00000000..d077e63c --- /dev/null +++ b/src/segment-schema-draft-pr/generate.ts @@ -0,0 +1,242 @@ +import type { Octokit } from '@octokit/rest'; +import fs from 'fs/promises'; +import path from 'path'; +import ts from 'typescript'; + +import { + extractAnalyticsModel, + extractEnumCatalog, + mergeAnalyticsModels, +} from './analytics-model'; +import { applyChanges } from './apply-changes'; +import { + getFileAtRef, + hasAnalyticsDiff, + loadPullRequestTsDiff, +} from './client-diff'; +import { getPlatformConfig } from './config'; +import { + changesetHasContent, + changesetHasWritableChanges, + diffModels, +} from './diff-models'; +import type { RepoId } from './github-api'; +import { buildSchemaIndex } from './schema-index'; +import type { + AnalyticsModel, + GenerateSummary, + Platform, + PlatformConfig, +} from './types'; + +/** + * Generates schema YAML in the workspace from a client PR diff. + * + * @param params - Octokit client, SHAs, and platform. + * @param params.octokit - Client-repo Octokit. + * @param params.clientRepo - Mobile or Extension repo. + * @param params.prNumber - Client pull request number. + * @param params.schemaDir - Schema working tree to write. + * @param params.previousDir - Previous bot-branch checkout, if any. + * @param params.platform - Mobile or extension. + * @param params.baseSha - Pull request base SHA. + * @param params.headSha - Pull request head SHA. + * @param params.branch - Bot branch name. + * @param params.defaultLibrary - Optional unsorted library override. + * @returns Summary for GITHUB_OUTPUT and publish. + */ +export async function generateSchemaDraft(params: { + octokit: Octokit; + clientRepo: RepoId; + prNumber: number; + schemaDir: string; + previousDir: string | undefined; + platform: Platform; + baseSha: string; + headSha: string; + branch: string; + defaultLibrary: string | undefined; +}): Promise { + const config = { ...getPlatformConfig(params.platform) }; + if (params.defaultLibrary) { + config.defaultLibrary = params.defaultLibrary; + } + const { files: changed, mentionsAnalytics } = await loadPullRequestTsDiff( + params.octokit, + params.clientRepo, + params.prNumber, + ); + + if (!hasAnalyticsDiff({ files: changed, mentionsAnalytics }, config)) { + return emptySummary(params.branch); + } + + const files = new Set(changed); + files.add(config.catalogFile); + + const baseModel = await modelAtSha( + params.octokit, + params.clientRepo, + params.baseSha, + [...files], + config, + ); + const headModel = await modelAtSha( + params.octokit, + params.clientRepo, + params.headSha, + [...files], + config, + ); + + const changeset = diffModels(baseModel, headModel); + const hasChanges = changesetHasContent(changeset); + + if (!hasChanges || !changesetHasWritableChanges(changeset)) { + return { + hasChanges, + branch: params.branch, + changeset, + intendedFiles: [], + schemaPrNumber: null, + schemaPrUrl: null, + }; + } + + const index = await buildSchemaIndex(params.schemaDir); + const applied = await applyChanges( + params.schemaDir, + params.previousDir, + config, + changeset, + index, + ); + + return { + hasChanges: true, + branch: params.branch, + changeset, + intendedFiles: applied.intendedFiles, + schemaPrNumber: null, + schemaPrUrl: null, + }; +} + +/** + * Builds an analytics model from file contents at one SHA. + * + * @param octokit - Client-repo Octokit. + * @param clientRepo - Mobile or Extension repo. + * @param sha - Git SHA. + * @param files - Paths to extract. + * @param config - Platform config. + * @returns Merged model. + */ +async function modelAtSha( + octokit: Octokit, + clientRepo: RepoId, + sha: string, + files: string[], + config: PlatformConfig, +): Promise { + const catalogText = await getFileAtRef( + octokit, + clientRepo, + config.catalogFile, + sha, + ); + const sharedCatalog = + catalogText === null + ? undefined + : extractEnumCatalog( + ts.createSourceFile( + config.catalogFile, + catalogText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ), + config.enumName, + ); + + const models: AnalyticsModel[] = []; + if (catalogText !== null) { + models.push( + extractAnalyticsModel( + config.catalogFile, + catalogText, + config, + sharedCatalog, + ), + ); + } + for (const file of files) { + if (file === config.catalogFile) { + continue; + } + const text = await getFileAtRef(octokit, clientRepo, file, sha); + if (text === null) { + continue; + } + models.push(extractAnalyticsModel(file, text, config, sharedCatalog)); + } + return mergeAnalyticsModels(models); +} + +/** + * Summary when the PR has no analytics-looking diff. + * + * @param branch - Bot branch name. + * @returns Empty generate summary. + */ +export function emptySummary(branch: string): GenerateSummary { + return { + hasChanges: false, + branch, + changeset: { + eventsAdded: [], + eventsRemoved: [], + eventsRenamed: [], + propertiesAdded: [], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], + }, + intendedFiles: [], + schemaPrNumber: null, + schemaPrUrl: null, + }; +} + +/** + * Writes the generate summary JSON next to the schema checkout. + * + * @param schemaDir - Schema working tree. + * @param summary - Generate result. + * @returns Absolute path of the summary file. + */ +export async function writeSummaryFile( + schemaDir: string, + summary: GenerateSummary, +): Promise { + const filePath = path.join( + schemaDir, + '.segment-schema-draft-pr-summary.json', + ); + await fs.writeFile(filePath, JSON.stringify(summary, replacer, 2)); + return filePath; +} + +/** + * JSON replacer that serializes Maps. + * + * @param _key - JSON key. + * @param value - Value. + * @returns JSON-safe value. + */ +function replacer(_key: string, value: unknown): unknown { + if (value instanceof Map) { + return Object.fromEntries(value); + } + return value; +} diff --git a/src/segment-schema-draft-pr/github-api.test.ts b/src/segment-schema-draft-pr/github-api.test.ts new file mode 100644 index 00000000..62521c4a --- /dev/null +++ b/src/segment-schema-draft-pr/github-api.test.ts @@ -0,0 +1,184 @@ +import type { Octokit } from '@octokit/rest'; + +import { OPT_OUT_LABEL, PROPOSAL_MARKER } from './constants'; +import { + createSchemaPr, + findOpenSchemaPr, + parseRepo, + resolveClientPr, + updateProposalCommentIfExists, + upsertProposalComment, +} from './github-api'; + +const PULL_NUMBER = 'pull_number'; +const PER_PAGE = 'per_page'; +const COMMENT_ID = 'comment_id'; +const HTML_URL = 'html_url'; +const FULL_NAME = 'full_name'; + +describe('github-api', () => { + it('parses owner/repo', () => { + expect(parseRepo('Consensys/segment-schema')).toStrictEqual({ + owner: 'Consensys', + repo: 'segment-schema', + }); + }); + + it('resolves the client PR including fork and opt-out label', async () => { + const pullsGet = jest.fn().mockResolvedValue({ + data: { + number: 12, + base: { sha: 'base' }, + head: { sha: 'head', repo: { [FULL_NAME]: 'fork/mobile' } }, + user: { login: 'alice' }, + state: 'open', + merged: false, + labels: [{ name: OPT_OUT_LABEL }], + }, + }); + const octokit = { pulls: { get: pullsGet } } as unknown as Octokit; + + const pr = await resolveClientPr( + octokit, + { owner: 'MetaMask', repo: 'metamask-mobile' }, + 12, + 'MetaMask/metamask-mobile', + ); + + expect(pr.isFork).toBe(true); + expect(pr.hasOptOutLabel).toBe(true); + expect(pr.baseSha).toBe('base'); + expect(pr.headSha).toBe('head'); + expect(pullsGet).toHaveBeenCalledWith({ + owner: 'MetaMask', + repo: 'metamask-mobile', + [PULL_NUMBER]: 12, + }); + }); + + it('finds an open schema PR by bot head branch', async () => { + const pullsList = jest.fn().mockResolvedValue({ + data: [ + { + number: 9, + [HTML_URL]: 'https://github.com/Consensys/segment-schema/pull/9', + body: 'body', + draft: true, + }, + ], + }); + const octokit = { pulls: { list: pullsList } } as unknown as Octokit; + const found = await findOpenSchemaPr( + octokit, + { owner: 'Consensys', repo: 'segment-schema' }, + 'metamaskbot/mobile-pr-12', + ); + expect(found?.number).toBe(9); + expect(pullsList).toHaveBeenCalledWith({ + owner: 'Consensys', + repo: 'segment-schema', + head: 'Consensys:metamaskbot/mobile-pr-12', + state: 'open', + [PER_PAGE]: 5, + }); + }); + + it('creates a draft schema PR', async () => { + const pullsCreate = jest.fn().mockResolvedValue({ + data: { + number: 3, + [HTML_URL]: 'https://github.com/Consensys/segment-schema/pull/3', + body: 'x', + draft: true, + }, + }); + const octokit = { pulls: { create: pullsCreate } } as unknown as Octokit; + const created = await createSchemaPr( + octokit, + { owner: 'Consensys', repo: 'segment-schema' }, + { + branch: 'metamaskbot/mobile-pr-12', + base: 'main', + title: 'Draft', + body: 'body', + }, + ); + expect(created.draft).toBe(true); + expect(pullsCreate).toHaveBeenCalledWith( + expect.objectContaining({ draft: true }), + ); + }); + + it('edits the existing proposal comment instead of posting a second one', async () => { + const paginate = jest + .fn() + .mockResolvedValue([{ id: 77, body: `${PROPOSAL_MARKER}\nold` }]); + const updateComment = jest.fn().mockResolvedValue({}); + const createComment = jest.fn().mockResolvedValue({}); + const octokit = { + paginate, + issues: { updateComment, createComment }, + } as unknown as Octokit; + + await upsertProposalComment( + octokit, + { owner: 'MetaMask', repo: 'metamask-mobile' }, + 12, + `${PROPOSAL_MARKER}\nnew`, + ); + expect(updateComment).toHaveBeenCalledWith({ + owner: 'MetaMask', + repo: 'metamask-mobile', + [COMMENT_ID]: 77, + body: `${PROPOSAL_MARKER}\nnew`, + }); + expect(createComment).not.toHaveBeenCalled(); + }); + + it('updates an existing proposal comment and skips create', async () => { + const paginate = jest + .fn() + .mockResolvedValue([{ id: 77, body: `${PROPOSAL_MARKER}\nold` }]); + const updateComment = jest.fn().mockResolvedValue({}); + const createComment = jest.fn().mockResolvedValue({}); + const octokit = { + paginate, + issues: { updateComment, createComment }, + } as unknown as Octokit; + + const updated = await updateProposalCommentIfExists( + octokit, + { owner: 'MetaMask', repo: 'metamask-mobile' }, + 12, + `${PROPOSAL_MARKER}\nnew`, + ); + expect(updated).toBe(true); + expect(updateComment).toHaveBeenCalledWith({ + owner: 'MetaMask', + repo: 'metamask-mobile', + [COMMENT_ID]: 77, + body: `${PROPOSAL_MARKER}\nnew`, + }); + expect(createComment).not.toHaveBeenCalled(); + }); + + it('no-ops when no proposal comment exists', async () => { + const paginate = jest.fn().mockResolvedValue([]); + const updateComment = jest.fn().mockResolvedValue({}); + const createComment = jest.fn().mockResolvedValue({}); + const octokit = { + paginate, + issues: { updateComment, createComment }, + } as unknown as Octokit; + + const updated = await updateProposalCommentIfExists( + octokit, + { owner: 'MetaMask', repo: 'metamask-mobile' }, + 12, + `${PROPOSAL_MARKER}\nnew`, + ); + expect(updated).toBe(false); + expect(updateComment).not.toHaveBeenCalled(); + expect(createComment).not.toHaveBeenCalled(); + }); +}); diff --git a/src/segment-schema-draft-pr/github-api.ts b/src/segment-schema-draft-pr/github-api.ts new file mode 100644 index 00000000..a7649944 --- /dev/null +++ b/src/segment-schema-draft-pr/github-api.ts @@ -0,0 +1,335 @@ +import { Octokit } from '@octokit/rest'; + +import { OPT_OUT_LABEL, PROPOSAL_MARKER } from './constants'; +import type { ResolvedClientPr, SchemaPr } from './types'; + +const PULL_NUMBER = 'pull_number'; +const PER_PAGE = 'per_page'; +const ISSUE_NUMBER = 'issue_number'; +const COMMENT_ID = 'comment_id'; + +export type RepoId = { + owner: string; + repo: string; +}; + +/** + * Parses `owner/repo` from an action input. + * + * @param fullName - `Consensys/segment-schema`. + * @returns Owner and repo. + */ +export function parseRepo(fullName: string): RepoId { + const [owner, repo] = fullName.split('/'); + if (!owner || !repo) { + throw new Error(`Invalid repository: ${fullName}`); + } + return { owner, repo }; +} + +/** + * Builds authenticated Octokit clients for the client PR and schema repo. + * + * @param githubToken - Token for Mobile/Extension. + * @param schemaToken - GitHub App token for segment-schema. + * @returns Two Octokit instances. + */ +export function createOctokitClients( + githubToken: string, + schemaToken: string, +): { client: Octokit; schema: Octokit } { + return { + client: new Octokit({ auth: githubToken }), + schema: new Octokit({ auth: schemaToken }), + }; +} + +/** + * Loads the client pull request (needed on `issue_comment`). + * + * @param octokit - Client-repo Octokit. + * @param repoId - Mobile or Extension repo. + * @param prNumber - Pull request number. + * @param currentRepository - `github.repository` of the workflow. + * @returns Resolved SHAs, author, fork, labels, open/merged. + */ +export async function resolveClientPr( + octokit: Octokit, + repoId: RepoId, + prNumber: number, + currentRepository: string, +): Promise { + const { data } = await octokit.pulls.get({ + owner: repoId.owner, + repo: repoId.repo, + [PULL_NUMBER]: prNumber, + }); + + const labels = data.labels.map((label) => { + return typeof label === 'string' ? label : (label.name ?? ''); + }); + + const headRepoFullName = data.head.repo?.full_name ?? ''; + + return { + number: data.number, + baseSha: data.base.sha, + headSha: data.head.sha, + authorLogin: data.user?.login ?? '', + isOpen: data.state === 'open', + merged: Boolean(data.merged), + isFork: headRepoFullName !== currentRepository, + hasOptOutLabel: labels.includes(OPT_OUT_LABEL), + headRepoFullName, + }; +} + +/** + * Finds an open schema PR whose head is the bot branch. + * + * @param octokit - Schema-repo Octokit. + * @param repoId - Schema repo. + * @param branch - `metamaskbot/-pr-`. + * @returns Open PR, or null. + */ +export async function findOpenSchemaPr( + octokit: Octokit, + repoId: RepoId, + branch: string, +): Promise { + const { data } = await octokit.pulls.list({ + owner: repoId.owner, + repo: repoId.repo, + head: `${repoId.owner}:${branch}`, + state: 'open', + [PER_PAGE]: 5, + }); + + const match = data[0]; + if (!match) { + return null; + } + + return { + number: match.number, + htmlUrl: match.html_url, + body: match.body ?? '', + draft: Boolean(match.draft), + }; +} + +/** + * Creates a draft schema PR. + * + * @param octokit - Schema-repo Octokit. + * @param repoId - Schema repo. + * @param params - Branch, title, body, base. + * @param params.branch - Bot branch name. + * @param params.base - Schema base branch. + * @param params.title - Pull request title. + * @param params.body - Pull request body. + * @returns Created PR. + */ +export async function createSchemaPr( + octokit: Octokit, + repoId: RepoId, + params: { branch: string; base: string; title: string; body: string }, +): Promise { + const { data } = await octokit.pulls.create({ + owner: repoId.owner, + repo: repoId.repo, + head: params.branch, + base: params.base, + title: params.title, + body: params.body, + draft: true, + }); + + return { + number: data.number, + htmlUrl: data.html_url, + body: data.body ?? '', + draft: Boolean(data.draft), + }; +} + +/** + * Updates a schema PR body. + * + * @param octokit - Schema-repo Octokit. + * @param repoId - Schema repo. + * @param prNumber - Schema PR number. + * @param body - Full body after splice. + */ +export async function updateSchemaPrBody( + octokit: Octokit, + repoId: RepoId, + prNumber: number, + body: string, +): Promise { + await octokit.pulls.update({ + owner: repoId.owner, + repo: repoId.repo, + [PULL_NUMBER]: prNumber, + body, + }); +} + +/** + * Closes a schema PR. + * + * @param octokit - Schema-repo Octokit. + * @param repoId - Schema repo. + * @param prNumber - Schema PR number. + */ +export async function closeSchemaPr( + octokit: Octokit, + repoId: RepoId, + prNumber: number, +): Promise { + await octokit.pulls.update({ + owner: repoId.owner, + repo: repoId.repo, + [PULL_NUMBER]: prNumber, + state: 'closed', + }); +} + +/** + * Posts or edits the single proposal comment on the client PR. + * + * @param octokit - Client-repo Octokit. + * @param repoId - Client repo. + * @param prNumber - Client PR number. + * @param body - New comment body (includes the proposal marker). + */ +export async function upsertProposalComment( + octokit: Octokit, + repoId: RepoId, + prNumber: number, + body: string, +): Promise { + const existing = await findProposalComment(octokit, repoId, prNumber); + if (existing) { + await octokit.issues.updateComment({ + owner: repoId.owner, + repo: repoId.repo, + [COMMENT_ID]: existing.id, + body, + }); + return; + } + + await octokit.issues.createComment({ + owner: repoId.owner, + repo: repoId.repo, + [ISSUE_NUMBER]: prNumber, + body, + }); +} + +/** + * Edits the sticky proposal comment when it already exists. + * + * Why: a later head with no analytics diff must not post a new "no longer + * needed" comment on every pull request that never had a proposal. + * + * @param octokit - Client-repo Octokit. + * @param repoId - Client repo. + * @param prNumber - Client PR number. + * @param body - New comment body (includes the proposal marker). + * @returns Whether an existing proposal comment was updated. + */ +export async function updateProposalCommentIfExists( + octokit: Octokit, + repoId: RepoId, + prNumber: number, + body: string, +): Promise { + const existing = await findProposalComment(octokit, repoId, prNumber); + if (!existing) { + return false; + } + + await octokit.issues.updateComment({ + owner: repoId.owner, + repo: repoId.repo, + [COMMENT_ID]: existing.id, + body, + }); + return true; +} + +/** + * Finds the sticky proposal comment on a client PR, if one was posted. + * + * @param octokit - Client-repo Octokit. + * @param repoId - Client repo. + * @param prNumber - Client PR number. + * @returns The matching comment, or undefined. + */ +async function findProposalComment( + octokit: Octokit, + repoId: RepoId, + prNumber: number, +): Promise<{ id: number } | undefined> { + const comments = await octokit.paginate( + 'GET /repos/{owner}/{repo}/issues/{issue_number}/comments', + { + owner: repoId.owner, + repo: repoId.repo, + [ISSUE_NUMBER]: prNumber, + [PER_PAGE]: 100, + }, + ); + + return comments.find((comment) => comment.body?.includes(PROPOSAL_MARKER)); +} + +/** + * Posts a new (non-proposal) comment on a PR. + * + * @param octokit - Octokit for that repo. + * @param repoId - Repo. + * @param issueNumber - PR/issue number. + * @param body - Markdown. + */ +export async function postComment( + octokit: Octokit, + repoId: RepoId, + issueNumber: number, + body: string, +): Promise { + await octokit.issues.createComment({ + owner: repoId.owner, + repo: repoId.repo, + [ISSUE_NUMBER]: issueNumber, + body, + }); +} + +/** + * Loads the schema repo pull request template, if present. + * + * @param octokit - Schema-repo Octokit. + * @param repoId - Schema repo. + * @returns Template text, or empty string. + */ +export async function loadPrTemplate( + octokit: Octokit, + repoId: RepoId, +): Promise { + try { + const { data } = await octokit.repos.getContent({ + owner: repoId.owner, + repo: repoId.repo, + path: '.github/pull-request-template.md', + }); + if ('content' in data && data.content) { + return Buffer.from(data.content, 'base64').toString('utf8'); + } + } catch { + return ''; + } + return ''; +} diff --git a/src/segment-schema-draft-pr/github-output.test.ts b/src/segment-schema-draft-pr/github-output.test.ts new file mode 100644 index 00000000..27055b6e --- /dev/null +++ b/src/segment-schema-draft-pr/github-output.test.ts @@ -0,0 +1,28 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { writeGithubOutput } from './github-output'; + +describe('writeGithubOutput', () => { + it('appends key=value lines to the output file', async () => { + const outputFile = path.join( + await fs.mkdtemp(path.join(os.tmpdir(), 'gh-out-')), + 'output', + ); + writeGithubOutput(outputFile, [ + ['has_changes', 'true'], + ['branch', 'metamaskbot/mobile-pr-1'], + ]); + const text = await fs.readFile(outputFile, 'utf8'); + expect(text).toBe('has_changes=true\nbranch=metamaskbot/mobile-pr-1\n'); + await fs.rm(path.dirname(outputFile), { recursive: true, force: true }); + }); + + it('prints lines when GITHUB_OUTPUT is unset', () => { + const log = jest.spyOn(console, 'log').mockImplementation(); + writeGithubOutput(undefined, [['skip', 'true']]); + expect(log).toHaveBeenCalledWith('skip=true'); + log.mockRestore(); + }); +}); diff --git a/src/segment-schema-draft-pr/github-output.ts b/src/segment-schema-draft-pr/github-output.ts new file mode 100644 index 00000000..b61b1f83 --- /dev/null +++ b/src/segment-schema-draft-pr/github-output.ts @@ -0,0 +1,21 @@ +import { appendFileSync } from 'fs'; + +/** + * Appends key=value lines to GITHUB_OUTPUT when running in Actions. + * + * @param githubOutput - Path from GITHUB_OUTPUT, if set. + * @param entries - Output keys and values. + */ +export function writeGithubOutput( + githubOutput: string | undefined, + entries: readonly (readonly [string, string])[], +): void { + const lines = entries.map(([key, value]) => `${key}=${value}`); + for (const line of lines) { + console.log(line); + } + if (!githubOutput) { + return; + } + appendFileSync(githubOutput, `${lines.join('\n')}\n`); +} diff --git a/src/segment-schema-draft-pr/names.test.ts b/src/segment-schema-draft-pr/names.test.ts new file mode 100644 index 00000000..aec3d54c --- /dev/null +++ b/src/segment-schema-draft-pr/names.test.ts @@ -0,0 +1,16 @@ +import { toKebabSlug, toSnakeCase } from './names'; + +describe('names', () => { + it('converts camelCase keys to snake_case', () => { + expect(toSnakeCase('usdValue')).toBe('usd_value'); + expect(toSnakeCase('chainId')).toBe('chain_id'); + expect(toSnakeCase('already_snake')).toBe('already_snake'); + }); + + it('converts event display names to kebab slugs', () => { + expect(toKebabSlug('App Opened')).toBe('app-opened'); + expect(toKebabSlug('Perp Trade Transaction')).toBe( + 'perp-trade-transaction', + ); + }); +}); diff --git a/src/segment-schema-draft-pr/names.ts b/src/segment-schema-draft-pr/names.ts new file mode 100644 index 00000000..248e1f53 --- /dev/null +++ b/src/segment-schema-draft-pr/names.ts @@ -0,0 +1,25 @@ +/** + * Converts a client camelCase property key to schema snake_case. + * + * @param key - Property key from TypeScript. + * @returns The snake_case key. + */ +export function toSnakeCase(key: string): string { + return key + .replace(/([a-z0-9])([A-Z])/gu, '$1_$2') + .replace(/([A-Z])([A-Z][a-z])/gu, '$1_$2') + .toLowerCase(); +} + +/** + * Converts an event display name to a YAML file slug. + * + * @param eventName - Segment event name, e.g. `App Opened`. + * @returns The kebab-case file stem. + */ +export function toKebabSlug(eventName: string): string { + return eventName + .toLowerCase() + .replace(/[^a-z0-9]+/gu, '-') + .replace(/^-|-$/gu, ''); +} diff --git a/src/segment-schema-draft-pr/parse-args.test.ts b/src/segment-schema-draft-pr/parse-args.test.ts new file mode 100644 index 00000000..2b40e0fa --- /dev/null +++ b/src/segment-schema-draft-pr/parse-args.test.ts @@ -0,0 +1,60 @@ +import { parseCliArgs } from './parse-args'; + +describe('parseCliArgs', () => { + it('parses required flags and optional previous-dir', () => { + const args = parseCliArgs( + [ + '--phase', + 'generate', + '--mode', + 'propose', + '--platform', + 'mobile', + '--schema', + '../segment-schema', + '--client-repository', + 'MetaMask/metamask-mobile', + '--base-sha', + 'aaa', + '--head-sha', + 'bbb', + '--pr-number', + '12', + '--previous-dir', + '../segment-schema-previous', + '--dry-run', + ], + { + GITHUB_TOKEN: 'ghs_client', + SEGMENT_SCHEMA_TOKEN: 'ghs_schema', + }, + ); + + expect(args.phase).toBe('generate'); + expect(args.previousDir).toBe('../segment-schema-previous'); + expect(args.dryRun).toBe(true); + expect(args.prNumber).toBe(12); + expect(args.clientRepository).toBe('MetaMask/metamask-mobile'); + expect(args.noAnalyticsDiff).toBe(false); + expect(args.tooManyFiles).toBe(false); + }); + + it('parses --no-analytics-diff and --too-many-files', () => { + const args = parseCliArgs( + [ + '--phase', + 'publish', + '--mode', + 'propose', + '--platform', + 'mobile', + '--no-analytics-diff', + '--too-many-files', + ], + {}, + ); + + expect(args.noAnalyticsDiff).toBe(true); + expect(args.tooManyFiles).toBe(true); + }); +}); diff --git a/src/segment-schema-draft-pr/parse-args.ts b/src/segment-schema-draft-pr/parse-args.ts new file mode 100644 index 00000000..16382880 --- /dev/null +++ b/src/segment-schema-draft-pr/parse-args.ts @@ -0,0 +1,91 @@ +import type { Mode, Phase, Platform } from './types'; + +export type CliArgs = { + phase: Phase; + mode: Mode; + platform: Platform; + schema: string; + previousDir: string | undefined; + baseSha: string; + headSha: string; + prNumber: number; + clientRepository: string; + segmentSchemaRepository: string; + segmentSchemaBase: string; + githubToken: string; + segmentSchemaToken: string; + dryRun: boolean; + defaultLibrary: string | undefined; + pushed: boolean; + githubOutput: string | undefined; + noAnalyticsDiff: boolean; + tooManyFiles: boolean; +}; + +/** + * Parses CLI flags used by the composite action. + * + * @param argv - Arguments after the node entrypoint. + * @param env - Process environment map. + * @returns Typed args. + */ +export function parseCliArgs(argv: string[], env: NodeJS.ProcessEnv): CliArgs { + const raw = new Map(); + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + if (!token?.startsWith('--')) { + continue; + } + const key = token.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith('--')) { + raw.set(key, 'true'); + continue; + } + raw.set(key, next); + i += 1; + } + + const phase = requireFlag(raw, 'phase') as Phase; + const mode = requireFlag(raw, 'mode') as Mode; + const platform = requireFlag(raw, 'platform') as Platform; + + return { + phase, + mode, + platform, + schema: raw.get('schema') ?? '', + previousDir: raw.get('previous-dir'), + baseSha: raw.get('base-sha') ?? '', + headSha: raw.get('head-sha') ?? '', + prNumber: Number(raw.get('pr-number') ?? env.PR_NUMBER ?? '0'), + clientRepository: + raw.get('client-repository') ?? env.GITHUB_REPOSITORY ?? '', + segmentSchemaRepository: + raw.get('segment-schema-repository') ?? 'Consensys/segment-schema', + segmentSchemaBase: raw.get('segment-schema-base') ?? 'main', + githubToken: env.GITHUB_TOKEN ?? '', + segmentSchemaToken: env.SEGMENT_SCHEMA_TOKEN ?? '', + dryRun: raw.get('dry-run') === 'true', + defaultLibrary: raw.get('default-library'), + pushed: raw.get('pushed') === 'true', + githubOutput: env.GITHUB_OUTPUT, + noAnalyticsDiff: raw.get('no-analytics-diff') === 'true', + tooManyFiles: raw.get('too-many-files') === 'true', + }; +} + +/** + * Requires a flag to be present. + * + * @param raw - Parsed flag map. + * @param name - Flag name. + * @returns Value. + */ +function requireFlag(raw: Map, name: string): string { + const value = raw.get(name); + if (!value) { + throw new Error(`Missing --${name}`); + } + return value; +} diff --git a/src/segment-schema-draft-pr/pr-body.test.ts b/src/segment-schema-draft-pr/pr-body.test.ts new file mode 100644 index 00000000..130267a6 --- /dev/null +++ b/src/segment-schema-draft-pr/pr-body.test.ts @@ -0,0 +1,96 @@ +import { + AGREEMENT_PHRASE, + BODY_END_MARKER, + BODY_START_MARKER, + MAX_CHANGED_TS_FILES, + PROPOSAL_MARKER, +} from './constants'; +import { + renderProposalComment, + renderTooManyFilesComment, + spliceSchemaPrBody, +} from './pr-body'; +import type { AnalyticsChangeSet } from './types'; + +const CHANGESET: AnalyticsChangeSet = { + eventsAdded: [{ enumKey: 'NEW_EVENT', eventName: 'New Event' }], + eventsRemoved: [{ enumKey: 'GONE', eventName: 'Gone' }], + eventsRenamed: [], + propertiesAdded: [{ eventName: 'New Event', key: 'source', type: 'string' }], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], +}; + +const TEMPLATE = `### 1️⃣ Why is this change needed? + +_Type here..._ + +### 2️⃣ What changed? + +_Type here..._ + +### 3️⃣ Business Value checklist +`; + +describe('pr-body', () => { + it('includes the agreement sentence in a copy-paste block', () => { + const body = renderProposalComment(CHANGESET, [ + { + path: 'libraries/events/metamask-mobile-unsorted/new-event.yaml', + kind: 'create', + eventName: 'New Event', + }, + ]); + expect(body).toContain(PROPOSAL_MARKER); + expect(body).toContain(AGREEMENT_PHRASE); + expect(body).toContain('New Event'); + expect(body).toContain('Gone'); + }); + + it('omits the agreement sentence when there are no writable YAML changes', () => { + const listedOnly: AnalyticsChangeSet = { + eventsAdded: [], + eventsRemoved: [{ enumKey: 'GONE', eventName: 'Gone' }], + eventsRenamed: [], + propertiesAdded: [], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], + }; + const body = renderProposalComment(listedOnly, []); + expect(body).toContain(PROPOSAL_MARKER); + expect(body).toContain('Gone'); + expect(body).not.toContain(AGREEMENT_PHRASE); + expect(body).toContain('No draft will be opened from this pull request.'); + }); + + it('tells the author to open the schema PR themselves when the file cap is exceeded', () => { + const body = renderTooManyFilesComment(MAX_CHANGED_TS_FILES); + expect(body).toContain(PROPOSAL_MARKER); + expect(body).toContain(String(MAX_CHANGED_TS_FILES)); + expect(body).toContain( + 'Open the draft on Consensys/segment-schema yourself', + ); + }); + + it('splices only the generated block and keeps human template sections', () => { + const first = spliceSchemaPrBody( + '', + TEMPLATE, + 'Client PR: https://example', + ); + expect(first).toContain(BODY_START_MARKER); + expect(first).toContain('Client PR: https://example'); + expect(first).toContain('### 3️⃣ Business Value checklist'); + + const edited = first.replace('_Type here..._', 'Human why'); + const second = spliceSchemaPrBody(edited, TEMPLATE, 'Updated block'); + expect(second).toContain('Human why'); + expect(second).toContain('Updated block'); + expect(second).not.toContain('Client PR: https://example'); + expect(second.indexOf(BODY_START_MARKER)).toBeLessThan( + second.indexOf(BODY_END_MARKER), + ); + }); +}); diff --git a/src/segment-schema-draft-pr/pr-body.ts b/src/segment-schema-draft-pr/pr-body.ts new file mode 100644 index 00000000..750aa1f3 --- /dev/null +++ b/src/segment-schema-draft-pr/pr-body.ts @@ -0,0 +1,326 @@ +import { + AGREEMENT_PHRASE, + BODY_END_MARKER, + BODY_START_MARKER, + MAX_CHANGED_TS_FILES, + PROPOSAL_MARKER, +} from './constants'; +import { changesetHasWritableChanges } from './diff-models'; +import type { AnalyticsChangeSet, IntendedFileChange, SchemaPr } from './types'; + +/** + * Proposal comment asking the author to agree to open a draft schema PR. + * + * Why: listed-only diffs (removals, renames, type changes, unresolved) cannot + * produce YAML, so the copy-paste agreement sentence is omitted. + * + * @param changeset - Detected analytics diff. + * @param intendedFiles - YAML paths that would be written. + * @returns Markdown body including the copy-paste agreement sentence when writable. + */ +export function renderProposalComment( + changeset: AnalyticsChangeSet, + intendedFiles: IntendedFileChange[], +): string { + const lines = [ + PROPOSAL_MARKER, + 'This pull request looks like it changes analytics events or properties.', + '', + '### What triggered this', + renderChangeset(changeset), + '', + '### Intended Segment schema files', + renderIntendedFiles(intendedFiles), + '', + ]; + + if (changesetHasWritableChanges(changeset)) { + lines.push( + 'This does not block merge. To open a **draft** PR on the Segment schema repo, reply with:', + '', + '```', + AGREEMENT_PHRASE, + '```', + '', + ); + } else { + lines.push( + 'Removals, renames, type changes, and unresolved keys need a human decision on the Segment schema repo. No draft will be opened from this pull request.', + '', + ); + } + + return lines.join('\n'); +} + +/** + * Proposal comment after the draft schema PR exists. + * + * @param schemaPr - Open schema PR. + * @returns Status markdown. + */ +export function renderDraftOpenComment(schemaPr: SchemaPr): string { + return [ + PROPOSAL_MARKER, + `A draft Segment schema PR is open: ${schemaPr.htmlUrl}`, + '', + 'Later analytics pushes on this pull request update that draft. Fill TODO descriptions, tick the checklists, then mark it ready for Data Council.', + '', + ].join('\n'); +} + +/** + * Proposal comment when the current head no longer needs schema YAML. + * + * @returns Status markdown. + */ +export function renderNoLongerNeededComment(): string { + return [ + PROPOSAL_MARKER, + 'The current head no longer has analytics changes that would update the Segment schema. No draft schema PR will be opened unless those changes return.', + '', + ].join('\n'); +} + +/** + * Client PR comment after the draft is created. + * + * @param schemaPr - New schema PR. + * @returns Markdown. + */ +export function renderCreatedComment(schemaPr: SchemaPr): string { + return `Opened a draft Segment schema PR: ${schemaPr.htmlUrl}`; +} + +/** + * Client PR comment after the draft YAML is updated. + * + * @param schemaPr - Existing schema PR. + * @returns Markdown. + */ +export function renderUpdatedComment(schemaPr: SchemaPr): string { + return `Updated the draft Segment schema PR: ${schemaPr.htmlUrl}`; +} + +/** + * Client PR comment when the schema PR is kept but YAML is unchanged. + * + * @param schemaPr - Existing schema PR. + * @returns Markdown. + */ +export function renderStaleComment(schemaPr: SchemaPr): string { + return [ + PROPOSAL_MARKER, + `This pull request no longer needs a Segment schema change. The draft stays open for reuse: ${schemaPr.htmlUrl}`, + '', + ].join('\n'); +} + +/** + * Comment on the schema PR before closing it. + * + * @returns Markdown. + */ +export function renderClosedSchemaComment(): string { + return 'Closing this draft because the linked client pull request was closed without merging.'; +} + +/** + * Client comment when the opt-in window ends because the PR closed unmerged. + * + * @returns Markdown. + */ +export function renderWindowClosedComment(): string { + return 'The Segment schema draft opt-in window ended because this pull request closed without merging. If you still need a schema change, open the draft on Consensys/segment-schema yourself.'; +} + +/** + * Sticky comment when the PR is too large for automatic extraction. + * + * @param cap - Maximum non-test TypeScript files the generator will walk. + * @returns Markdown. + */ +export function renderTooManyFilesComment( + cap: number = MAX_CHANGED_TS_FILES, +): string { + return [ + PROPOSAL_MARKER, + `This pull request changes more than ${cap} non-test TypeScript files, so automatic Segment schema detection is skipped. Open the draft on Consensys/segment-schema yourself if this pull request changes analytics events.`, + '', + ].join('\n'); +} + +/** + * Sticky comment when detection is skipped because of the file cap and a draft already exists. + * + * @param schemaPr - Existing schema PR. + * @param cap - Maximum non-test TypeScript files the generator will walk. + * @returns Markdown. + */ +export function renderTooManyFilesStaleComment( + schemaPr: SchemaPr, + cap: number = MAX_CHANGED_TS_FILES, +): string { + return [ + PROPOSAL_MARKER, + `This pull request no longer needs a Segment schema change. The draft stays open for reuse: ${schemaPr.htmlUrl}`, + '', + `Automatic detection was skipped because this pull request changes more than ${cap} non-test TypeScript files.`, + '', + ].join('\n'); +} + +/** + * Proposal comment when the author agreed but YAML would be empty. + * + * @param changeset - Listed-only diff. + * @returns Markdown. + */ +export function renderListedOnlyComment(changeset: AnalyticsChangeSet): string { + return [ + PROPOSAL_MARKER, + 'Agreement received, but there are no additive YAML changes to open a draft schema PR. Removals, renames, type changes, and unresolved keys are listed only.', + '', + renderChangeset(changeset), + '', + ].join('\n'); +} + +/** + * Replaces only the generated block in a schema PR body. + * + * @param existingBody - Current PR body, or empty for create. + * @param template - Schema repo PR template (used when creating). + * @param generatedBlock - Markdown for the generated section. + * @returns Full body with markers preserved around human edits. + */ +export function spliceSchemaPrBody( + existingBody: string, + template: string, + generatedBlock: string, +): string { + const block = [ + BODY_START_MARKER, + generatedBlock.trim(), + BODY_END_MARKER, + ].join('\n'); + const source = existingBody.trim() === '' ? template : existingBody; + + if (source.includes(BODY_START_MARKER) && source.includes(BODY_END_MARKER)) { + const start = source.indexOf(BODY_START_MARKER); + const end = source.indexOf(BODY_END_MARKER) + BODY_END_MARKER.length; + return `${source.slice(0, start)}${block}${source.slice(end)}`; + } + + if (source.includes('### 2️⃣ What changed?')) { + return source.replace( + '### 2️⃣ What changed?', + `### 2️⃣ What changed?\n\n${block}`, + ); + } + + return `${source.trim()}\n\n${block}\n`; +} + +/** + * Generated "What changed" block for the schema PR. + * + * @param clientPrUrl - Link to the Mobile/Extension PR. + * @param changeset - Diff. + * @param intendedFiles - Files written. + * @returns Markdown inside the splice markers. + */ +export function renderGeneratedSchemaBlock( + clientPrUrl: string, + changeset: AnalyticsChangeSet, + intendedFiles: IntendedFileChange[], +): string { + return [ + `Client PR: ${clientPrUrl}`, + '', + renderChangeset(changeset), + '', + '### Files', + renderIntendedFiles(intendedFiles), + '', + 'Removals, renames, type changes, and `required: true` are listed for a human and were not applied.', + ].join('\n'); +} + +/** + * Renders the changeset as markdown lists. + * + * @param changeset - Diff. + * @returns Markdown. + */ +function renderChangeset(changeset: AnalyticsChangeSet): string { + const lines: string[] = []; + + if (changeset.eventsAdded.length > 0) { + lines.push('**Events added**'); + for (const event of changeset.eventsAdded) { + lines.push(`- \`${event.eventName}\` (\`${event.enumKey}\`)`); + } + } + if (changeset.propertiesAdded.length > 0) { + lines.push('**Properties added**'); + for (const property of changeset.propertiesAdded) { + lines.push( + `- \`${property.eventName}\`.${property.key} (${property.type})`, + ); + } + } + if (changeset.eventsRemoved.length > 0) { + lines.push('**Events removed (not applied)**'); + for (const event of changeset.eventsRemoved) { + lines.push(`- \`${event.eventName}\``); + } + } + if (changeset.eventsRenamed.length > 0) { + lines.push('**Events renamed (not applied)**'); + for (const event of changeset.eventsRenamed) { + lines.push(`- \`${event.fromName}\` → \`${event.toName}\``); + } + } + if (changeset.propertiesRemoved.length > 0) { + lines.push('**Properties removed (not applied)**'); + for (const property of changeset.propertiesRemoved) { + lines.push(`- \`${property.eventName}\`.${property.key}`); + } + } + if (changeset.typeChanges.length > 0) { + lines.push('**Type changes (not applied)**'); + for (const property of changeset.typeChanges) { + lines.push( + `- \`${property.eventName}\`.${property.key} → ${property.type}`, + ); + } + } + if (changeset.unresolved.length > 0) { + lines.push('**Unresolved (not guessed as object)**'); + for (const item of changeset.unresolved) { + lines.push(`- \`${item.eventName}\`.${item.key} in \`${item.file}\``); + } + } + + if (lines.length === 0) { + return '_No additive analytics changes detected._'; + } + + return lines.join('\n'); +} + +/** + * Renders intended YAML paths. + * + * @param intendedFiles - File list. + * @returns Markdown list. + */ +function renderIntendedFiles(intendedFiles: IntendedFileChange[]): string { + if (intendedFiles.length === 0) { + return '_No YAML files would be written (listed-only changes)._'; + } + return intendedFiles + .map((file) => `- \`${file.path}\` (${file.kind}) — \`${file.eventName}\``) + .join('\n'); +} diff --git a/src/segment-schema-draft-pr/publish.test.ts b/src/segment-schema-draft-pr/publish.test.ts new file mode 100644 index 00000000..857ed4e9 --- /dev/null +++ b/src/segment-schema-draft-pr/publish.test.ts @@ -0,0 +1,176 @@ +import type { Octokit } from '@octokit/rest'; + +import { PROPOSAL_MARKER } from './constants'; +import type { CliArgs } from './parse-args'; +import { publishGenerateResult, publishTooManyFiles } from './publish'; +import type { GenerateSummary } from './types'; + +const HTML_URL = 'html_url'; +const COMMENT_ID = 'comment_id'; + +const EMPTY_CHANGESET = { + eventsAdded: [], + eventsRemoved: [], + eventsRenamed: [], + propertiesAdded: [], + propertiesRemoved: [], + typeChanges: [], + unresolved: [], +}; + +const SUMMARY: GenerateSummary = { + hasChanges: false, + branch: 'metamaskbot/mobile-pr-12', + changeset: EMPTY_CHANGESET, + intendedFiles: [], + schemaPrNumber: null, + schemaPrUrl: null, +}; + +const ARGS: CliArgs = { + phase: 'publish', + mode: 'propose', + platform: 'mobile', + schema: '', + previousDir: undefined, + baseSha: 'base', + headSha: 'head', + prNumber: 12, + clientRepository: 'MetaMask/metamask-mobile', + segmentSchemaRepository: 'Consensys/segment-schema', + segmentSchemaBase: 'main', + githubToken: 'ghs_client', + segmentSchemaToken: 'ghs_schema', + dryRun: false, + defaultLibrary: undefined, + pushed: false, + githubOutput: undefined, + noAnalyticsDiff: false, + tooManyFiles: false, +}; + +/** + * Builds paired Octokit mocks for publishGenerateResult. + * + * @param params - List/paginate fixtures. + * @param params.schemaPulls - Open schema PRs. + * @param params.comments - Client PR comments. + * @returns Client and schema Octokit plus spies. + */ +function mockOctokits(params: { + schemaPulls: Record[]; + comments: { id: number; body: string }[]; +}): { + client: Octokit; + schema: Octokit; + updateComment: jest.Mock; + createComment: jest.Mock; +} { + const updateComment = jest.fn().mockResolvedValue({}); + const createComment = jest.fn().mockResolvedValue({}); + const paginate = jest.fn().mockResolvedValue(params.comments); + const client = { + paginate, + issues: { updateComment, createComment }, + } as unknown as Octokit; + const schema = { + pulls: { + list: jest.fn().mockResolvedValue({ data: params.schemaPulls }), + }, + } as unknown as Octokit; + return { client, schema, updateComment, createComment }; +} + +describe('publishGenerateResult', () => { + it('edits an existing proposal when propose has no schema PR and no changes', async () => { + const { client, schema, updateComment, createComment } = mockOctokits({ + schemaPulls: [], + comments: [{ id: 77, body: `${PROPOSAL_MARKER}\nold` }], + }); + + await publishGenerateResult({ + args: ARGS, + summary: SUMMARY, + client, + schema, + }); + + expect(updateComment).toHaveBeenCalledWith( + expect.objectContaining({ + [COMMENT_ID]: 77, + body: expect.stringContaining('no longer has analytics changes'), + }), + ); + expect(createComment).not.toHaveBeenCalled(); + }); + + it('does not post when propose has no schema PR, no changes, and no proposal comment', async () => { + const { client, schema, updateComment, createComment } = mockOctokits({ + schemaPulls: [], + comments: [], + }); + + await publishGenerateResult({ + args: ARGS, + summary: SUMMARY, + client, + schema, + }); + + expect(updateComment).not.toHaveBeenCalled(); + expect(createComment).not.toHaveBeenCalled(); + }); + + it('upserts the sticky stale comment and does not post a second thread', async () => { + const { client, schema, updateComment, createComment } = mockOctokits({ + schemaPulls: [ + { + number: 9, + [HTML_URL]: 'https://github.com/Consensys/segment-schema/pull/9', + body: 'body', + draft: true, + }, + ], + comments: [{ id: 77, body: `${PROPOSAL_MARKER}\nold` }], + }); + + await publishGenerateResult({ + args: ARGS, + summary: SUMMARY, + client, + schema, + }); + + expect(updateComment).toHaveBeenCalledWith( + expect.objectContaining({ + [COMMENT_ID]: 77, + body: expect.stringContaining(PROPOSAL_MARKER), + }), + ); + expect(updateComment.mock.calls[0]?.[0].body).toContain( + 'https://github.com/Consensys/segment-schema/pull/9', + ); + expect(createComment).not.toHaveBeenCalled(); + }); + + it('edits the sticky comment when the file cap is exceeded', async () => { + const { client, schema, updateComment, createComment } = mockOctokits({ + schemaPulls: [], + comments: [{ id: 77, body: `${PROPOSAL_MARKER}\nold` }], + }); + + await publishTooManyFiles({ + args: { ...ARGS, tooManyFiles: true }, + client, + schema, + }); + + expect(updateComment).toHaveBeenCalledWith( + expect.objectContaining({ + [COMMENT_ID]: 77, + body: expect.stringContaining('more than 150'), + }), + ); + expect(createComment).not.toHaveBeenCalled(); + }); +}); diff --git a/src/segment-schema-draft-pr/publish.ts b/src/segment-schema-draft-pr/publish.ts new file mode 100644 index 00000000..8ab0f164 --- /dev/null +++ b/src/segment-schema-draft-pr/publish.ts @@ -0,0 +1,268 @@ +import type { Octokit } from '@octokit/rest'; +import fs from 'fs/promises'; +import path from 'path'; + +import { botBranchName } from './config'; +import { MAX_CHANGED_TS_FILES } from './constants'; +import { emptySummary } from './generate'; +import { + closeSchemaPr, + createOctokitClients, + createSchemaPr, + findOpenSchemaPr, + loadPrTemplate, + parseRepo, + postComment, + resolveClientPr, + updateProposalCommentIfExists, + updateSchemaPrBody, + upsertProposalComment, +} from './github-api'; +import type { CliArgs } from './parse-args'; +import { + renderClosedSchemaComment, + renderCreatedComment, + renderDraftOpenComment, + renderGeneratedSchemaBlock, + renderListedOnlyComment, + renderNoLongerNeededComment, + renderProposalComment, + renderStaleComment, + renderTooManyFilesComment, + renderTooManyFilesStaleComment, + renderUpdatedComment, + renderWindowClosedComment, + spliceSchemaPrBody, +} from './pr-body'; +import type { GenerateSummary, SchemaPr } from './types'; + +/** + * Octokit-only phase: comments and schema PR create/update/close. + * + * @param args - CLI args. + */ +export async function runPublish(args: CliArgs): Promise { + if (args.dryRun) { + console.log('dry-run: skip GitHub writes'); + return; + } + + const clientRepo = parseRepo(args.clientRepository); + const schemaRepo = parseRepo(args.segmentSchemaRepository); + const { client, schema } = createOctokitClients( + args.githubToken, + args.segmentSchemaToken, + ); + const branch = botBranchName(args.platform, args.prNumber); + + if (args.mode === 'close') { + const pr = await resolveClientPr( + client, + clientRepo, + args.prNumber, + args.clientRepository, + ); + const schemaPr = await findOpenSchemaPr(schema, schemaRepo, branch); + if (schemaPr && !pr.merged) { + await postComment( + schema, + schemaRepo, + schemaPr.number, + renderClosedSchemaComment(), + ); + await closeSchemaPr(schema, schemaRepo, schemaPr.number); + await postComment( + client, + clientRepo, + args.prNumber, + renderWindowClosedComment(), + ); + } + return; + } + + if (args.tooManyFiles) { + await publishTooManyFiles({ args, client, schema }); + return; + } + + const summary = args.noAnalyticsDiff + ? emptySummary(branch) + : await readSummary(args.schema); + await publishGenerateResult({ + args, + summary, + client, + schema, + }); +} + +/** + * Upserts the sticky comment when the PR exceeds the TypeScript file cap. + * + * @param params - Parsed CLI args and Octokit clients. + * @param params.args - CLI args. + * @param params.client - Client-repo Octokit. + * @param params.schema - Schema-repo Octokit. + */ +export async function publishTooManyFiles(params: { + args: CliArgs; + client: Octokit; + schema: Octokit; +}): Promise { + const { args, client, schema } = params; + const clientRepo = parseRepo(args.clientRepository); + const schemaRepo = parseRepo(args.segmentSchemaRepository); + const branch = botBranchName(args.platform, args.prNumber); + const existing = await findOpenSchemaPr(schema, schemaRepo, branch); + const body = existing + ? renderTooManyFilesStaleComment(existing, MAX_CHANGED_TS_FILES) + : renderTooManyFilesComment(MAX_CHANGED_TS_FILES); + await upsertProposalComment(client, clientRepo, args.prNumber, body); +} + +/** + * Posts proposal/status comments and creates or updates the schema PR. + * + * @param params - Parsed CLI args, generate summary, and Octokit clients. + * @param params.args - CLI args. + * @param params.summary - Generate summary from the schema working tree. + * @param params.client - Client-repo Octokit. + * @param params.schema - Schema-repo Octokit. + */ +export async function publishGenerateResult(params: { + args: CliArgs; + summary: GenerateSummary; + client: Octokit; + schema: Octokit; +}): Promise { + const { args, summary, client, schema } = params; + const clientRepo = parseRepo(args.clientRepository); + const schemaRepo = parseRepo(args.segmentSchemaRepository); + const branch = botBranchName(args.platform, args.prNumber); + const existing = await findOpenSchemaPr(schema, schemaRepo, branch); + const clientPrUrl = `https://github.com/${args.clientRepository}/pull/${args.prNumber}`; + + if (args.mode === 'propose' && !existing) { + if (!summary.hasChanges) { + await updateProposalCommentIfExists( + client, + clientRepo, + args.prNumber, + renderNoLongerNeededComment(), + ); + return; + } + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderProposalComment(summary.changeset, summary.intendedFiles), + ); + return; + } + + if (args.mode === 'propose' && existing && !summary.hasChanges) { + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderStaleComment(existing), + ); + return; + } + + if (args.mode === 'create' && !args.pushed) { + if (!summary.hasChanges) { + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderNoLongerNeededComment(), + ); + return; + } + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderListedOnlyComment(summary.changeset), + ); + return; + } + + if (!args.pushed && args.mode === 'propose' && existing) { + const template = await loadPrTemplate(schema, schemaRepo); + const generated = renderGeneratedSchemaBlock( + clientPrUrl, + summary.changeset, + summary.intendedFiles, + ); + const body = spliceSchemaPrBody(existing.body, template, generated); + await updateSchemaPrBody(schema, schemaRepo, existing.number, body); + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderDraftOpenComment(existing), + ); + return; + } + + const template = await loadPrTemplate(schema, schemaRepo); + const generated = renderGeneratedSchemaBlock( + clientPrUrl, + summary.changeset, + summary.intendedFiles, + ); + const title = `Draft schema for ${args.platform} PR #${args.prNumber}`; + + let schemaPr: SchemaPr; + if (existing) { + const body = spliceSchemaPrBody(existing.body, template, generated); + await updateSchemaPrBody(schema, schemaRepo, existing.number, body); + schemaPr = { ...existing, body }; + await postComment( + client, + clientRepo, + args.prNumber, + renderUpdatedComment(schemaPr), + ); + } else { + const body = spliceSchemaPrBody('', template, generated); + schemaPr = await createSchemaPr(schema, schemaRepo, { + branch, + base: args.segmentSchemaBase, + title, + body, + }); + await postComment( + client, + clientRepo, + args.prNumber, + renderCreatedComment(schemaPr), + ); + } + + await upsertProposalComment( + client, + clientRepo, + args.prNumber, + renderDraftOpenComment(schemaPr), + ); +} + +/** + * Reads the generate summary written next to the schema checkout. + * + * @param schemaDir - Schema working tree. + * @returns Parsed summary. + */ +async function readSummary(schemaDir: string): Promise { + const filePath = path.join( + schemaDir, + '.segment-schema-draft-pr-summary.json', + ); + const text = await fs.readFile(filePath, 'utf8'); + return JSON.parse(text) as GenerateSummary; +} diff --git a/src/segment-schema-draft-pr/schema-index.ts b/src/segment-schema-draft-pr/schema-index.ts new file mode 100644 index 00000000..f4fd0677 --- /dev/null +++ b/src/segment-schema-draft-pr/schema-index.ts @@ -0,0 +1,228 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { parseDocument } from 'yaml'; + +export type IndexedEvent = { + name: string; + filePath: string; + library: string; + defaultProps: string[]; + propertyKeys: Set; + propertyTypes: Map; +}; + +export type SchemaIndex = { + eventsByName: Map; + propertiesByLibrary: Map>; +}; + +/** + * Indexes event YAML and property libraries under a schema checkout. + * + * @param schemaDir - Root of Consensys/segment-schema. + * @returns Events by display name and default_props property sets. + */ +export async function buildSchemaIndex( + schemaDir: string, +): Promise { + const eventsByName = new Map(); + const propertiesByLibrary = new Map>(); + + const eventsRoot = path.join(schemaDir, 'libraries', 'events'); + const eventFiles = await listYamlFiles(eventsRoot); + + for (const filePath of eventFiles) { + const text = await fs.readFile(filePath, 'utf8'); + const doc = parseDocument(text); + const name = asString(doc.get('name')); + if (!name) { + continue; + } + + const relative = path.relative(schemaDir, filePath); + const library = libraryFromEventPath(relative); + const defaultProps = asStringSeq(doc.get('default_props')); + const properties = doc.get('properties'); + const propertyKeys = new Set(); + const propertyTypes = new Map(); + + if (isYamlMap(properties)) { + for (const item of properties.items) { + const key = asString(item.key); + if (!key) { + continue; + } + propertyKeys.add(key); + const typeValue = isYamlMap(item.value) + ? asString(item.value.get('type')) + : undefined; + if (typeValue) { + propertyTypes.set(key, typeValue); + } + } + } + + eventsByName.set(name, { + name, + filePath, + library, + defaultProps, + propertyKeys, + propertyTypes, + }); + } + + const propertiesRoot = path.join(schemaDir, 'libraries', 'properties'); + const propertyFiles = await listYamlFiles(propertiesRoot); + for (const filePath of propertyFiles) { + const text = await fs.readFile(filePath, 'utf8'); + const doc = parseDocument(text); + const library = path.basename(filePath, path.extname(filePath)); + const keys = new Set(); + const properties = doc.get('properties'); + if (isYamlMap(properties)) { + for (const item of properties.items) { + const key = asString(item.key); + if (key) { + keys.add(key); + } + } + } + propertiesByLibrary.set(library, keys); + } + + return { eventsByName, propertiesByLibrary }; +} + +/** + * Property keys already supplied by an event's default_props libraries. + * + * @param index - Schema index. + * @param defaultProps - Library ids listed on the event. + * @returns Union of those libraries' keys. + */ +export function defaultPropKeys( + index: SchemaIndex, + defaultProps: string[], +): Set { + const keys = new Set(); + for (const library of defaultProps) { + const libraryKeys = index.propertiesByLibrary.get(library); + if (!libraryKeys) { + continue; + } + for (const key of libraryKeys) { + keys.add(key); + } + } + return keys; +} + +/** + * Recursively lists YAML files under a directory. + * + * @param dir - Directory to walk. + * @returns Absolute file paths. + */ +async function listYamlFiles(dir: string): Promise { + const results: string[] = []; + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch (error) { + const { code } = error as { code?: string }; + if (code === 'ENOENT') { + return results; + } + throw error; + } + + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...(await listYamlFiles(full))); + continue; + } + if ( + entry.isFile() && + (entry.name.endsWith('.yaml') || entry.name.endsWith('.yml')) + ) { + results.push(full); + } + } + + return results; +} + +/** + * Reads the library folder from `libraries/events//...`. + * + * @param relativePath - Path from the schema root. + * @returns Library id. + */ +function libraryFromEventPath(relativePath: string): string { + const parts = relativePath.split(path.sep); + const eventsIndex = parts.indexOf('events'); + if (eventsIndex >= 0) { + return parts[eventsIndex + 1] ?? ''; + } + return ''; +} + +/** + * Narrows a YAML node to a map. + * + * @param value - Parsed YAML node. + * @returns Whether it is a YAML map. + */ +function isYamlMap(value: unknown): value is { + items: { key: unknown; value: unknown }[]; + get: (key: string) => unknown; +} { + return ( + typeof value === 'object' && + value !== null && + 'items' in value && + Array.isArray((value as { items: unknown }).items) + ); +} + +/** + * Reads a YAML scalar as a string. + * + * @param value - Parsed node. + * @returns String, or undefined. + */ +function asString(value: unknown): string | undefined { + if (typeof value === 'string') { + return value; + } + if (value && typeof value === 'object' && 'value' in value) { + const inner = (value as { value: unknown }).value; + if (typeof inner === 'string') { + return inner; + } + } + return undefined; +} + +/** + * Reads a YAML sequence of strings. + * + * @param value - Parsed node. + * @returns String items. + */ +function asStringSeq(value: unknown): string[] { + if (!value || typeof value !== 'object' || !('items' in value)) { + return []; + } + const { items } = value as { items: unknown[] }; + const result: string[] = []; + for (const item of items) { + const text = asString(item); + if (text) { + result.push(text); + } + } + return result; +} diff --git a/src/segment-schema-draft-pr/types.ts b/src/segment-schema-draft-pr/types.ts new file mode 100644 index 00000000..a4f18412 --- /dev/null +++ b/src/segment-schema-draft-pr/types.ts @@ -0,0 +1,105 @@ +export type Platform = 'mobile' | 'extension'; + +export type Mode = 'propose' | 'create' | 'close'; + +export type Phase = 'generate' | 'publish'; + +export type PropertyType = 'string' | 'number' | 'boolean' | 'array'; + +export type EventCatalog = Map; + +export type PropertyBag = { + key: string; + type: PropertyType; +}; + +export type UnresolvedProperty = { + eventName: string; + key: string; + file: string; +}; + +export type EventModel = { + properties: Map; + unresolved: UnresolvedProperty[]; +}; + +export type AnalyticsModel = { + catalog: EventCatalog; + events: Map; +}; + +export type EventChange = { + enumKey: string; + eventName: string; +}; + +export type RenameChange = { + enumKey: string; + fromName: string; + toName: string; +}; + +export type PropertyChange = { + eventName: string; + key: string; + type: PropertyType; +}; + +export type RemovedProperty = { + eventName: string; + key: string; +}; + +export type AnalyticsChangeSet = { + eventsAdded: EventChange[]; + eventsRemoved: EventChange[]; + eventsRenamed: RenameChange[]; + propertiesAdded: PropertyChange[]; + propertiesRemoved: RemovedProperty[]; + typeChanges: PropertyChange[]; + unresolved: UnresolvedProperty[]; +}; + +export type IntendedFileChange = { + path: string; + kind: 'create' | 'update'; + eventName: string; +}; + +export type GenerateSummary = { + hasChanges: boolean; + branch: string; + changeset: AnalyticsChangeSet; + intendedFiles: IntendedFileChange[]; + schemaPrNumber: number | null; + schemaPrUrl: string | null; +}; + +export type ResolvedClientPr = { + number: number; + baseSha: string; + headSha: string; + authorLogin: string; + isOpen: boolean; + merged: boolean; + isFork: boolean; + hasOptOutLabel: boolean; + headRepoFullName: string; +}; + +export type SchemaPr = { + number: number; + htmlUrl: string; + body: string; + draft: boolean; +}; + +export type PlatformConfig = { + catalogFile: string; + enumName: string; + eventRefPrefix: string; + trackingPlan: string; + globals: string; + defaultLibrary: string; +}; diff --git a/yarn.lock b/yarn.lock index 61b5138a..ccaf96e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1065,6 +1065,7 @@ __metadata: ts-node: "npm:^10.9.1" typescript: "npm:^5.1.3" unzipper: "npm:^0.12.3" + yaml: "npm:^2.9.0" languageName: unknown linkType: soft @@ -8544,6 +8545,15 @@ __metadata: languageName: node linkType: hard +"yaml@npm:^2.9.0": + version: 2.9.0 + resolution: "yaml@npm:2.9.0" + bin: + yaml: bin.mjs + checksum: 10/9a95e8e08651c3d292ab6a5befeb5f57b76801caa097c75bb45c9a70ce19c1b11f57e87a6ef84a579ea070ed2c2c8ac541c88c0ae684d544d5f42c7e77d11b7b + languageName: node + linkType: hard + "yargs-parser@npm:^20.2.2": version: 20.2.7 resolution: "yargs-parser@npm:20.2.7"