diff --git a/.changeset/silver-lands-cut.md b/.changeset/silver-lands-cut.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/silver-lands-cut.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f76a5e1564..916932893e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -504,6 +504,15 @@ jobs: sudo apt-get install -y xvfb fi + - name: Configure test user cleanup + run: echo "INTEGRATION_TEST_RUN_KEY=${RUN_ID}-${RUN_ATTEMPT}-${TEST_NAME}-${TEST_PROJECT}-${NEXT_VERSION:-default}" >> "$GITHUB_ENV" + env: + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + TEST_NAME: ${{ matrix.test-name }} + TEST_PROJECT: ${{ matrix.test-project }} + NEXT_VERSION: ${{ matrix.next-version }} + - name: Run Integration Tests id: integration-tests timeout-minutes: 25 @@ -525,6 +534,14 @@ jobs: NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + - name: Delete integration-test users + if: ${{ always() && steps.integration-tests.outcome != 'skipped' }} + timeout-minutes: 4 + run: pnpm test:integration:cleanup + env: + INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} + NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem + - name: Sanitize artifact name if: ${{ cancelled() || failure() }} id: sanitize diff --git a/integration/cleanup/cleanup.setup.ts b/integration/cleanup/cleanup.setup.ts index 5bd5a194265..5340fb34ed7 100644 --- a/integration/cleanup/cleanup.setup.ts +++ b/integration/cleanup/cleanup.setup.ts @@ -5,8 +5,11 @@ import { isStaging } from '@clerk/shared/utils'; import { test as setup } from '@playwright/test'; import { appConfigs } from '../presets/'; +import { findE2ERunUsers, getE2ERunMarker } from '../testUtils/e2eRun'; +import { withRetry } from '../testUtils/retryableClerkClient'; setup('cleanup instances ', async () => { + const runMarker = getE2ERunMarker(); const entries = Array.from(appConfigs.secrets.instanceKeys.values()) .map(({ pk, sk }) => { const secretKey = sk; @@ -32,6 +35,9 @@ setup('cleanup instances ', async () => { }> = []; console.log('🧹 Starting E2E Test Cleanup Process...\n'); + if (runMarker) { + console.log(`Cleaning users for run marker ${runMarker}\n`); + } for (const entry of entries) { const instanceSummary = { @@ -43,29 +49,32 @@ setup('cleanup instances ', async () => { }; try { - const clerkClient = createClerkClient({ secretKey: entry.secretKey, apiUrl: entry.apiUrl }); + const clerkClient = withRetry(createClerkClient({ secretKey: entry.secretKey, apiUrl: entry.apiUrl })); // Get users with error handling let users: any[] = []; try { - const { data: usersWithEmail } = await clerkClient.users.getUserList({ - orderBy: '-created_at', - query: 'clerkcookie', - limit: 500, - }); - - const { data: usersWithPhoneNumber } = await clerkClient.users.getUserList({ - orderBy: '-created_at', - query: '55501', - limit: 500, - }); - - // Deduplicate users by ID - const allUsersMap = new Map(); - [...usersWithEmail, ...usersWithPhoneNumber].forEach(user => { - allUsersMap.set(user.id, user); - }); - users = Array.from(allUsersMap.values()); + if (runMarker) { + users = await findE2ERunUsers(clerkClient, runMarker); + } else { + const { data: usersWithEmail } = await clerkClient.users.getUserList({ + orderBy: '-created_at', + query: 'clerkcookie', + limit: 500, + }); + + const { data: usersWithPhoneNumber } = await clerkClient.users.getUserList({ + orderBy: '-created_at', + query: '55501', + limit: 500, + }); + + const allUsersMap = new Map(); + [...usersWithEmail, ...usersWithPhoneNumber].forEach(user => { + allUsersMap.set(user.id, user); + }); + users = Array.from(allUsersMap.values()); + } } catch (error) { instanceSummary.errors.push(`Failed to get users: ${error.message}`); console.error(`Error getting users for ${entry.instanceName}:`, error); @@ -75,10 +84,14 @@ setup('cleanup instances ', async () => { // Get organizations with error handling let orgs: any[] = []; try { - const { data: orgsData } = await clerkClient.organizations.getOrganizationList({ - limit: 500, - }); - orgs = orgsData; + if (runMarker) { + orgs = []; + } else { + const { data: orgsData } = await clerkClient.organizations.getOrganizationList({ + limit: 500, + }); + orgs = orgsData; + } } catch (error) { // Treat 404 (not found) and 403 (forbidden) as "no orgs" // 404 = no organizations exist, 403 = no permission to access organizations @@ -91,8 +104,11 @@ setup('cleanup instances ', async () => { } } - const usersToDelete = batchElements(skipObjectsThatWereCreatedWithinTheLast10Minutes(users), 5); - const orgsToDelete = batchElements(skipObjectsThatWereCreatedWithinTheLast10Minutes(orgs), 5); + const usersToDelete = batchElements( + runMarker ? users : skipObjectsThatWereCreatedWithinTheLast10Minutes(users), + 5, + ); + const orgsToDelete = batchElements(runMarker ? [] : skipObjectsThatWereCreatedWithinTheLast10Minutes(orgs), 5); // Delete users with tracking for (const batch of usersToDelete) { @@ -142,6 +158,13 @@ setup('cleanup instances ', async () => { await new Promise(r => setTimeout(r, 1000)); } + if (runMarker) { + const remainingUsers = await findE2ERunUsers(clerkClient, runMarker); + if (remainingUsers.length > 0) { + instanceSummary.errors.push(`Users remain after cleanup: ${remainingUsers.map(user => user.id).join(', ')}`); + } + } + // Report instance results const maskedKey = entry.secretKey.replace(/(sk_(test|live)_)(.+)(...)/, '$1***$4'); if (instanceSummary.usersDeleted > 0 || instanceSummary.orgsDeleted > 0) { diff --git a/integration/playwright.cleanup.config.ts b/integration/playwright.cleanup.config.ts index 82ee4ab4b80..4f35a4e1248 100644 --- a/integration/playwright.cleanup.config.ts +++ b/integration/playwright.cleanup.config.ts @@ -9,6 +9,7 @@ config({ path: path.resolve(__dirname, '.env.local') }); export default defineConfig({ ...common, testDir: './cleanup', + retries: 0, projects: [ { name: 'setup', diff --git a/integration/testUtils/e2eRun.ts b/integration/testUtils/e2eRun.ts new file mode 100644 index 00000000000..e09c280ef45 --- /dev/null +++ b/integration/testUtils/e2eRun.ts @@ -0,0 +1,42 @@ +import { createHash } from 'node:crypto'; + +import type { ClerkClient, User } from '@clerk/backend'; + +type E2EUserRecord = { + username: string | null; + emailAddresses: Array<{ emailAddress: string }>; + privateMetadata: Record; +}; + +export const getE2ERunMarker = (runKey = process.env.INTEGRATION_TEST_RUN_KEY): string | undefined => { + if (!runKey) { + return; + } + + const digest = createHash('sha256').update(runKey).digest('hex').slice(0, 20); + return `e2e_${digest}`; +}; + +export const userMatchesE2ERun = (user: E2EUserRecord, marker: string): boolean => + Boolean( + user.username?.includes(marker) || + user.emailAddresses.some(email => email.emailAddress.includes(marker)) || + user.privateMetadata.e2eRunMarker === marker, + ); + +export const findE2ERunUsers = async (clerkClient: ClerkClient, marker: string): Promise => { + const usersById = new Map(); + let offset = 0; + + while (true) { + const { data } = await clerkClient.users.getUserList({ query: marker, limit: 100, offset }); + data.filter(user => userMatchesE2ERun(user, marker)).forEach(user => usersById.set(user.id, user)); + + if (data.length < 100) { + break; + } + offset += data.length; + } + + return Array.from(usersById.values()); +}; diff --git a/integration/testUtils/usersService.ts b/integration/testUtils/usersService.ts index e055dd68cc8..d24973ed4bb 100644 --- a/integration/testUtils/usersService.ts +++ b/integration/testUtils/usersService.ts @@ -3,6 +3,7 @@ import { faker } from '@faker-js/faker'; import type { TestInfo } from '@playwright/test'; import { fakerPassword, hash } from '../models/helpers'; +import { getE2ERunMarker } from './e2eRun'; async function withErrorLogging(operation: string, fn: () => Promise): Promise { try { @@ -129,24 +130,29 @@ export const createUserService = (clerkClient: ClerkClient) => { withUsername = false, } = options || {}; const randomHash = hash(); + const runMarker = getE2ERunMarker(); + const markedHash = runMarker ? `${runMarker}_${randomHash}` : randomHash; const email = fictionalEmail - ? `${randomHash}+clerk_test@clerkcookie.com` - : `clerkcookie+${randomHash}@mailsac.com`; + ? `${markedHash}+clerk_test@clerkcookie.com` + : `clerkcookie+${markedHash}@mailsac.com`; const phoneNumber = fakerPhoneNumber(); const { file, line, title, titlePath } = test.info(); + const fakeUserEmail = withEmail ? email : undefined; + const fakeUserPhoneNumber = withPhoneNumber ? phoneNumber : undefined; return { firstName: faker.person.firstName(), lastName: faker.person.lastName(), - email: withEmail ? email : undefined, - username: withUsername ? `${randomHash}_clerk_cookie` : undefined, + email: fakeUserEmail, + username: withUsername ? `${markedHash}_clerk_cookie` : undefined, password: withPassword ? fakerPassword() : undefined, - phoneNumber: withPhoneNumber ? phoneNumber : undefined, + phoneNumber: fakeUserPhoneNumber, privateMetadata: { title, titlePath, file, line, + ...(runMarker ? { e2eRunMarker: runMarker } : {}), }, deleteIfExists: () => self.deleteIfExists({ email, phoneNumber }), }; @@ -173,24 +179,35 @@ export const createUserService = (clerkClient: ClerkClient) => { return await self.createBapiUser(fakeUser); }, deleteIfExists: async (opts: { id?: string; email?: string; phoneNumber?: string }) => { - let id = opts.id; + const [usersByEmail, usersByPhoneNumber] = await Promise.all([ + opts.email + ? withErrorLogging('getUserList', () => + clerkClient.users.getUserList({ + emailAddress: [opts.email], + }), + ) + : undefined, + opts.phoneNumber + ? withErrorLogging('getUserList', () => + clerkClient.users.getUserList({ + phoneNumber: [opts.phoneNumber], + }), + ) + : undefined, + ]); - if (!id) { - const { data: users } = await withErrorLogging('getUserList', () => - clerkClient.users.getUserList({ - emailAddress: [opts.email], - phoneNumber: [opts.phoneNumber], - }), - ); - id = users[0]?.id; - } + const ids = new Set([ + ...(opts.id ? [opts.id] : []), + ...(usersByEmail?.data.map(user => user.id) ?? []), + ...(usersByPhoneNumber?.data.map(user => user.id) ?? []), + ]); - if (!id) { + if (ids.size === 0) { console.log(`User "${opts.email || opts.phoneNumber}" does not exist!`); return; } - await withErrorLogging('deleteUser', () => clerkClient.users.deleteUser(id)); + await Promise.all(Array.from(ids, id => withErrorLogging('deleteUser', () => clerkClient.users.deleteUser(id)))); }, getUser: async (opts: { id?: string; email?: string }) => { if (opts.id) { diff --git a/turbo.json b/turbo.json index 504ed7d4142..c93746b2ffb 100644 --- a/turbo.json +++ b/turbo.json @@ -34,6 +34,7 @@ "globalPassThroughEnv": [ "AWS_SECRET_KEY", "GITHUB_TOKEN", + "INTEGRATION_TEST_RUN_KEY", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG", "VERCEL_AUTOMATION_BYPASS_SECRET",