From 2e2404018b13eb2bca12949e5315c648c2cf3355 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 16 May 2026 22:56:38 -0400 Subject: [PATCH 1/8] chore(plans): mark cutover-prep in-progress --- plans/cutover-prep.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/cutover-prep.md b/plans/cutover-prep.md index 9ee70c6..0fc4a77 100644 --- a/plans/cutover-prep.md +++ b/plans/cutover-prep.md @@ -1,5 +1,5 @@ --- -status: planned +status: in-progress depends: - workspace - test-harness From 4e872eebd82ca08453ac6576b8ff8ee8e81b8bc8 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 16 May 2026 23:05:52 -0400 Subject: [PATCH 2/8] feat(cutover-prep): reconcile, dry-run, and T+90 mailout scripts Adds three cutover-prep scripts in apps/api/scripts/: - reconcile.ts walks the public people sheet + private store and flags orphans (both directions), inconsistent newsletter state, and drained LegacyPasswordCredentials. --fix mode regenerates missing unsubscribe tokens and deletes drained credentials. Supersedes the narrower reconcile-private-store.ts (whose scope is fully absorbed here). - cutover-dry-run.ts orchestrates an end-to-end rehearsal: imports a mysqldump, compares per-table row counts vs. per-sheet imported counts, and optionally smokes a staging target (10 random Persons/Projects, legacy-id redirects, SAML metadata, OAuth start, health probes). - cutover-mailout.ts collects unclaimed Persons with valid emails and sends a single reminder via Resend. --dry-run is mandatory for CI. Tests cover each script against fixtures: orphan flagging, newsletter repair, drained-credential cleanup, fixture row-count parsing, recipient selection, and HTML escaping in the email body. --- apps/api/package.json | 4 +- apps/api/scripts/cutover-dry-run.ts | 473 ++++++++++++++++++++ apps/api/scripts/cutover-mailout.ts | 329 ++++++++++++++ apps/api/scripts/reconcile-private-store.ts | 147 ------ apps/api/scripts/reconcile.ts | 339 ++++++++++++++ apps/api/tests/cutover-dry-run.test.ts | 180 ++++++++ apps/api/tests/cutover-mailout.test.ts | 205 +++++++++ apps/api/tests/reconcile.test.ts | 244 ++++++++++ 8 files changed, 1773 insertions(+), 148 deletions(-) create mode 100644 apps/api/scripts/cutover-dry-run.ts create mode 100644 apps/api/scripts/cutover-mailout.ts delete mode 100644 apps/api/scripts/reconcile-private-store.ts create mode 100644 apps/api/scripts/reconcile.ts create mode 100644 apps/api/tests/cutover-dry-run.test.ts create mode 100644 apps/api/tests/cutover-mailout.test.ts create mode 100644 apps/api/tests/reconcile.test.ts diff --git a/apps/api/package.json b/apps/api/package.json index 190a5d6..a3870d0 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -13,7 +13,9 @@ "script:scrub-data": "tsx scripts/scrub-data.ts", "script:setup-dev-data": "tsx scripts/setup-dev-data.ts", "script:import-laddr": "tsx scripts/import-laddr.ts", - "script:reconcile-private-store": "tsx scripts/reconcile-private-store.ts" + "script:reconcile": "tsx scripts/reconcile.ts", + "script:cutover-dry-run": "tsx scripts/cutover-dry-run.ts", + "script:cutover-mailout": "tsx scripts/cutover-mailout.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1048.0", diff --git a/apps/api/scripts/cutover-dry-run.ts b/apps/api/scripts/cutover-dry-run.ts new file mode 100644 index 0000000..6ee545d --- /dev/null +++ b/apps/api/scripts/cutover-dry-run.ts @@ -0,0 +1,473 @@ +/** + * cutover-dry-run.ts — End-to-end staging rehearsal + * + * Walks the full cutover pipeline against a non-production target so the team + * can rehearse before T-0. Stages, in order: + * + * 1. Run the importer (apps/api/scripts/import-laddr/importer.ts) against a + * mysqldump → fresh data-repo + private store. + * 2. Optionally hit a live target (`--target=`) to smoke-test: + * - 10 random Persons resolve at /api/people/:slug + * - 10 random Projects resolve at /api/projects/:slug + * - legacy redirect for /projects?ID= returns 301 + * - SAML metadata is reachable at /api/saml/idp/metadata + * - GitHub OAuth start endpoint redirects (302) + * 3. Compare importer's per-sheet counts vs. the raw mysqldump's row counts. + * + * Output: a JSON report with per-stage results + warnings + smoke-check timings. + * Exit 0 if every stage passed; non-zero with details otherwise. + * + * Usage: + * npm run -w apps/api script:cutover-dry-run -- \ + * --sql=./scratch/laddr.sql \ + * --data-repo=./scratch/dry-run-data \ + * --private-store=./scratch/dry-run-private \ + * [--target=https://codeforphilly-rewrite-staging.k8s.phl.io] \ + * [--sample=10] \ + * [--json=./scratch/dry-run-report.json] + * + * `--target` is optional: when omitted the script runs steps 1 + 3 only + * (useful before a staging cluster is up). + */ +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { FilesystemPrivateStore } from '../src/store/private/filesystem.js'; +import { importLaddr, type ImportReport } from './import-laddr/importer.js'; +import { openPublicStore } from '../src/store/public.js'; + +// --------------------------------------------------------------------------- +// Report types — exported for tests +// --------------------------------------------------------------------------- + +export interface SmokeCheckResult { + readonly name: string; + readonly url: string; + readonly status: number | null; + readonly ok: boolean; + readonly durationMs: number; + readonly note?: string; +} + +export interface CountDiff { + readonly sheet: string; + readonly sourceRows: number; + readonly importedRecords: number; + /** True when sourceRows === importedRecords. */ + readonly matched: boolean; +} + +export interface DryRunReport { + readonly runAt: string; + readonly target: string | null; + readonly importReport: Pick; + readonly countDiffs: ReadonlyArray; + readonly smokeChecks: ReadonlyArray; + readonly stages: { + readonly import: boolean; + readonly countDiff: boolean; + readonly smoke: boolean; + }; + readonly passed: boolean; +} + +// --------------------------------------------------------------------------- +// mysqldump row-count parser +// +// We only need row counts per table — not full parsing. Sum the number of +// row tuples across all `INSERT INTO \`\` ... VALUES (...),(...);` +// statements. This is far cheaper than re-parsing every value. +// --------------------------------------------------------------------------- + +/** + * Map laddr table name → v1 sheet name. Mirrors translators.ts. Production + * laddr dumps vary between CamelCase (older Emergence schema) and snake_case + * (newer), so we accept either. Tables not listed here surface as + * `unmapped:
` in the count diff so we can spot drift in the dump shape. + */ +const TABLE_TO_SHEET: ReadonlyMap = new Map([ + ['People', 'people'], + ['people', 'people'], + ['Projects', 'projects'], + ['projects', 'projects'], + ['ProjectMembers', 'project-memberships'], + ['project_members', 'project-memberships'], + ['ProjectUpdates', 'project-updates'], + ['project_updates', 'project-updates'], + ['ProjectBuzz', 'project-buzz'], + ['project_buzz', 'project-buzz'], + ['Tags', 'tags'], + ['tags', 'tags'], + ['TagAssignments', 'tag-assignments'], + ['tag_assignments', 'tag-assignments'], + ['tag_items', 'tag-assignments'], +]); + +/** Tables we know exist in laddr dumps but intentionally don't migrate. */ +const IGNORED_TABLES: ReadonlySet = new Set([ + 'member_checkins', + 'sessions', + '_history_People', + '_history_Projects', +]); + +/** + * Count rows in INSERT statements per table. Cheap streaming-friendly parse: + * walks the dump linewise; each `INSERT INTO \`Table\`` line contributes one + * statement whose value-tuples we count via a one-pass parenthesis depth + * tracker that respects quoted strings. + */ +export function countRowsByTable(sql: string): Map { + const result = new Map(); + const insertRe = /^INSERT INTO `([^`]+)`/m; + // Split statements on `;\n` boundaries. Simple but adequate for our dumps. + const statements = sql.split(/;\s*\n/); + for (const stmt of statements) { + const m = stmt.match(insertRe); + if (!m || m[1] === undefined) continue; + const table = m[1]; + const tuples = countValueTuples(stmt); + result.set(table, (result.get(table) ?? 0) + tuples); + } + return result; +} + +function countValueTuples(stmt: string): number { + const valuesIdx = stmt.indexOf('VALUES'); + if (valuesIdx === -1) return 0; + const tail = stmt.slice(valuesIdx + 'VALUES'.length); + + let count = 0; + let depth = 0; + let inStr = false; + let escape = false; + + for (let i = 0; i < tail.length; i++) { + const ch = tail[i]; + if (escape) { + escape = false; + continue; + } + if (ch === '\\') { + escape = true; + continue; + } + if (inStr) { + if (ch === "'") inStr = false; + continue; + } + if (ch === "'") { + inStr = true; + continue; + } + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + if (depth === 0) count++; + } + } + return count; +} + +// --------------------------------------------------------------------------- +// Smoke checks against a live target +// --------------------------------------------------------------------------- + +interface SmokeTarget { + readonly url: string; + readonly samplePeople: ReadonlyArray; + readonly samplePeopleLegacyIds: ReadonlyArray; + readonly sampleProjects: ReadonlyArray; + readonly sampleProjectLegacyIds: ReadonlyArray; +} + +async function timedFetch( + name: string, + url: string, + init?: RequestInit, +): Promise { + const started = Date.now(); + try { + const res = await fetch(url, { ...init, redirect: 'manual' }); + return { + name, + url, + status: res.status, + // 2xx is OK; 3xx is OK (legacy redirects); 401 on auth-protected smoke + // endpoints isn't a fail by itself, but we don't probe any here. + ok: res.status >= 200 && res.status < 400, + durationMs: Date.now() - started, + }; + } catch (err) { + return { + name, + url, + status: null, + ok: false, + durationMs: Date.now() - started, + note: err instanceof Error ? err.message : String(err), + }; + } +} + +export async function runSmokeChecks(target: SmokeTarget): Promise { + const base = target.url.replace(/\/$/, ''); + const results: SmokeCheckResult[] = []; + + for (const slug of target.samplePeople) { + results.push(await timedFetch(`person:${slug}`, `${base}/api/people/${slug}`)); + } + for (const slug of target.sampleProjects) { + results.push(await timedFetch(`project:${slug}`, `${base}/api/projects/${slug}`)); + } + for (const legacyId of target.sampleProjectLegacyIds) { + results.push( + await timedFetch(`project-legacy:${legacyId}`, `${base}/projects?ID=${legacyId}`), + ); + } + for (const legacyId of target.samplePeopleLegacyIds) { + results.push( + await timedFetch(`person-legacy:${legacyId}`, `${base}/people/${legacyId}`), + ); + } + results.push(await timedFetch('saml-metadata', `${base}/api/saml/idp/metadata`)); + results.push(await timedFetch('oauth-start', `${base}/api/auth/github/start`)); + results.push(await timedFetch('health', `${base}/api/health`)); + results.push(await timedFetch('health-ready', `${base}/api/health/ready`)); + + return results; +} + +// --------------------------------------------------------------------------- +// Sample selection +// --------------------------------------------------------------------------- + +/** Pick at most `n` items deterministically by hashing each one against seed. */ +export function deterministicSample(items: ReadonlyArray, n: number, seed: string): T[] { + if (items.length <= n) return [...items]; + const scored = items.map((item, idx) => ({ + item, + score: hashScore(`${seed}:${idx}:${String(item)}`), + })); + scored.sort((a, b) => a.score - b.score); + return scored.slice(0, n).map((s) => s.item); +} + +function hashScore(s: string): number { + let h = 2166136261; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return h >>> 0; +} + +// --------------------------------------------------------------------------- +// Orchestration +// --------------------------------------------------------------------------- + +export interface DryRunOptions { + readonly sql: string; + readonly dataRepo: string; + readonly privateStore: string; + readonly target: string | null; + readonly sampleSize: number; + readonly now?: string; + readonly seed?: string; +} + +export async function runDryRun(opts: DryRunOptions): Promise { + const runAt = opts.now ?? new Date().toISOString(); + const seed = opts.seed ?? runAt; + + const privateStore = new FilesystemPrivateStore({ + CFP_PRIVATE_STORAGE_PATH: opts.privateStore, + }); + await privateStore.load(); + + const importReport = await importLaddr({ + sql: opts.sql, + dataRepo: opts.dataRepo, + privateStore, + now: runAt, + }); + + const sql = await readFile(opts.sql, 'utf8'); + const tableCounts = countRowsByTable(sql); + const importsBySheet = importReport.entities; + + const seenSheets = new Set(); + const countDiffs: CountDiff[] = []; + for (const [table, sheet] of TABLE_TO_SHEET.entries()) { + const sourceRows = tableCounts.get(table) ?? 0; + if (sourceRows === 0) continue; + seenSheets.add(sheet); + const imported = importsBySheet[sheet]?.imported ?? 0; + countDiffs.push({ + sheet, + sourceRows, + importedRecords: imported, + matched: sourceRows === imported, + }); + } + // Surface unmapped tables that did appear in the dump. IGNORED_TABLES + // (e.g. checkins) are intentionally not migrated; everything else + // signals dump-shape drift that warrants attention. + for (const [table, sourceRows] of tableCounts) { + if (TABLE_TO_SHEET.has(table)) continue; + if (IGNORED_TABLES.has(table)) continue; + countDiffs.push({ + sheet: `unmapped:${table}`, + sourceRows, + importedRecords: 0, + matched: false, + }); + } + + let smokeChecks: SmokeCheckResult[] = []; + if (opts.target) { + const publicStore = await openPublicStore(opts.dataRepo); + const people = await publicStore.people.queryAll(); + const projects = await publicStore.projects.queryAll(); + const liveProjects = projects.filter((p) => !p.deletedAt); + const livePeople = people.filter((p) => !p.deletedAt); + + smokeChecks = await runSmokeChecks({ + url: opts.target, + samplePeople: deterministicSample( + livePeople.map((p) => p.slug), + opts.sampleSize, + `${seed}:people`, + ), + samplePeopleLegacyIds: deterministicSample( + livePeople + .map((p) => p.legacyId) + .filter((id): id is number => typeof id === 'number'), + opts.sampleSize, + `${seed}:people-legacy`, + ), + sampleProjects: deterministicSample( + liveProjects.map((p) => p.slug), + opts.sampleSize, + `${seed}:projects`, + ), + sampleProjectLegacyIds: deterministicSample( + liveProjects + .map((p) => p.legacyId) + .filter((id): id is number => typeof id === 'number'), + opts.sampleSize, + `${seed}:projects-legacy`, + ), + }); + } + + const importPassed = importReport.warnings.length === 0 + ? true + : importReport.warnings.every((w) => !w.toLowerCase().includes('error')); + const countDiffPassed = countDiffs.every((d) => d.matched); + const smokePassed = opts.target ? smokeChecks.every((c) => c.ok) : true; + + return { + runAt, + target: opts.target, + importReport: { + runAt: importReport.runAt, + sourceSha256: importReport.sourceSha256, + entities: importReport.entities, + warnings: importReport.warnings, + }, + countDiffs, + smokeChecks, + stages: { + import: importPassed, + countDiff: countDiffPassed, + smoke: smokePassed, + }, + passed: importPassed && countDiffPassed && smokePassed, + }; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +interface CliArgs { + readonly sql: string; + readonly dataRepo: string; + readonly privateStore: string; + readonly target: string | null; + readonly sampleSize: number; + readonly jsonPath: string | undefined; +} + +function parseArgs(argv: readonly string[]): CliArgs { + const opts: Record = {}; + for (const a of argv) { + if (!a.startsWith('--')) continue; + const eq = a.indexOf('='); + if (eq === -1) opts[a.slice(2)] = true; + else opts[a.slice(2, eq)] = a.slice(eq + 1); + } + const need = (k: string): string => { + const v = opts[k]; + if (typeof v !== 'string' || !v) { + process.stderr.write(`missing --${k}=\n`); + process.exit(2); + } + return v; + }; + const sampleRaw = opts['sample']; + const sampleSize = typeof sampleRaw === 'string' ? Number.parseInt(sampleRaw, 10) : 10; + return { + sql: resolve(need('sql')), + dataRepo: resolve(need('data-repo')), + privateStore: resolve(need('private-store')), + target: typeof opts['target'] === 'string' ? opts['target'] : null, + sampleSize: Number.isFinite(sampleSize) ? sampleSize : 10, + jsonPath: typeof opts['json'] === 'string' ? opts['json'] : undefined, + }; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + process.stderr.write(`[cutover-dry-run] sql=${args.sql}\n`); + process.stderr.write(`[cutover-dry-run] data-repo=${args.dataRepo}\n`); + process.stderr.write(`[cutover-dry-run] private-store=${args.privateStore}\n`); + process.stderr.write(`[cutover-dry-run] target=${args.target ?? '(none)'}\n`); + + const report = await runDryRun({ + sql: args.sql, + dataRepo: args.dataRepo, + privateStore: args.privateStore, + target: args.target, + sampleSize: args.sampleSize, + }); + + const json = `${JSON.stringify(report, null, 2)}\n`; + if (args.jsonPath) { + await writeFile(resolve(args.jsonPath), json, 'utf8'); + } else { + process.stdout.write(json); + } + + process.stderr.write( + `[cutover-dry-run] import=${report.stages.import} ` + + `countDiff=${report.stages.countDiff} ` + + `smoke=${report.stages.smoke} ` + + `passed=${report.passed}\n`, + ); + + process.exitCode = report.passed ? 0 : 1; +} + +const isMain = + process.argv[1] !== undefined && + import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')); + +if (isMain) { + main().catch((err: unknown) => { + process.stderr.write(`[cutover-dry-run] failed: ${String(err)}\n`); + process.exit(2); + }); +} diff --git a/apps/api/scripts/cutover-mailout.ts b/apps/api/scripts/cutover-mailout.ts new file mode 100644 index 0000000..e959cdd --- /dev/null +++ b/apps/api/scripts/cutover-mailout.ts @@ -0,0 +1,329 @@ +/** + * cutover-mailout.ts — T+90 reminder mailout to unclaimed Persons + * + * Pulls every public Person whose GitHub identity is still null and who has + * a matching PrivateProfile.email, and sends each a one-shot reminder asking + * them to sign in and claim their account. Run manually at T+90 per + * specs/behaviors/account-migration.md#cutover-window-policy. + * + * --dry-run prints the would-be send list and exits — no Resend calls, no + * disk writes. The CI test exercises only --dry-run. + * + * Usage: + * npm run -w apps/api script:cutover-mailout -- --dry-run + * npm run -w apps/api script:cutover-mailout -- --send --from=hello@codeforphilly.org + * + * Env: + * RESEND_API_KEY — required for actual sends (otherwise --send refuses) + * CFP_PUBLIC_URL — base URL used in the email body (defaults to + * https://codeforphilly.org) + * CFP_DATA_REPO_PATH + STORAGE_BACKEND + bucket envs — same shape as the API + */ +import { writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { openPublicStore } from '../src/store/public.js'; +import { + FilesystemPrivateStore, + S3PrivateStore, + type PrivateStore, +} from '../src/store/private/index.js'; + +// --------------------------------------------------------------------------- +// Exported types (tests rely on these) +// --------------------------------------------------------------------------- + +export interface MailoutRecipient { + readonly personId: string; + readonly slug: string; + readonly email: string; + readonly fullName: string | null; +} + +export interface MailoutReport { + readonly runAt: string; + readonly mode: 'dry-run' | 'send'; + readonly recipients: ReadonlyArray; + readonly skipped: ReadonlyArray<{ personId: string; reason: string }>; + readonly sent: number; + readonly failed: ReadonlyArray<{ personId: string; error: string }>; +} + +export interface MailoutOptions { + readonly publicStore: Awaited>; + readonly privateStore: PrivateStore; + readonly mode: 'dry-run' | 'send'; + readonly from?: string; + readonly publicUrl?: string; + readonly send?: (input: { to: string; from: string; subject: string; html: string; text: string }) => Promise; + readonly now?: string; +} + +// --------------------------------------------------------------------------- +// Recipient selection +// --------------------------------------------------------------------------- + +export async function collectRecipients( + publicStore: Awaited>, + privateStore: PrivateStore, +): Promise<{ recipients: MailoutRecipient[]; skipped: Array<{ personId: string; reason: string }> }> { + const people = await publicStore.people.queryAll(); + const recipients: MailoutRecipient[] = []; + const skipped: Array<{ personId: string; reason: string }> = []; + + for (const person of people) { + if (person.deletedAt) { + skipped.push({ personId: person.id, reason: 'deleted' }); + continue; + } + if (person.githubUserId) { + skipped.push({ personId: person.id, reason: 'github-linked' }); + continue; + } + const profile = await privateStore.getProfile(person.id); + if (!profile) { + skipped.push({ personId: person.id, reason: 'no-private-profile' }); + continue; + } + if (profile.email.endsWith('@example.invalid') || profile.email.endsWith('.invalid')) { + skipped.push({ personId: person.id, reason: 'invalid-email' }); + continue; + } + recipients.push({ + personId: person.id, + slug: person.slug, + email: profile.email, + fullName: person.fullName ?? null, + }); + } + return { recipients, skipped }; +} + +// --------------------------------------------------------------------------- +// Email body +// --------------------------------------------------------------------------- + +export function buildEmailBody( + recipient: MailoutRecipient, + publicUrl: string, +): { subject: string; html: string; text: string } { + const claimUrl = `${publicUrl.replace(/\/$/, '')}/account/sign-in`; + const name = recipient.fullName ?? recipient.slug; + const subject = 'Action needed: claim your Code for Philly account'; + const text = [ + `Hi ${name},`, + '', + `We migrated codeforphilly.org to a new platform a few months ago. ` + + `Your account at @${recipient.slug} is still waiting to be claimed.`, + '', + `Sign in with GitHub to claim it — your profile, projects, and Slack ` + + `identity all carry over: ${claimUrl}`, + '', + `If you don't recognize this account, you can ignore the email. ` + + `Accounts unclaimed for one year may be retired.`, + '', + '— Code for Philly', + ].join('\n'); + const html = + `

Hi ${escapeHtml(name)},

` + + `

We migrated codeforphilly.org to a new platform a few months ago. ` + + `Your account at @${escapeHtml(recipient.slug)} is still ` + + `waiting to be claimed.

` + + `

Sign in with GitHub to claim it — your ` + + `profile, projects, and Slack identity all carry over.

` + + `

If you don't recognize this account, you can ignore the email. ` + + `Accounts unclaimed for one year may be retired.

` + + `

— Code for Philly

`; + return { subject, html, text }; +} + +function escapeHtml(s: string): string { + return s + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +// --------------------------------------------------------------------------- +// Core run +// --------------------------------------------------------------------------- + +export async function runMailout(opts: MailoutOptions): Promise { + const runAt = opts.now ?? new Date().toISOString(); + const publicUrl = opts.publicUrl ?? 'https://codeforphilly.org'; + const from = opts.from ?? 'hello@codeforphilly.org'; + + const { recipients, skipped } = await collectRecipients(opts.publicStore, opts.privateStore); + + let sent = 0; + const failed: Array<{ personId: string; error: string }> = []; + + if (opts.mode === 'send') { + if (!opts.send) { + throw new Error('send mode requires a send() implementation'); + } + for (const recipient of recipients) { + const body = buildEmailBody(recipient, publicUrl); + try { + await opts.send({ + to: recipient.email, + from, + subject: body.subject, + html: body.html, + text: body.text, + }); + sent++; + } catch (err) { + failed.push({ + personId: recipient.personId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + + return { + runAt, + mode: opts.mode, + recipients, + skipped, + sent, + failed, + }; +} + +// --------------------------------------------------------------------------- +// Env wiring + Resend send +// --------------------------------------------------------------------------- + +function requireEnv(name: string): string { + const v = process.env[name]; + if (!v) throw new Error(`Missing required env var ${name}`); + return v; +} + +function buildPrivateStore(): PrivateStore { + const backend = requireEnv('STORAGE_BACKEND'); + if (backend === 's3') { + return new S3PrivateStore({ + S3_ENDPOINT: requireEnv('S3_ENDPOINT'), + S3_BUCKET: requireEnv('S3_BUCKET'), + S3_ACCESS_KEY_ID: requireEnv('S3_ACCESS_KEY_ID'), + S3_SECRET_ACCESS_KEY: requireEnv('S3_SECRET_ACCESS_KEY'), + S3_REGION: requireEnv('S3_REGION'), + }); + } + return new FilesystemPrivateStore({ + CFP_PRIVATE_STORAGE_PATH: requireEnv('CFP_PRIVATE_STORAGE_PATH'), + }); +} + +/** Resend HTTP send. Fetch-based to avoid adding a new dep at this stage. */ +async function resendSend(input: { + to: string; + from: string; + subject: string; + html: string; + text: string; +}): Promise { + const apiKey = requireEnv('RESEND_API_KEY'); + const res = await fetch('https://api.resend.com/emails', { + method: 'POST', + headers: { + 'authorization': `Bearer ${apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + from: input.from, + to: input.to, + subject: input.subject, + html: input.html, + text: input.text, + }), + }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Resend ${res.status}: ${body.slice(0, 200)}`); + } +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +interface CliArgs { + readonly dryRun: boolean; + readonly send: boolean; + readonly from: string | undefined; + readonly publicUrl: string | undefined; + readonly jsonPath: string | undefined; +} + +function parseArgs(argv: readonly string[]): CliArgs { + const opts: Record = {}; + for (const a of argv) { + if (!a.startsWith('--')) continue; + const eq = a.indexOf('='); + if (eq === -1) opts[a.slice(2)] = true; + else opts[a.slice(2, eq)] = a.slice(eq + 1); + } + return { + dryRun: opts['dry-run'] === true, + send: opts['send'] === true, + from: typeof opts['from'] === 'string' ? opts['from'] : undefined, + publicUrl: typeof opts['public-url'] === 'string' ? opts['public-url'] : undefined, + jsonPath: typeof opts['json'] === 'string' ? opts['json'] : undefined, + }; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!args.dryRun && !args.send) { + process.stderr.write('refusing to run without --dry-run or --send\n'); + process.exit(2); + } + if (args.dryRun && args.send) { + process.stderr.write('--dry-run and --send are mutually exclusive\n'); + process.exit(2); + } + + const publicStore = await openPublicStore(requireEnv('CFP_DATA_REPO_PATH')); + const privateStore = buildPrivateStore(); + await privateStore.load(); + + const report = await runMailout({ + publicStore, + privateStore, + mode: args.dryRun ? 'dry-run' : 'send', + from: args.from, + publicUrl: args.publicUrl ?? process.env['CFP_PUBLIC_URL'], + send: args.send ? resendSend : undefined, + }); + + process.stderr.write( + `[cutover-mailout] mode=${report.mode} recipients=${report.recipients.length} ` + + `skipped=${report.skipped.length} sent=${report.sent} failed=${report.failed.length}\n`, + ); + + const json = `${JSON.stringify(report, null, 2)}\n`; + if (args.jsonPath) { + await writeFile(resolve(args.jsonPath), json, 'utf8'); + } else { + process.stdout.write(json); + } + + process.exitCode = report.failed.length === 0 ? 0 : 1; +} + +const isMain = + process.argv[1] !== undefined && + import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')); + +if (isMain) { + main().catch((err: unknown) => { + process.stderr.write(`[cutover-mailout] failed: ${String(err)}\n`); + process.exit(2); + }); +} diff --git a/apps/api/scripts/reconcile-private-store.ts b/apps/api/scripts/reconcile-private-store.ts deleted file mode 100644 index 877aa00..0000000 --- a/apps/api/scripts/reconcile-private-store.ts +++ /dev/null @@ -1,147 +0,0 @@ -// Reconcile the private store against the public people sheet. -// -// Walks every Person in the public gitsheets repo and confirms each one has -// a corresponding `PrivateProfile` entry in the bucket. Flags orphans on -// both sides: -// -// - public Person with no matching private profile -// - private profile referencing a personId that does not exist publicly -// -// Optionally repairs missing private profiles with a `--fix` flag — creates -// a placeholder profile with `email: @example.invalid` so the API -// boot can find a row to read. Use `--fix` only in dev / disaster-recovery -// — production should investigate the underlying split before bulk-fixing. -// -// Usage: -// npm run -w apps/api script:reconcile-private-store # report only -// npm run -w apps/api script:reconcile-private-store -- --fix # report + repair missing profiles -// -// Reads CFP_DATA_REPO_PATH + STORAGE_BACKEND + CFP_PRIVATE_STORAGE_PATH (or -// the S3 vars) from the env, same as the API. -import 'dotenv/config'; -import { PrivateProfileSchema, type PrivateProfile } from '@cfp/shared/schemas'; -import { openPublicStore } from '../src/store/public.js'; -import { FilesystemPrivateStore } from '../src/store/private/filesystem.js'; -import { S3PrivateStore } from '../src/store/private/s3.js'; -import type { PrivateStore } from '../src/store/private/index.js'; - -interface ReconcileReport { - readonly publicCount: number; - readonly privateCount: number; - readonly missingPrivateForPublic: ReadonlyArray<{ personId: string; slug: string }>; - readonly orphanedPrivate: ReadonlyArray<{ personId: string }>; -} - -function requireEnv(name: string): string { - const v = process.env[name]; - if (!v) throw new Error(`Missing required env var ${name}`); - return v; -} - -function buildPrivateStore(): PrivateStore { - const backend = requireEnv('STORAGE_BACKEND'); - if (backend === 's3') { - return new S3PrivateStore({ - S3_ENDPOINT: requireEnv('S3_ENDPOINT'), - S3_BUCKET: requireEnv('S3_BUCKET'), - S3_ACCESS_KEY_ID: requireEnv('S3_ACCESS_KEY_ID'), - S3_SECRET_ACCESS_KEY: requireEnv('S3_SECRET_ACCESS_KEY'), - S3_REGION: requireEnv('S3_REGION'), - }); - } - return new FilesystemPrivateStore({ - CFP_PRIVATE_STORAGE_PATH: requireEnv('CFP_PRIVATE_STORAGE_PATH'), - }); -} - -async function reconcile(): Promise { - const repoPath = requireEnv('CFP_DATA_REPO_PATH'); - const publicStore = await openPublicStore(repoPath); - const privateStore = buildPrivateStore(); - await privateStore.load(); - - const people = await publicStore.people.queryAll(); - const publicIds = new Set(people.map((p) => p.id)); - - const missingPrivateForPublic: Array<{ personId: string; slug: string }> = []; - for (const person of people) { - if (person.deletedAt) continue; - const profile = await privateStore.getProfile(person.id); - if (!profile) { - missingPrivateForPublic.push({ personId: person.id, slug: person.slug }); - } - } - - const orphanedPrivate: Array<{ personId: string }> = []; - let privateCount = 0; - for await (const profile of privateStore.listAllProfiles()) { - privateCount++; - if (!publicIds.has(profile.personId)) { - orphanedPrivate.push({ personId: profile.personId }); - } - } - - return { - publicCount: people.length, - privateCount, - missingPrivateForPublic, - orphanedPrivate, - }; -} - -async function fixMissing(): Promise { - const repoPath = requireEnv('CFP_DATA_REPO_PATH'); - const publicStore = await openPublicStore(repoPath); - const privateStore = buildPrivateStore(); - await privateStore.load(); - - const people = await publicStore.people.queryAll(); - let fixed = 0; - for (const person of people) { - if (person.deletedAt) continue; - const existing = await privateStore.getProfile(person.id); - if (existing) continue; - const now = new Date().toISOString(); - const profile: PrivateProfile = PrivateProfileSchema.parse({ - personId: person.id, - email: `${person.slug}@example.invalid`, - emailRefreshedAt: now, - newsletter: null, - updatedAt: now, - }); - await privateStore.putProfile(profile); - fixed++; - } - return fixed; -} - -async function main(): Promise { - const argv = process.argv.slice(2); - const wantFix = argv.includes('--fix'); - - const report = await reconcile(); - - process.stdout.write(`Public people: ${report.publicCount}\n`); - process.stdout.write(`Private profiles: ${report.privateCount}\n`); - process.stdout.write( - `Missing private for public: ${report.missingPrivateForPublic.length}\n`, - ); - for (const m of report.missingPrivateForPublic) { - process.stdout.write(` - ${m.slug} (${m.personId})\n`); - } - process.stdout.write(`Orphaned private profiles: ${report.orphanedPrivate.length}\n`); - for (const o of report.orphanedPrivate) { - process.stdout.write(` - ${o.personId}\n`); - } - - if (wantFix && report.missingPrivateForPublic.length > 0) { - process.stdout.write(`\nApplying --fix...\n`); - const fixed = await fixMissing(); - process.stdout.write(`Fixed ${fixed} missing profiles\n`); - } -} - -main().catch((err) => { - process.stderr.write(`reconcile-private-store failed: ${String(err)}\n`); - process.exitCode = 1; -}); diff --git a/apps/api/scripts/reconcile.ts b/apps/api/scripts/reconcile.ts new file mode 100644 index 0000000..6cd06ea --- /dev/null +++ b/apps/api/scripts/reconcile.ts @@ -0,0 +1,339 @@ +/** + * reconcile.ts — Cutover + post-cutover reconciliation + * + * Walks the public Person sheet and the private store and reports + * inconsistencies across the four classes documented in + * plans/cutover-prep.md and specs/behaviors/account-migration.md: + * + * 1. Orphan public Persons — Person row with no matching PrivateProfile + * 2. Orphan private profiles — PrivateProfile with no matching Person + * 3. Inconsistent newsletter — newsletter.optedIn=true but no unsubscribeToken, + * or token without an optedIn payload + * 4. Drained legacy passwords — Person.githubUserId set but + * LegacyPasswordCredential still exists + * + * Output: a JSON report (machine-readable) preceded by a short human-readable + * summary on stderr. Exit code is 0 when nothing was flagged, 1 when issues + * remain *after* --fix has had a chance to repair the safe cases. + * + * --fix mode applies the safe repairs only: + * - regenerate missing unsubscribe tokens for opted-in profiles + * - delete LegacyPasswordCredential records when the Person is GitHub-linked + * + * Orphans (either direction) require human review and are never auto-fixed. + * + * Usage: + * npm run -w apps/api script:reconcile # report only + * npm run -w apps/api script:reconcile -- --fix # report + repair safe cases + * npm run -w apps/api script:reconcile -- --json=out.json + * + * Reads CFP_DATA_REPO_PATH + STORAGE_BACKEND + bucket envs from the env, same + * shape as the API. + * + * This script supersedes reconcile-private-store.ts (cutover-prep absorbed + * its scope) — see plans/cutover-prep.md Notes. + */ +import { randomBytes } from 'node:crypto'; +import { writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { + PrivateProfileSchema, + type PrivateProfile, +} from '@cfp/shared/schemas'; +import { openPublicStore } from '../src/store/public.js'; +import { + FilesystemPrivateStore, + S3PrivateStore, + type PrivateStore, +} from '../src/store/private/index.js'; + +// --------------------------------------------------------------------------- +// Report types — exported for tests +// --------------------------------------------------------------------------- + +export interface OrphanPublic { + readonly personId: string; + readonly slug: string; +} + +export interface OrphanPrivate { + readonly personId: string; +} + +export interface InconsistentNewsletter { + readonly personId: string; + readonly reason: 'opted_in_without_token' | 'token_without_optin_payload'; +} + +export interface DrainedLegacyPassword { + readonly personId: string; + readonly slug: string; + readonly githubUserId: number; +} + +export interface ReconcileReport { + readonly runAt: string; + readonly publicPeopleCount: number; + readonly privateProfileCount: number; + readonly legacyPasswordCount: number; + readonly orphanPublic: ReadonlyArray; + readonly orphanPrivate: ReadonlyArray; + readonly inconsistentNewsletter: ReadonlyArray; + readonly drainedLegacyPasswords: ReadonlyArray; + readonly fixesApplied: { + readonly newsletterTokens: number; + readonly legacyPasswordsDeleted: number; + }; +} + +// --------------------------------------------------------------------------- +// Env wiring +// --------------------------------------------------------------------------- + +function requireEnv(name: string): string { + const v = process.env[name]; + if (!v) throw new Error(`Missing required env var ${name}`); + return v; +} + +function buildPrivateStore(): PrivateStore { + const backend = requireEnv('STORAGE_BACKEND'); + if (backend === 's3') { + return new S3PrivateStore({ + S3_ENDPOINT: requireEnv('S3_ENDPOINT'), + S3_BUCKET: requireEnv('S3_BUCKET'), + S3_ACCESS_KEY_ID: requireEnv('S3_ACCESS_KEY_ID'), + S3_SECRET_ACCESS_KEY: requireEnv('S3_SECRET_ACCESS_KEY'), + S3_REGION: requireEnv('S3_REGION'), + }); + } + return new FilesystemPrivateStore({ + CFP_PRIVATE_STORAGE_PATH: requireEnv('CFP_PRIVATE_STORAGE_PATH'), + }); +} + +// --------------------------------------------------------------------------- +// Core reconcile (exported for tests) +// --------------------------------------------------------------------------- + +export interface ReconcileOptions { + readonly publicStore: Awaited>; + readonly privateStore: PrivateStore; + readonly fix?: boolean; + readonly now?: string; +} + +export async function reconcile( + opts: ReconcileOptions, +): Promise { + const { publicStore, privateStore, fix = false } = opts; + const runAt = opts.now ?? new Date().toISOString(); + + const people = await publicStore.people.queryAll(); + const liveById = new Map(); + for (const p of people) { + if (p.deletedAt) continue; + liveById.set(p.id, { + id: p.id, + slug: p.slug, + githubUserId: p.githubUserId ?? null, + }); + } + + const orphanPublic: OrphanPublic[] = []; + const orphanPrivate: OrphanPrivate[] = []; + const inconsistentNewsletter: InconsistentNewsletter[] = []; + const drainedLegacyPasswords: DrainedLegacyPassword[] = []; + + const profilesById = new Map(); + let privateProfileCount = 0; + + for await (const profile of privateStore.listAllProfiles()) { + privateProfileCount++; + profilesById.set(profile.personId, profile); + if (!liveById.has(profile.personId)) { + orphanPrivate.push({ personId: profile.personId }); + } + + const nl = profile.newsletter; + if (nl) { + if (nl.optedIn && !nl.unsubscribeToken) { + inconsistentNewsletter.push({ + personId: profile.personId, + reason: 'opted_in_without_token', + }); + } else if (!nl.optedIn && nl.unsubscribeToken && !nl.optedOutAt) { + inconsistentNewsletter.push({ + personId: profile.personId, + reason: 'token_without_optin_payload', + }); + } + } + } + + for (const person of liveById.values()) { + if (!profilesById.has(person.id)) { + orphanPublic.push({ personId: person.id, slug: person.slug }); + } + if (person.githubUserId) { + const cred = await privateStore.getLegacyPassword(person.id); + if (cred) { + drainedLegacyPasswords.push({ + personId: person.id, + slug: person.slug, + githubUserId: person.githubUserId, + }); + } + } + } + + const legacyPasswordCount = await privateStore.countLegacyPasswords(); + + let newsletterTokens = 0; + let legacyPasswordsDeleted = 0; + + if (fix) { + for (const issue of inconsistentNewsletter) { + if (issue.reason !== 'opted_in_without_token') continue; + const profile = profilesById.get(issue.personId); + if (!profile || !profile.newsletter) continue; + const token = randomBytes(32).toString('base64url'); + const repaired: PrivateProfile = PrivateProfileSchema.parse({ + ...profile, + newsletter: { ...profile.newsletter, unsubscribeToken: token }, + updatedAt: runAt, + }); + await privateStore.putProfile(repaired); + newsletterTokens++; + } + + for (const drained of drainedLegacyPasswords) { + await privateStore.deleteLegacyPassword(drained.personId); + legacyPasswordsDeleted++; + } + } + + return { + runAt, + publicPeopleCount: liveById.size, + privateProfileCount, + legacyPasswordCount, + orphanPublic, + orphanPrivate, + inconsistentNewsletter, + drainedLegacyPasswords, + fixesApplied: { + newsletterTokens, + legacyPasswordsDeleted, + }, + }; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +interface CliArgs { + readonly fix: boolean; + readonly jsonPath: string | undefined; +} + +function parseArgs(argv: readonly string[]): CliArgs { + let fix = false; + let jsonPath: string | undefined; + for (const a of argv) { + if (a === '--fix') fix = true; + else if (a.startsWith('--json=')) jsonPath = a.slice('--json='.length); + } + return { fix, jsonPath }; +} + +function summarize(report: ReconcileReport, fix: boolean): string { + const lines: string[] = []; + lines.push(`=== reconcile report (${report.runAt}) ===`); + lines.push(`public people: ${report.publicPeopleCount}`); + lines.push(`private profiles: ${report.privateProfileCount}`); + lines.push(`legacy password creds: ${report.legacyPasswordCount}`); + lines.push(`orphan public Persons: ${report.orphanPublic.length}`); + for (const o of report.orphanPublic.slice(0, 10)) { + lines.push(` - ${o.slug} (${o.personId})`); + } + if (report.orphanPublic.length > 10) { + lines.push(` ...and ${report.orphanPublic.length - 10} more`); + } + lines.push(`orphan private profiles: ${report.orphanPrivate.length}`); + for (const o of report.orphanPrivate.slice(0, 10)) lines.push(` - ${o.personId}`); + if (report.orphanPrivate.length > 10) { + lines.push(` ...and ${report.orphanPrivate.length - 10} more`); + } + lines.push(`inconsistent newsletter: ${report.inconsistentNewsletter.length}`); + for (const i of report.inconsistentNewsletter.slice(0, 10)) { + lines.push(` - ${i.personId} (${i.reason})`); + } + lines.push(`drained legacy passwords: ${report.drainedLegacyPasswords.length}`); + for (const d of report.drainedLegacyPasswords.slice(0, 10)) { + lines.push(` - ${d.slug} (${d.personId}) gh=${d.githubUserId}`); + } + if (fix) { + lines.push(''); + lines.push(`fixes applied:`); + lines.push(` newsletter tokens regenerated: ${report.fixesApplied.newsletterTokens}`); + lines.push(` legacy passwords deleted: ${report.fixesApplied.legacyPasswordsDeleted}`); + } + return lines.join('\n'); +} + +/** + * Anything that needs human intervention. Even after --fix, orphan rows in + * either direction stay listed — they require operator review. + */ +function unresolvedIssueCount(report: ReconcileReport): number { + return ( + report.orphanPublic.length + + report.orphanPrivate.length + + // After --fix the safe-fix categories below should be empty, but if + // --fix wasn't passed they still count as unresolved. + report.inconsistentNewsletter.length + + report.drainedLegacyPasswords.length - + report.fixesApplied.newsletterTokens - + report.fixesApplied.legacyPasswordsDeleted + ); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + + const repoPath = requireEnv('CFP_DATA_REPO_PATH'); + const publicStore = await openPublicStore(repoPath); + const privateStore = buildPrivateStore(); + await privateStore.load(); + + const report = await reconcile({ + publicStore, + privateStore, + fix: args.fix, + }); + + process.stderr.write(`${summarize(report, args.fix)}\n`); + + const json = `${JSON.stringify(report, null, 2)}\n`; + if (args.jsonPath) { + await writeFile(resolve(args.jsonPath), json, 'utf8'); + } else { + process.stdout.write(json); + } + + process.exitCode = unresolvedIssueCount(report) === 0 ? 0 : 1; +} + +const isMain = + process.argv[1] !== undefined && + import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')); + +if (isMain) { + main().catch((err: unknown) => { + process.stderr.write(`reconcile failed: ${String(err)}\n`); + process.exit(2); + }); +} diff --git a/apps/api/tests/cutover-dry-run.test.ts b/apps/api/tests/cutover-dry-run.test.ts new file mode 100644 index 0000000..4952a5b --- /dev/null +++ b/apps/api/tests/cutover-dry-run.test.ts @@ -0,0 +1,180 @@ +/** + * Tests for apps/api/scripts/cutover-dry-run.ts + * + * Exercises the orchestration end-to-end against the laddr fixture mysqldump: + * - importer runs and produces records + * - per-table row counts match per-sheet imported counts + * - smoke checks fire only when a target URL is provided + * + * The smoke-check leg is exercised against a stub fetch by injecting it as + * a global override — we don't spin up the API in this test (covered in + * api-skeleton.test.ts and read-api.test.ts). + */ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + countRowsByTable, + deterministicSample, + runDryRun, + runSmokeChecks, +} from '../scripts/cutover-dry-run.js'; + +const exec = promisify(execFile); +const FIXTURE_SQL = resolve(__dirname, '../scripts/fixtures/laddr-fixture.sql'); + +const SHEET_CONFIGS: ReadonlyArray<{ name: string; path: string }> = [ + { name: 'people', path: '${{ slug }}' }, + { name: 'projects', path: '${{ slug }}' }, + { name: 'project-memberships', path: '${{ projectSlug }}/${{ personSlug }}' }, + { name: 'project-updates', path: '${{ projectSlug }}/${{ number }}' }, + { name: 'project-buzz', path: '${{ projectSlug }}/${{ slug }}' }, + { name: 'help-wanted-roles', path: '${{ projectSlug }}/${{ id }}' }, + { name: 'help-wanted-interest', path: '${{ roleId }}/${{ personSlug }}' }, + { name: 'tags', path: '${{ namespace }}/${{ slug }}' }, + { name: 'tag-assignments', path: '${{ tagId }}/${{ taggableType }}/${{ taggableId }}' }, + { name: 'slug-history', path: '${{ entityType }}/${{ oldSlug }}' }, + { name: 'revocations', path: '${{ jti }}' }, +]; + +async function makeRepo(): Promise<{ path: string; cleanup: () => Promise }> { + const dir = await mkdtemp(join(tmpdir(), 'cfp-dryrun-')); + const git = (...a: string[]) => exec('git', a, { cwd: dir }); + await git('init', '-b', 'main'); + await git('config', 'user.email', 'test@cfp.test'); + await git('config', 'user.name', 'test'); + await git('config', 'commit.gpgsign', 'false'); + await git('commit', '--allow-empty', '-m', 'initial'); + + await mkdir(join(dir, '.gitsheets'), { recursive: true }); + for (const { name, path } of SHEET_CONFIGS) { + const cfg = `[gitsheet]\nroot = '${name}'\npath = '${path}'\n`; + await writeFile(join(dir, '.gitsheets', `${name}.toml`), cfg); + } + await git('add', '.gitsheets'); + await git('commit', '-m', 'configs'); + + return { path: dir, cleanup: () => rm(dir, { recursive: true, force: true }) }; +} + +async function makePrivate(): Promise<{ path: string; cleanup: () => Promise }> { + const dir = await mkdtemp(join(tmpdir(), 'cfp-dryrun-priv-')); + return { path: dir, cleanup: () => rm(dir, { recursive: true, force: true }) }; +} + +describe('countRowsByTable', () => { + it('counts rows across multiple statements per table', () => { + const sql = [ + "INSERT INTO `People` (`ID`, `Username`) VALUES (1,'alice'),(2,'bob');", + "INSERT INTO `People` (`ID`, `Username`) VALUES (3,'carol');", + "INSERT INTO `Projects` (`ID`, `Title`) VALUES (1,'A'),(2,'B'),(3,'C');", + ].join('\n'); + const counts = countRowsByTable(sql); + expect(counts.get('People')).toBe(3); + expect(counts.get('Projects')).toBe(3); + }); + + it('ignores parentheses inside quoted strings', () => { + const sql = "INSERT INTO `People` (`ID`, `Note`) VALUES (1, 'hello (world)'), (2, 'fun()');"; + expect(countRowsByTable(sql).get('People')).toBe(2); + }); + + it('handles the laddr fixture', async () => { + const sql = await readFile(FIXTURE_SQL, 'utf8'); + const counts = countRowsByTable(sql); + // The fixture has 4 people, 2 projects, etc — match against the same + // expectations as import-laddr.test.ts so they evolve together. + expect(counts.get('people')).toBe(4); + expect(counts.get('projects')).toBe(2); + }); +}); + +describe('deterministicSample', () => { + it('returns all items when n >= length', () => { + expect(deterministicSample(['a', 'b'], 5, 'seed')).toEqual(['a', 'b']); + }); + + it('is deterministic across runs with the same seed', () => { + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']; + const first = deterministicSample(items, 3, 'seed-1'); + const second = deterministicSample(items, 3, 'seed-1'); + expect(first).toEqual(second); + expect(first).toHaveLength(3); + }); + + it('produces different results for different seeds', () => { + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']; + const a = deterministicSample(items, 3, 'seed-1').sort(); + const b = deterministicSample(items, 3, 'seed-99').sort(); + expect(a).not.toEqual(b); + }); +}); + +describe('runDryRun (no target)', () => { + it('runs the importer and emits a count diff per sheet', async () => { + const repo = await makeRepo(); + const priv = await makePrivate(); + try { + const report = await runDryRun({ + sql: FIXTURE_SQL, + dataRepo: repo.path, + privateStore: priv.path, + target: null, + sampleSize: 10, + now: '2026-05-16T00:00:00.000Z', + }); + + expect(report.target).toBeNull(); + expect(report.smokeChecks).toEqual([]); + expect(report.importReport.entities['people']!.imported).toBeGreaterThan(0); + + const peopleDiff = report.countDiffs.find((d) => d.sheet === 'people'); + expect(peopleDiff?.sourceRows).toBe(4); + expect(peopleDiff?.importedRecords).toBe(4); + expect(peopleDiff?.matched).toBe(true); + + expect(report.stages.import).toBe(true); + expect(report.stages.countDiff).toBe(true); + expect(report.stages.smoke).toBe(true); + expect(report.passed).toBe(true); + } finally { + await repo.cleanup(); + await priv.cleanup(); + } + }, 120_000); +}); + +describe('runSmokeChecks (stub fetch)', () => { + const originalFetch = globalThis.fetch; + beforeEach(() => { + globalThis.fetch = vi.fn(async () => { + return new Response('ok', { status: 200 }); + }) as typeof fetch; + }); + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it('hits each smoke-test endpoint and records timings', async () => { + const results = await runSmokeChecks({ + url: 'http://stub.local', + samplePeople: ['alice'], + samplePeopleLegacyIds: [], + sampleProjects: ['squadquest'], + sampleProjectLegacyIds: [42], + // legacy-id smoke checks + // - sample people legacy: skipped (empty) + // - sample project legacy: /projects?ID=42 + }); + // We expect: 1 person + 1 project + 1 project-legacy + saml + oauth + 2 health + expect(results).toHaveLength(7); + expect(results.every((r) => r.ok)).toBe(true); + expect(results.some((r) => r.name === 'saml-metadata')).toBe(true); + expect(results.some((r) => r.name === 'health-ready')).toBe(true); + }); +}); diff --git a/apps/api/tests/cutover-mailout.test.ts b/apps/api/tests/cutover-mailout.test.ts new file mode 100644 index 0000000..51ead17 --- /dev/null +++ b/apps/api/tests/cutover-mailout.test.ts @@ -0,0 +1,205 @@ +/** + * Tests for apps/api/scripts/cutover-mailout.ts + * + * Covers recipient selection, email-body construction, and dry-run mode. + * `send` mode is exercised against an injected fake send() that records calls. + */ +import { describe, expect, it } from 'vitest'; +import { openRepo } from 'gitsheets'; + +import { buildEmailBody, runMailout } from '../scripts/cutover-mailout.js'; +import { openPublicStore } from '../src/store/public.js'; +import { FilesystemPrivateStore } from '../src/store/private/filesystem.js'; +import { createFullDataRepo, createPrivateStorageDir } from './helpers/test-full-repo.js'; + +const NOW = '2026-08-15T00:00:00.000Z'; + +function uuid(n: number): string { + return `01951a3c-0000-7000-8000-${String(n).padStart(12, '0')}`; +} + +async function seedPerson( + repoPath: string, + fields: { id: string; slug: string; fullName?: string; githubUserId?: number; deletedAt?: string }, +): Promise { + const repo = await openRepo({ gitDir: `${repoPath}/.git`, workTree: repoPath }); + await repo.transact( + { message: `seed person ${fields.slug}`, author: { name: 'test', email: 'test@cfp.test' } }, + async (tx) => { + await tx.sheet('people').upsert({ + id: fields.id, + slug: fields.slug, + fullName: fields.fullName ?? 'Test Person', + accountLevel: 'user', + ...(fields.githubUserId !== undefined ? { githubUserId: fields.githubUserId } : {}), + ...(fields.deletedAt !== undefined ? { deletedAt: fields.deletedAt } : {}), + createdAt: NOW, + updatedAt: NOW, + }); + }, + ); +} + +describe('cutover-mailout', () => { + it('selects only unclaimed Persons with valid emails', async () => { + const repo = await createFullDataRepo(); + const priv = await createPrivateStorageDir(); + try { + const privateStore = new FilesystemPrivateStore({ + CFP_PRIVATE_STORAGE_PATH: priv.path, + }); + await privateStore.load(); + + const aliceId = uuid(1); // unclaimed, valid email → recipient + const bobId = uuid(2); // claimed (githubUserId set) → skipped + const carolId = uuid(3); // unclaimed but no profile → skipped + const danId = uuid(4); // unclaimed, invalid email → skipped + const eveId = uuid(5); // deleted → skipped + + await seedPerson(repo.path, { id: aliceId, slug: 'alice', fullName: 'Alice A.' }); + await seedPerson(repo.path, { id: bobId, slug: 'bob', githubUserId: 12345 }); + await seedPerson(repo.path, { id: carolId, slug: 'carol' }); + await seedPerson(repo.path, { id: danId, slug: 'dan' }); + await seedPerson(repo.path, { id: eveId, slug: 'eve', deletedAt: NOW }); + + const publicStore = await openPublicStore(repo.path); + + await privateStore.putProfile({ + personId: aliceId, + email: 'alice@example.com', + emailRefreshedAt: NOW, + updatedAt: NOW, + }); + await privateStore.putProfile({ + personId: bobId, + email: 'bob@example.com', + emailRefreshedAt: NOW, + updatedAt: NOW, + }); + // carol has no profile + await privateStore.putProfile({ + personId: danId, + email: 'dan@example.invalid', + emailRefreshedAt: NOW, + updatedAt: NOW, + }); + + const report = await runMailout({ + publicStore, + privateStore, + mode: 'dry-run', + now: NOW, + }); + + expect(report.mode).toBe('dry-run'); + expect(report.recipients).toHaveLength(1); + expect(report.recipients[0]?.slug).toBe('alice'); + expect(report.recipients[0]?.email).toBe('alice@example.com'); + expect(report.sent).toBe(0); + + const skipReasons = report.skipped.map((s) => s.reason).sort(); + expect(skipReasons).toEqual(['deleted', 'github-linked', 'invalid-email', 'no-private-profile']); + } finally { + await repo.cleanup(); + await priv.cleanup(); + } + }); + + it('builds a properly formatted email body', () => { + const body = buildEmailBody( + { personId: uuid(1), slug: 'alice', email: 'alice@example.com', fullName: 'Alice Adams' }, + 'https://codeforphilly.org', + ); + expect(body.subject).toBe('Action needed: claim your Code for Philly account'); + expect(body.text).toContain('Hi Alice Adams,'); + expect(body.text).toContain('@alice'); + expect(body.text).toContain('https://codeforphilly.org/account/sign-in'); + expect(body.html).toContain(''); + expect(body.html).not.toContain('<'); + }); + + it('escapes HTML in fullName', () => { + const body = buildEmailBody( + { personId: uuid(2), slug: 'x', email: 'x@example.com', fullName: '' }, + 'https://codeforphilly.org', + ); + expect(body.html).not.toContain('