diff --git a/scripts/cli-apps/README.md b/scripts/cli-apps/README.md new file mode 100644 index 00000000..e57b3126 --- /dev/null +++ b/scripts/cli-apps/README.md @@ -0,0 +1,68 @@ +# Reviewed CLI app corrections + +Use Node.js 22.18 or newer (the guard imports the directory's TypeScript option lists). + +```bash +node scripts/cli-apps/apply-reviewed-corrections.mjs \ + --env .env.local --file /path/to/reviewed-batch.json +``` + +This runs every chunk and rolls it back. Add `--apply` to commit. The default +chunk size is 25 apps; `--chunk-size` accepts integers from 1 to 25. Each chunk +commits app changes, field sources, public dates, and cache jobs together. A +failure stops the run and rolls back that chunk. Earlier committed chunks stay +applied. Re-run the same batch to resume: identical fields skip without touching +app timestamps, provenance, public dates, or cache jobs. This is chunk-level +atomicity, not a transaction across the entire file. + +Every field needs its reviewed old value and an HTTPS evidence URL: + +```json +{ + "apps": [{ + "slug": "example", + "reviewed_at": "2026-11-02", + "fields": { "platforms": ["linux", "macos"], "install_methods": ["brew", "binary"] }, + "expected": { "platforms": ["linux"], "install_methods": ["binary"] }, + "sources": { + "platforms": "https://example.org/docs/platforms", + "install_methods": "https://example.org/docs/install" + } + }] +} +``` + +Allowed fields: `name`, `summary`, `website_url`, `repository_url`, `docs_url`, +`license`, `language`, `platforms`, `categories`, `primary_type`, `additional_types`, +`pricing_model`, `source_model`, `aliases`, and `install_methods`. + +Lists replace the entire field. Values must be unique. Categories need 1 to 3 +entries. Platforms, categories, types, prices, source models, and install methods +use the site's allowed values. Aliases accept nonempty text. Empty arrays are +allowed except for categories. Only `website_url`, `docs_url`, `license`, and +`language` accept null. URLs and sources must be valid HTTPS URLs. Summary rules +still apply. All input is checked before the first transaction; prior values +are checked again under row locks. Current values must match either the expected +old value or the requested value. Any third value stops the chunk. + +A changed `repository_url` also changes `github_project_id` in the same update. +GitHub owner/repo parsing and normalization match the refresh script. Existing +projects are matched case-insensitively and reused without updating their facts. +If older imports left case variants, the most recently checked row wins, with +a stable ID tie-breaker. Before inserting a missing project, a transaction-scoped +table lock serializes writers and the lookup runs again. This prevents duplicate +creation without changing existing rows or the database schema. Missing projects get a minimal +row with `checked_at = null`. GitHub owner/repo parsing also runs during whole-batch +validation, so malformed URLs stop the run before any chunk commits. +Non-GitHub repositories clear the link. This derived +link cannot be supplied as a reviewed field. Evidence is recorded for the +`repository_url` correction. + +Published app changes use the existing `/cli-apps` public-date and cache triggers. +The tool never publishes records or changes `last_verified_at`. Dry runs roll +back trigger effects and any newly created GitHub projects too. A dry run does +not reserve values; a later apply checks them again. + +For slow links, each query has a 180-second client timeout, a 120-second database +statement timeout, and a 15-second lock timeout. A connection or commit error +can leave the commit outcome uncertain; re-run the same file to check and resume. diff --git a/scripts/cli-apps/apply-reviewed-corrections.mjs b/scripts/cli-apps/apply-reviewed-corrections.mjs index 36506da8..c58ab2f9 100644 --- a/scripts/cli-apps/apply-reviewed-corrections.mjs +++ b/scripts/cli-apps/apply-reviewed-corrections.mjs @@ -1,43 +1,23 @@ #!/usr/bin/env node // Partial corrections only. No publishing, deletion, or whole-record date changes. -import { isDeepStrictEqual } from 'node:util'; import { readFile } from "node:fs/promises"; import { parseArgs, parseEnv } from "node:util"; import pg from "pg"; -import { guardCorrection } from "./lib/reviewed-corrections.mjs"; +import { applyCorrections, validateBatch } from "./lib/reviewed-corrections.mjs"; -const { values } = parseArgs({ options: { env: { type: "string" }, file: { type: "string" }, apply: { type: "boolean", default: false } } }); -if (!values.env || !values.file) throw new Error("Use --env --file [--apply]. Default is a rollback-only dry run."); +const { values } = parseArgs({ options: { env: { type: "string" }, file: { type: "string" }, apply: { type: "boolean", default: false }, "chunk-size": { type: "string", default: "25" } } }); +if (!values.env || !values.file) throw new Error("Use --env --file [--apply] [--chunk-size 25]. Default is a rollback-only dry run."); const environment = parseEnv(await readFile(values.env, "utf8")); if (!environment.DATABASE_URL) throw new Error("Selected env file has no DATABASE_URL"); const batch = JSON.parse(await readFile(values.file, "utf8")); -if (!Array.isArray(batch.apps) || batch.apps.length === 0) throw new Error("Batch must contain reviewed apps"); -if (new Set(batch.apps.map(app => app.slug)).size !== batch.apps.length) throw new Error("Duplicate slug in correction batch"); -const pool = new pg.Pool({ connectionString: environment.DATABASE_URL, max: 1, connectionTimeoutMillis: 15000, query_timeout: 15000 }); +const chunkSize = Number(values["chunk-size"]); +validateBatch(batch, chunkSize); +// Small transactions bound upload size; allow time for slow uplinks on each query. +const pool = new pg.Pool({ connectionString: environment.DATABASE_URL, max: 1, connectionTimeoutMillis: 30000, query_timeout: 180000, statement_timeout: 120000, lock_timeout: 15000 }); try { const client = await pool.connect(); try { - await client.query("begin"); - for (const record of batch.apps) { - const current = (await client.query("select * from public.terminal_apps where slug = $1 for update", [record.slug])).rows[0]; - if (!current) throw new Error(`Missing CLI app: ${record.slug}`); - guardCorrection(record, current); - const fields = Object.keys(record.fields).filter(field=>!isDeepStrictEqual(current[field],record.fields[field])); - // Identifiers come exclusively from the guard's fixed allowlist. - if (fields.length) await client.query(`update public.terminal_apps set ${fields.map((field, index) => `${field} = $${index + 2}`).join(", ")}, updated_at = now() where id = $1`, [current.id, ...fields.map(field => record.fields[field])]); - for (const field of fields) { - await client.query(`insert into public.terminal_app_field_sources (app_id, field_name, source, source_url, confidence, checked_at, note) - values ($1, $2, 'stack127_primary_source_review', $3, 'high', $4::timestamptz, $5) - on conflict (app_id, field_name) do update set source = excluded.source, source_url = excluded.source_url, - confidence = excluded.confidence, checked_at = excluded.checked_at, note = excluded.note`, - [current.id, field, record.sources[field], record.reviewed_at, record.notes?.[field] ?? "Partial field review; other app facts remain unverified by this batch."]); - } - console.log(`${values.apply ? "Applied" : "Checked"} CLI correction: ${record.slug}`); - } - await client.query(values.apply ? "commit" : "rollback"); - console.log(`${values.apply ? "Committed" : "Dry run passed for"} ${batch.apps.length} CLI correction(s).`); - } catch (error) { - await client.query("rollback"); - throw error; + const totals = await applyCorrections(client, batch, { apply: values.apply, chunkSize }); + console.log(`${values.apply ? "Committed" : "Dry run passed for"} ${totals.apps} CLI correction(s); ${totals.changedApps} changed, ${totals.skippedApps} identical apps skipped.`); } finally { client.release(); } } finally { await pool.end(); } diff --git a/scripts/cli-apps/lib/reviewed-corrections.mjs b/scripts/cli-apps/lib/reviewed-corrections.mjs index c9ee434f..a56dff0c 100644 --- a/scripts/cli-apps/lib/reviewed-corrections.mjs +++ b/scripts/cli-apps/lib/reviewed-corrections.mjs @@ -1,27 +1,141 @@ +import { parseRepo } from "../../self-host-apps/lib/github.mjs"; import { isDeepStrictEqual } from "node:util"; import { validateSummary } from "../../self-host-apps/lib/summary-rules.mjs"; +import { platformOptions, pricingOptions, sourceModelOptions, installLabels } from "../../../app/(site)/(directories)/cli-apps/cli-app-options.ts"; +import { terminalAppTypes, terminalAppCategories } from "../../../lib/terminal-app-taxonomy.ts"; -export const correctionFields = new Set(["name", "summary", "website_url", "repository_url", "docs_url", "license", "language", "install_methods"]); +const arrayValues = { + platforms: new Set(platformOptions.map(option => option.value)), + categories: new Set(terminalAppCategories.map(category => category.slug)), + additional_types: new Set(terminalAppTypes), + install_methods: new Set(installLabels.keys()), + aliases: null, +}; +const scalarValues = { + primary_type: new Set(terminalAppTypes), + pricing_model: new Set(pricingOptions.map(option => option.value)), + source_model: new Set(sourceModelOptions.map(option => option.value)), +}; +// These columns are nullable in terminal_apps. Repository removal is outside this workflow. +const nullableFields = new Set(["website_url", "docs_url", "license", "language"]); +export const correctionFields = new Set(["name", "summary", "website_url", "repository_url", "docs_url", "license", "language", ...Object.keys(arrayValues), ...Object.keys(scalarValues)]); + +function isHttps(value) { + if (typeof value !== "string" || !value.startsWith("https://") || /\s/.test(value)) return false; + try { return new URL(value).protocol === "https:" && Boolean(new URL(value).hostname); } + catch { return false; } +} export function guardCorrection(record, current) { - if (!record.slug || !record.fields || Object.keys(record.fields).length === 0) throw new Error("A slug and reviewed fields are required"); + if (!record || typeof record.slug !== "string" || !record.slug.trim() || !record.fields || Array.isArray(record.fields) || typeof record.fields !== "object" || Object.keys(record.fields).length === 0) throw new Error("A slug and reviewed fields are required"); const reviewTimestamp = Date.parse(record.reviewed_at); if (!/^\d{4}-\d{2}-\d{2}$/.test(record.reviewed_at ?? "") || !Number.isFinite(reviewTimestamp) || new Date(reviewTimestamp).toISOString().slice(0, 10) !== record.reviewed_at) throw new Error("A valid review date is required"); for (const [field, value] of Object.entries(record.fields)) { if (!correctionFields.has(field)) throw new Error(`${field}: partial correction not allowed`); - if (field === "install_methods") { - if (!Array.isArray(value) || !Array.isArray(current.install_methods) || new Set(value).size !== value.length || value.some(method => typeof method !== "string" || !current.install_methods.includes(method))) { - throw new Error("install_methods: this partial workflow only removes unsupported methods from the existing list"); - } - } else if (value !== null && typeof value !== "string") throw new Error(`${field}: expected text or null`); + if (Object.hasOwn(arrayValues, field)) { + if (!Array.isArray(value) || new Set(value).size !== value.length || value.some(item => typeof item !== "string" || !item.trim() || (arrayValues[field] && !arrayValues[field].has(item)))) throw new Error(`${field}: expected unique allowed text values`); + if (field === "categories" && (value.length < 1 || value.length > 3)) throw new Error("categories: expected 1 to 3 reviewed values"); + } else if (value === null ? !nullableFields.has(field) : typeof value !== "string" || !value.trim()) throw new Error(`${field}: expected nonempty text${nullableFields.has(field) ? " or null" : ""}`); + if (Object.hasOwn(scalarValues, field) && !scalarValues[field].has(value)) throw new Error(`${field}: invalid value`); if (!Object.hasOwn(record.expected ?? {}, field)) throw new Error(`${field}: expected value required`); - if (!isDeepStrictEqual(current[field], record.expected[field])) throw new Error(`${record.slug}: ${field} changed since review`); - if (!/^https:\/\//.test(record.sources?.[field] ?? "")) throw new Error(`${field}: reviewed source URL required`); - if (field.endsWith("_url") && value !== null && !/^https:\/\//.test(value)) throw new Error(`${field}: HTTPS URL required`); - if (["summary", "name"].includes(field) && !value?.trim()) throw new Error(`${field}: cannot be empty`); + // A committed chunk can be replayed, but a third value still blocks the chunk. + if (!isDeepStrictEqual(current[field], record.expected[field]) && !isDeepStrictEqual(current[field], value)) throw new Error(`${record.slug}: ${field} changed since review`); + if (!isHttps(record.sources?.[field])) throw new Error(`${field}: reviewed source URL required`); + if (field.endsWith("_url") && value !== null && !isHttps(value)) throw new Error(`${field}: HTTPS URL required`); + if (field === "repository_url") githubRepository(value); if (field === "summary") { const { errors } = validateSummary(value); if (errors.length) throw new Error(`${record.slug}: ${errors.join("; ")}`); } } } + +export function validateBatch(batch, chunkSize) { + if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 25) throw new Error("chunk-size must be an integer from 1 to 25"); + if (!Array.isArray(batch?.apps) || batch.apps.length === 0) throw new Error("Batch must contain reviewed apps"); + if (new Set(batch.apps.map(app => app?.slug)).size !== batch.apps.length) throw new Error("Duplicate slug in correction batch"); + // Validate all input before committing even the first chunk. DB comparisons happen under row locks. + for (const record of batch.apps) guardCorrection(record, record?.expected ?? {}); +} + +function githubRepository(url) { + if (new URL(url).hostname.toLowerCase() !== "github.com") return null; + const parsed = parseRepo(url); + const owner = parsed.owner.trim().toLowerCase(); + const repo = parsed.repo.trim().toLowerCase(); + if (!owner || !repo) throw new Error("GitHub owner and repository are required"); + return { owner, repo }; +} + +async function repositoryProjectId(client, url) { + const repository = githubRepository(url); + if (!repository) return null; + const { owner, repo } = repository; + // Older imports preserve casing, and some repos already have multiple case variants. + // Prefer the most recently checked row, with a stable ID tie-breaker. + const find = () => client.query("select id from public.github_projects where lower(owner) = $1 and lower(repo) = $2 order by checked_at desc nulls last, id limit 1", [owner, repo]); + const existing = (await find()).rows[0]; + if (existing) return existing.id; + // There is only a case-sensitive unique index. Serialize missing-project inserts + // with every writer (including importers that do not use advisory locks), then + // recheck after the lock has waited for any in-flight inserts to commit. + await client.query("lock table public.github_projects in share row exclusive mode"); + const afterLock = (await find()).rows[0]; + if (afterLock) return afterLock.id; + const inserted = await client.query(`insert into public.github_projects (owner, repo, full_name, repository_url, checked_at) + values ($1, $2, $3, $4, null) on conflict (owner, repo) do nothing returning id`, + [owner, repo, `${owner}/${repo}`, `https://github.com/${owner}/${repo}`]); + // A concurrent importer may have created the project after the first lookup. + return inserted.rows[0]?.id ?? (await find()).rows[0].id; +} + +export async function applyCorrections(client, batch, { apply = false, chunkSize = 25, report = console.log } = {}) { + validateBatch(batch, chunkSize); + const totals = { apps: 0, changedApps: 0, changedFields: 0, skippedApps: 0 }; + const chunkCount = Math.ceil(batch.apps.length / chunkSize); + for (let offset = 0; offset < batch.apps.length; offset += chunkSize) { + const records = batch.apps.slice(offset, offset + chunkSize); + const label = `Chunk ${offset / chunkSize + 1}/${chunkCount}`; + const counts = { apps: records.length, changedApps: 0, changedFields: 0, skippedApps: 0 }; + let finishing = false; + try { + await client.query("begin"); + // No trigger suppression: public dates and cache jobs commit/roll back with the app changes. + for (const record of records) { + const current = (await client.query("select * from public.terminal_apps where slug = $1 for update", [record.slug])).rows[0]; + if (!current) throw new Error(`Missing CLI app: ${record.slug}`); + guardCorrection(record, current); + const fields = Object.keys(record.fields).filter(field => !isDeepStrictEqual(current[field], record.fields[field])); + if (!fields.length) { counts.skippedApps++; continue; } + // Identifiers come exclusively from the guard's fixed allowlist. + const updateFields = [...fields]; + const updateValues = fields.map(field => record.fields[field]); + if (fields.includes("repository_url")) { + updateFields.push("github_project_id"); + updateValues.push(await repositoryProjectId(client, record.fields.repository_url)); + } + await client.query(`update public.terminal_apps set ${updateFields.map((field, index) => `${field} = $${index + 2}`).join(", ")}, updated_at = now() where id = $1`, [current.id, ...updateValues]); + // One small provenance query per app rather than one round trip per field. + await client.query(`insert into public.terminal_app_field_sources (app_id, field_name, source, source_url, confidence, checked_at, note) + select $1, data.field_name, 'stack127_primary_source_review', data.source_url, 'high', $2::timestamptz, data.note + from unnest($3::text[], $4::text[], $5::text[]) as data(field_name, source_url, note) + on conflict (app_id, field_name) do update set source = excluded.source, source_url = excluded.source_url, + confidence = excluded.confidence, checked_at = excluded.checked_at, note = excluded.note`, + [current.id, record.reviewed_at, fields, fields.map(field => record.sources[field]), fields.map(field => record.notes?.[field] ?? "Partial field review; other app facts remain unverified by this batch.")]); + counts.changedApps++; + counts.changedFields += fields.length; + } + finishing = true; + await client.query(apply ? "commit" : "rollback"); + } catch (error) { + let rolledBack = false; + try { await client.query("rollback"); rolledBack = true; } catch { /* Preserve the original error if the connection is lost. */ } + const outcome = finishing || !rolledBack ? "failed; commit outcome may be uncertain" : "rolled back"; + report(`${label}: ${outcome}; stopped. Earlier committed chunks remain applied. Re-run the same file to resume (identical fields skip).`); + throw new Error(`${label}: ${error.message}`, { cause: error }); + } + for (const key of Object.keys(totals)) totals[key] += counts[key]; + report(`${label}: ${apply ? "committed" : "dry-run rolled back"}; ${counts.apps} apps, ${counts.changedApps} changed, ${counts.changedFields} fields, ${counts.skippedApps} identical apps skipped.`); + } + return totals; +} diff --git a/scripts/cli-apps/scripts.json b/scripts/cli-apps/scripts.json index 3a779d7c..0d9a61f9 100644 --- a/scripts/cli-apps/scripts.json +++ b/scripts/cli-apps/scripts.json @@ -27,5 +27,5 @@ { "name": "cli-apps:cache-icons", "command": "node scripts/cli-apps/cache-icons.mjs --file scripts/cli-apps/batches/curated-publish.json", "purpose": "Normalize and upload every reviewed CLI app icon to stack127-media, then record its managed URL and official source URL." }, { "name": "cli-apps:publish", "command": "node scripts/cli-apps/publish-apps.mjs --file [--dry-run]", "purpose": "Publish reviewed records into terminal_apps with field provenance and candidate linkage." }, { "name": "cli-apps:coverage", "command": "node scripts/cli-apps/coverage-report.mjs [--out ] [--markdown ]", "purpose": "Build per-source coverage report with discovered/deduplicated/excluded/enriched/published/failed counts and scope notes." }, - { "name": "cli-apps:apply-reviewed-corrections", "command": "node scripts/cli-apps/apply-reviewed-corrections.mjs --env --file [--apply]", "purpose": "Apply source-backed partial field corrections with prior-value checks. Defaults to rollback; preserves whole-record verification dates." } + { "name": "cli-apps:apply-reviewed-corrections", "command": "node scripts/cli-apps/apply-reviewed-corrections.mjs --env --file [--apply] [--chunk-size 25]", "purpose": "Apply source-backed partial fact corrections in atomic chunks of 1-25 apps (default 25). Defaults to rollback; identical fields skip on replay. Repository changes relink shared GitHub facts. Preserves whole-record verification dates. See README.md." } ] diff --git a/tests/pages/cli-apps/reviewed-corrections.test.mjs b/tests/pages/cli-apps/reviewed-corrections.test.mjs index a674e7e7..37e26a5c 100644 --- a/tests/pages/cli-apps/reviewed-corrections.test.mjs +++ b/tests/pages/cli-apps/reviewed-corrections.test.mjs @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { guardCorrection } from "../../../scripts/cli-apps/lib/reviewed-corrections.mjs"; +import { guardCorrection, applyCorrections } from "../../../scripts/cli-apps/lib/reviewed-corrections.mjs"; const correction = { slug: "example", reviewed_at: "2026-09-08", @@ -9,7 +9,7 @@ const correction = { sources: { summary: "https://example.org/docs" }, }; -test("unsupported install methods can be removed without inventing replacements", () => { +test("install methods can be replaced with reviewed allowed values", () => { const record = { ...correction, fields: { install_methods: ["source"] }, expected: { install_methods: ["brew", "apt", "source"] }, sources: { install_methods: "https://example.org/install" } }; assert.doesNotThrow(() => guardCorrection(record, record.expected)); for (const value of [null, "source", ["source", "source"], ["invented"]]) { @@ -42,3 +42,223 @@ test("review dates reject calendar rollover and preserve valid leap days", () => } assert.doesNotThrow(() => guardCorrection({ ...correction, reviewed_at: "2024-02-29" }, correction.expected)); }); + +function fieldCorrection(field, value, before = "old") { + return { ...correction, fields: { [field]: value }, expected: { [field]: before }, sources: { [field]: "https://example.org/facts" } }; +} + +test("reviewed lists and enums accept the directory vocabulary and reject bad shapes", () => { + const valid = { platforms: ["linux", "macos", "windows", "bsd", "web"], categories: ["development", "devops", "ai"], primary_type: "tui", additional_types: ["cli", "editor"], pricing_model: "usage_based", source_model: "source_available", aliases: ["Alternate name"], install_methods: ["brew-cask", "go", "binary", "source"] }; + for (const [field, value] of Object.entries(valid)) { + const record = fieldCorrection(field, value); + assert.doesNotThrow(() => guardCorrection(record, record.expected)); + for (const invalid of [null, 1, {}, "invented", ...(field === "aliases" ? [] : [["invented"]]), [""], ["cli", "cli"]]) { + assert.throws(() => guardCorrection(fieldCorrection(field, invalid), record.expected), new RegExp(field)); + } + } + for (const value of [[], ["development", "files", "text", "ai"]]) { + assert.throws(() => guardCorrection(fieldCorrection("categories", value), { categories: "old" }), /1 to 3/); + } + for (const field of ["aliases", "platforms", "additional_types", "install_methods"]) { + assert.doesNotThrow(() => guardCorrection(fieldCorrection(field, []), { [field]: "old" })); + } +}); + +test("nullable facts, HTTPS URLs, and unchanged replay still require evidence and expected values", () => { + for (const field of ["website_url", "docs_url", "license", "language"]) { + assert.doesNotThrow(() => guardCorrection(fieldCorrection(field, null), { [field]: "old" })); + } + for (const field of ["name", "summary", "repository_url", "primary_type", "pricing_model", "source_model"]) { + assert.throws(() => guardCorrection(fieldCorrection(field, null), { [field]: "old" }), new RegExp(field)); + } + for (const url of ["http://example.org", "https://", "https://bad host", "https://?oops"]) { + assert.throws(() => guardCorrection(fieldCorrection("docs_url", url), { docs_url: "old" }), /HTTPS/); + assert.throws(() => guardCorrection({ ...correction, sources: { summary: url } }, correction.expected), /source URL/); + } + assert.doesNotThrow(() => guardCorrection(correction, correction.fields)); + assert.throws(() => guardCorrection({ ...correction, expected: {} }, correction.fields), /expected value/); + assert.throws(() => guardCorrection({ ...correction, sources: {} }, correction.fields), /source URL/); +}); + +// Transactional test double: app and provenance changes commit/roll back together. +function database(records, projects = []) { + let rows = structuredClone(records); + let sources = []; + let repositories = structuredClone(projects); + let saved; + const queries = []; + return { + queries, + get rows() { return rows; }, get sources() { return sources; }, get projects() { return repositories; }, + async query(sql, params) { + queries.push(sql); + if (sql === "begin") saved = structuredClone({ rows, sources, repositories }); + else if (sql === "rollback") ({ rows, sources, repositories } = saved); + else if (sql === "commit") saved = undefined; + else if (sql.startsWith("lock table public.github_projects")) { + // Simulate a writer that committed a differently cased repo while the lock waited. + if (this.concurrentProject) repositories.push(structuredClone(this.concurrentProject)); + } + else if (sql.startsWith("select *")) return { rows: rows.filter(row => row.slug === params[0]) }; + else if (sql.startsWith("select id")) return { rows: repositories.filter(row => (sql.includes("lower(owner)") ? row.owner.toLowerCase() === params[0] && row.repo.toLowerCase() === params[1] : row.owner === params[0] && row.repo === params[1])) }; + else if (sql.startsWith("insert into public.github_projects")) { + const project = { id: `project-${repositories.length}`, owner: params[0], repo: params[1], full_name: params[2], repository_url: params[3], checked_at: null }; + repositories.push(project); + return { rows: [project] }; + } else if (sql.startsWith("update public.terminal_apps")) { + const row = rows.find(row => row.id === params[0]); + [...sql.matchAll(/(\w+) = \$(\d+)/g)].filter(match => match[1] !== "id").forEach(match => { row[match[1]] = params[Number(match[2]) - 1]; }); + row.updated_at = "changed"; + } else if (sql.startsWith("insert into public.terminal_app_field_sources")) { + sources.push({ app: params[0], date: params[1], fields: params[2], urls: params[3], notes: params[4] }); + if (this.failSources) throw new Error("source write failed"); + } else throw new Error(`Unexpected SQL: ${sql}`); + return { rows: [] }; + }, + }; +} +const silent = { report() {} }; +function batchOf(size) { + return { apps: Array.from({ length: size }, (_, i) => ({ ...fieldCorrection("name", `App ${i}`, `Old ${i}`), slug: `app-${i}` })) }; +} +function rowsFor(batch) { + return batch.apps.map((record, i) => ({ id: `id-${i}`, slug: record.slug, ...record.expected, published: true, last_verified_at: "2025-01-01", updated_at: "original" })); +} + +test("300 apps commit in 12 chunks, keep partial fields, record sources and replay without writes", async () => { + const batch = batchOf(300); + const db = database(rowsFor(batch)); + const reports = []; + const result = await applyCorrections(db, batch, { apply: true, report: line => reports.push(line) }); + assert.equal(result.changedApps, 300); + assert.equal(db.queries.filter(sql => sql === "commit").length, 12); + assert.equal(reports.length, 12); + assert.match(reports[11], /Chunk 12\/12: committed; 25 apps, 25 changed, 25 fields/); + assert.equal(db.sources.length, 300); + assert.deepEqual(db.sources[0].fields, ["name"]); + assert.deepEqual(db.sources[0].urls, ["https://example.org/facts"]); + assert.equal(db.rows[0].last_verified_at, "2025-01-01"); + db.queries.length = 0; + const replay = await applyCorrections(db, batch, { apply: true, ...silent }); + assert.equal(replay.skippedApps, 300); + assert.equal(db.queries.some(sql => /^(update|insert)/.test(sql)), false); + assert.equal(db.sources.length, 300); +}); + +test("dry run rolls back each chunk by default", async () => { + const batch = batchOf(3); + const original = rowsFor(batch); + const db = database(original); + await applyCorrections(db, batch, { chunkSize: 2, ...silent }); + assert.deepEqual(db.rows, original); + assert.deepEqual(db.sources, []); + assert.equal(db.queries.filter(sql => sql === "rollback").length, 2); + assert.equal(db.queries.includes("commit"), false); +}); + +test("stale fields roll back the whole failing chunk, preserve earlier chunks, and resume", async () => { + const batch = batchOf(4); + const original = rowsFor(batch); + original[3].name = "Unexpected"; + const db = database(original); + const reports = []; + await assert.rejects(applyCorrections(db, batch, { apply: true, chunkSize: 2, report: line => reports.push(line) }), /Chunk 2\/2: app-3: name changed since review/); + assert.deepEqual(db.rows.map(row => row.name), ["App 0", "App 1", "Old 2", "Unexpected"]); + assert.equal(db.sources.length, 2); + assert.match(reports[1], /rolled back; stopped/); + db.rows[3].name = "Old 3"; + const result = await applyCorrections(db, batch, { apply: true, chunkSize: 2, ...silent }); + assert.equal(result.skippedApps, 2); + assert.equal(result.changedApps, 2); +}); + +test("source failures roll back app changes and malformed later input prevents all commits", async () => { + const batch = batchOf(2); + const original = rowsFor(batch); + const db = database(original); + db.failSources = true; + await assert.rejects(applyCorrections(db, batch, { apply: true, ...silent }), /source write failed/); + assert.deepEqual(db.rows, original); + assert.deepEqual(db.sources, []); + db.queries.length = 0; + batch.apps[1].sources = {}; + await assert.rejects(applyCorrections(db, batch, { apply: true, chunkSize: 1, ...silent }), /source URL/); + assert.deepEqual(db.queries, []); + for (const chunkSize of [0, -1, 1.5, 26, NaN]) { + await assert.rejects(applyCorrections(db, batch, { chunkSize, ...silent }), /chunk-size/); + } + await assert.rejects(applyCorrections(db, { apps: [batch.apps[0], batch.apps[0]] }, silent), /Duplicate slug/); +}); + +test("repository corrections reuse normalized GitHub projects, create unchecked projects, and unlink non-GitHub repos", async () => { + for (const existing of [true, false]) { + const record = fieldCorrection("repository_url", "https://github.com/Micro-Editor/Micro.git/tree/main", "https://github.com/zyedidia/micro"); + const projects = existing ? [{ id: "existing", owner: "micro-editor", repo: "micro", checked_at: "2026-01-01" }] : []; + const db = database([{ id: "app", slug: record.slug, ...record.expected, github_project_id: "old-project" }], projects); + await applyCorrections(db, { apps: [record] }, { apply: true, ...silent }); + assert.equal(db.rows[0].github_project_id, existing ? "existing" : "project-0"); + assert.equal(db.projects[0].checked_at, existing ? "2026-01-01" : null); + assert.equal(db.projects[0].owner, "micro-editor"); + assert.equal(db.projects[0].repo, "micro"); + assert.deepEqual(db.sources[0].fields, ["repository_url"]); + db.queries.length = 0; + await applyCorrections(db, { apps: [record] }, { apply: true, ...silent }); + assert.equal(db.queries.some(sql => /^(update|insert)/.test(sql)), false); + const nonGithub = fieldCorrection("repository_url", "https://gitlab.com/example/app", record.fields.repository_url); + await applyCorrections(db, { apps: [nonGithub] }, { apply: true, ...silent }); + assert.equal(db.rows[0].github_project_id, null); + } +}); + +test("new GitHub project creation rolls back with a failed correction", async () => { + const record = fieldCorrection("repository_url", "https://github.com/infracost/cli", "https://github.com/infracost/infracost"); + const original = [{ id: "app", slug: record.slug, ...record.expected, github_project_id: "old" }]; + const db = database(original); + db.failSources = true; + await assert.rejects(applyCorrections(db, { apps: [record] }, { apply: true, ...silent }), /source write failed/); + assert.deepEqual(db.projects, []); + assert.deepEqual(db.rows, original); +}); + +test("a lost commit response preserves the error, reports uncertainty, and allows replay", async () => { + const batch = batchOf(1); + const db = database(rowsFor(batch)); + let loseCommit = true; + const reports = []; + const client = { async query(sql, params) { + const result = await db.query(sql, params); + if (sql === "commit" && loseCommit) { loseCommit = false; throw new Error("lost commit response"); } + return result; + } }; + await assert.rejects(applyCorrections(client, batch, { apply: true, report: line => reports.push(line) }), /lost commit response/); + assert.match(reports[0], /commit outcome may be uncertain/); + const replay = await applyCorrections(client, batch, { apply: true, ...silent }); + assert.equal(replay.skippedApps, 1); + assert.equal(db.sources.length, 1); +}); + + +test("mixed-case existing GitHub projects are reused without creating duplicates", async () => { + const record = fieldCorrection("repository_url", "https://github.com/couchpotato/couchpotatoserver", "https://github.com/example/old"); + const original = [{ id: "app", slug: record.slug, ...record.expected, github_project_id: "old" }]; + const project = { id: "mixed-case", owner: "CouchPotato", repo: "CouchPotatoServer", checked_at: "2026-01-01" }; + for (const concurrent of [false, true]) { + const db = database(original, concurrent ? [] : [project]); + if (concurrent) db.concurrentProject = project; + await applyCorrections(db, { apps: [record] }, { apply: true, ...silent }); + assert.equal(db.rows[0].github_project_id, "mixed-case"); + assert.deepEqual(db.projects, [project]); + assert.equal(db.queries.some(sql => sql.startsWith("insert into public.github_projects")), false); + if (concurrent) assert.equal(db.queries.filter(sql => sql.startsWith("select id")).length, 2); + } +}); + +test("invalid GitHub URLs in later chunks fail whole-batch validation before any transaction", async () => { + for (const url of ["https://github.com/owner", "https://github.com/owner/.git"]) { + const batch = batchOf(2); + batch.apps[1] = { ...fieldCorrection("repository_url", url, "https://github.com/example/old"), slug: "app-1" }; + const db = database(rowsFor(batch)); + await assert.rejects(applyCorrections(db, batch, { apply: true, chunkSize: 1, ...silent }), /GitHub/); + assert.deepEqual(db.queries, []); + } +});