From 2f730f6f32fd8802062ea0f12f1baee343493f49 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Mon, 2 Nov 2026 15:03:32 +0530 Subject: [PATCH 1/4] Add a repeatable way to publish chosen self-host apps Claim exact candidates by ID or from a reviewed batch, look up an app by name or repository, gate production through the shared resolver with a confirm value, and refresh one app's source-host facts by slug. Document the steps and warn against sync-local-to-prod for single apps. Co-Authored-By: Claude Opus 5.5 --- scripts/self-host-apps/README.md | 4 +- .../self-host-apps/apply-candidate-review.mjs | 61 +++-- .../apply-resource-estimates.mjs | 213 +++++++++--------- scripts/self-host-apps/lib/environment.mjs | 14 ++ scripts/self-host-apps/refresh-github.mjs | 5 +- .../refresh-install-options.mjs | 8 +- .../self-host-apps/refresh-source-host.mjs | 11 +- scripts/self-host-apps/scripts.json | 14 +- .../self-host-apps/select-candidate-batch.mjs | 78 ++++++- scripts/self-host-apps/upsert-app.mjs | 5 +- skills/self-host-apps/enrichment/SKILL.md | 17 +- stack127_docs/self-host-apps-data.md | 28 +++ .../self-host-apps/chosen-candidates.test.mjs | 156 +++++++++++++ 13 files changed, 467 insertions(+), 147 deletions(-) create mode 100644 scripts/self-host-apps/lib/environment.mjs create mode 100644 tests/self-host-apps/chosen-candidates.test.mjs diff --git a/scripts/self-host-apps/README.md b/scripts/self-host-apps/README.md index c8dc10ed..f0cceee2 100644 --- a/scripts/self-host-apps/README.md +++ b/scripts/self-host-apps/README.md @@ -21,7 +21,7 @@ Discovery stores raw evidence. It cannot publish or assign final categories, sum ## Review movement -- `select-candidate-batch.mjs` atomically claims queued and name-review rows. +- `select-candidate-batch.mjs` atomically claims queued and name-review rows by rank, or exact IDs with `--ids` / `--ids-from `. `--match ` is read-only lookup. - `collect-github-evidence.mjs` collects official API evidence without editing apps. - `apply-candidate-review.mjs` validates reviewed decisions and writes them transactionally. - `release-candidate-batch.mjs` releases an abandoned claim. @@ -39,4 +39,6 @@ Discovery stores raw evidence. It cannot publish or assign final categories, sum Production is canonical. `sync-prod-to-local.mjs` rebuilds the disposable local mirror. `sync-local-to-prod.mjs` is an exceptional guarded upsert, requires `--confirm-reviewed-push`, and does not draft missing production rows unless `--draft-missing` is also explicit. +For targeted additions, follow “Add one app on request” in `stack127_docs/self-host-apps-data.md`. Do not use `sync-local-to-prod` for single apps. Editorial commands use `--environment development` (default) or `--environment production --confirm stack127-production`, including production previews. GitLab/Codeberg and install-option refreshes accept `--slug`; scheduled refresh callers keep their existing environment behavior. + See `content_docs/self-host-apps/data-workflow.md` and `scripts/self-host-apps/scripts.json`. diff --git a/scripts/self-host-apps/apply-candidate-review.mjs b/scripts/self-host-apps/apply-candidate-review.mjs index 897db5fc..b44ef0f7 100644 --- a/scripts/self-host-apps/apply-candidate-review.mjs +++ b/scripts/self-host-apps/apply-candidate-review.mjs @@ -16,6 +16,8 @@ import { buildInstallOptions } from "./lib/install-options.mjs"; import { installMethods, installOptionsToRow } from "./lib/streamlined-storage.mjs"; import { isManagedMediaUrl } from "../stack127/media/storage.mjs"; +import { databaseOptions } from "./lib/environment.mjs"; + const decisions = new Set(["publish", "duplicate", "reject", "needs_review"]); function reviewRecords(input) { @@ -100,7 +102,7 @@ async function ensureCategories(client, categories) { } } -export async function upsertApp(client, record, expectedAppId) { +export async function upsertApp(client, record, expectedAppId, dryRun = false) { const existing = await client.query("select id from public.self_host_apps where slug = $1", [record.slug]); if (existing.rows[0] && expectedAppId && existing.rows[0].id !== expectedAppId) { throw new Error(`${record.slug} belongs to a different app than app_id`); @@ -109,6 +111,8 @@ export async function upsertApp(client, record, expectedAppId) { throw new Error(`${record.slug} already exists; provide app_id after confirming the merge`); } + if (dryRun) return existing.rows[0]?.id ?? null; + const values = appColumns.map((column) => record[column]); const updates = appColumns .filter((column) => column !== "slug") @@ -281,23 +285,26 @@ export async function upsertInstallOptions(client, appId, options, catalogSource ); } -async function candidateById(client, id) { - const result = await client.query("select * from public.self_host_app_candidates where id = $1 for update", [id]); +async function candidateById(client, id, dryRun) { + const result = await client.query("select * from public.self_host_app_candidates where id = $1" + (dryRun ? "" : " for update"), [id]); if (!result.rows[0]) throw new Error(`Missing candidate: ${id}`); return result.rows[0]; } -async function applyReview(client, review) { +async function applyReview(client, review, dryRun = false) { const decision = review.decision ?? "publish"; if (!decisions.has(decision)) throw new Error(`Unsupported decision: ${decision}`); if (!review.candidate_id) throw new Error("candidate_id is required"); - const candidate = await candidateById(client, review.candidate_id); + const candidate = await candidateById(client, review.candidate_id, dryRun); if (candidate.status !== "enriching") { throw new Error(`Candidate ${candidate.id} is ${candidate.status}; claim it before applying a review`); } if (decision === "duplicate") { if (!review.app_id) throw new Error("duplicate decisions require app_id"); + const target = await client.query("select id from public.self_host_apps where id = $1", [review.app_id]); + if (!target.rows[0]) throw new Error(`Missing duplicate app: ${review.app_id}`); + if (dryRun) return { candidate_id: candidate.id, decision, app_id: review.app_id }; await preserveCandidateSources(client, candidate, review.app_id); await client.query( ` @@ -313,6 +320,7 @@ async function applyReview(client, review) { if (decision === "reject" || decision === "needs_review") { if (!review.reason) throw new Error(`${decision} decisions require reason`); + if (dryRun) return { candidate_id: candidate.id, decision }; const candidateStatus = decision === "reject" ? "rejected" : decision; await client.query( ` @@ -331,6 +339,16 @@ async function applyReview(client, review) { throw new Error(`${validation.record.slug}: ${validation.errors.join("; ")}`); } + for (const slug of validation.record.categories) { + if (!getSelfHostAppCategory(slug)) throw new Error(`Unknown category: ${slug}`); + } + if (dryRun) { + const sourceRows = catalogSourceRows(candidate, review); + const options = Array.isArray(review.install_options) ? review.install_options : buildInstallOptions(validation.record, sourceRows); + if (options.length) installOptionsToRow(options, { catalogSources: sourceRows, checkedAt: new Date().toISOString() }); + const appId = await upsertApp(client, validation.record, review.app_id ?? null, true); + return { candidate_id: candidate.id, decision, app_id: appId, slug: validation.record.slug }; + } await ensureCategories(client, validation.record.categories); const appId = await upsertApp(client, validation.record, review.app_id ?? null); await replaceCategoryLinks(client, appId, validation.record.categories); @@ -361,24 +379,27 @@ async function applyReview(client, review) { return { candidate_id: candidate.id, decision, app_id: appId, slug: validation.record.slug }; } +export async function applyReviews(client, reviews, dryRun = false) { + const results = []; + const ids = reviews.map((review) => review.candidate_id); + if (new Set(ids).size !== ids.length) throw new Error("Repeated candidate_id in review batch."); + await client.query(dryRun ? "begin read only" : "begin"); + try { + for (const review of reviews) results.push(await applyReview(client, review, dryRun)); + await client.query(dryRun ? "rollback" : "commit"); + return results; + } catch (error) { + await client.query("rollback"); + throw error; + } +} + async function main() { const args = parseArgs(); + const databaseUrl = databaseOptions(args); if (!args.file) throw new Error("Pass --file with reviewed candidate decisions."); - const input = await readJson(args.file); - const reviews = reviewRecords(input); - const results = []; - - await withClient(async (client) => { - await client.query("begin"); - try { - for (const review of reviews) results.push(await applyReview(client, review)); - if (args["dry-run"]) await client.query("rollback"); - else await client.query("commit"); - } catch (error) { - await client.query("rollback"); - throw error; - } - }, args["database-url"]); + const reviews = reviewRecords(await readJson(args.file)); + const results = await withClient((client) => applyReviews(client, reviews, Boolean(args["dry-run"])), databaseUrl); console.log(JSON.stringify({ dry_run: Boolean(args["dry-run"]), results }, null, 2)); } diff --git a/scripts/self-host-apps/apply-resource-estimates.mjs b/scripts/self-host-apps/apply-resource-estimates.mjs index 436a813b..e945bef4 100644 --- a/scripts/self-host-apps/apply-resource-estimates.mjs +++ b/scripts/self-host-apps/apply-resource-estimates.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { pathToFileURL } from "node:url"; import { existsSync, readFileSync } from "node:fs"; import { parseArgs } from "./lib/args.mjs"; import { withClient } from "./lib/db.mjs"; @@ -7,6 +8,8 @@ import { resourceEstimatesToRow } from "./lib/streamlined-storage.mjs"; import { validateResourceEstimate } from "./lib/resource-review.mjs"; +import { databaseOptions } from "./lib/environment.mjs"; + function fail(message) { throw new Error(message); } @@ -30,118 +33,118 @@ function validateRecords(records) { }); } -async function main() { - const args = parseArgs(); - const records = validateRecords(readRecords(args.file)); - const apply = Boolean(args["apply-reviewed"]); +export async function applyResourceEstimates(client, records, apply = false) { const checkedAt = new Date().toISOString(); + const slugs = [...new Set(records.map((record) => record.slug))]; + const result = await client.query( + ` + select id, slug + from public.self_host_apps + where published = true + and self_hosting_status = 'self_hostable' + and slug = any($1::text[]) + `, + [slugs], + ); + const appIds = new Map(result.rows.map((row) => [row.slug, row.id])); + const missing = slugs.filter((slug) => !appIds.has(slug)); + if (missing.length > 0) fail(`Published self-hostable app not found: ${missing.join(", ")}`); - const report = await withClient(async (client) => { - const slugs = [...new Set(records.map((record) => record.slug))]; - const result = await client.query( - ` - select id, slug - from public.self_host_apps - where published = true - and self_hosting_status = 'self_hostable' - and slug = any($1::text[]) - `, - [slugs], - ); - const appIds = new Map(result.rows.map((row) => [row.slug, row.id])); - const missing = slugs.filter((slug) => !appIds.has(slug)); - if (missing.length > 0) fail(`Published self-hostable app not found: ${missing.join(", ")}`); - - const byApp = new Map(); - for (const record of records) { - const list = byApp.get(record.slug) ?? []; - list.push(record); - byApp.set(record.slug, list); - } + const byApp = new Map(); + for (const record of records) { + const list = byApp.get(record.slug) ?? []; + list.push(record); + byApp.set(record.slug, list); + } - if (apply) { - const columns = [ - "app_id", - "ram_estimate_min_mb", - "ram_estimate_max_mb", - "ram_estimate_kind", - "ram_estimate_confidence", - "ram_estimate_basis", - "ram_source_urls", - "ram_note", - "storage_estimate_min_gb", - "storage_estimate_max_gb", - "storage_estimate_kind", - "storage_estimate_confidence", - "storage_estimate_basis", - "storage_source_urls", - "storage_note", - "checked_at", - ]; - const recordTypes = [ - "uuid", - "numeric", "numeric", "text", "text", "text", "text[]", "text", - "numeric", "numeric", "text", "text", "text", "text[]", "text", - "timestamptz", - ]; - const storedRows = [...byApp].map(([slug, appRecords]) => ({ - app_id: appIds.get(slug), - ...resourceEstimatesToRow(appRecords, { checkedAt }), - })); - const facts = columns.filter(column => !['app_id', 'checked_at'].includes(column)); - const currentFacts = facts.map(column => `self_host_app_resource_estimates.${column}`); - const incomingFacts = facts.map((column, index) => column.endsWith('_urls') - ? `case when cardinality(excluded.${column}) > 0 then excluded.${column} else ${currentFacts[index]} end` - : `coalesce(excluded.${column}, ${currentFacts[index]})`); - await client.query("begin"); + if (apply) { + const columns = [ + "app_id", + "ram_estimate_min_mb", + "ram_estimate_max_mb", + "ram_estimate_kind", + "ram_estimate_confidence", + "ram_estimate_basis", + "ram_source_urls", + "ram_note", + "storage_estimate_min_gb", + "storage_estimate_max_gb", + "storage_estimate_kind", + "storage_estimate_confidence", + "storage_estimate_basis", + "storage_source_urls", + "storage_note", + "checked_at", + ]; + const recordTypes = [ + "uuid", + "numeric", "numeric", "text", "text", "text", "text[]", "text", + "numeric", "numeric", "text", "text", "text", "text[]", "text", + "timestamptz", + ]; + const storedRows = [...byApp].map(([slug, appRecords]) => ({ + app_id: appIds.get(slug), + ...resourceEstimatesToRow(appRecords, { checkedAt }), + })); + const facts = columns.filter(column => !['app_id', 'checked_at'].includes(column)); + const currentFacts = facts.map(column => `self_host_app_resource_estimates.${column}`); + const incomingFacts = facts.map((column, index) => column.endsWith('_urls') + ? `case when cardinality(excluded.${column}) > 0 then excluded.${column} else ${currentFacts[index]} end` + : `coalesce(excluded.${column}, ${currentFacts[index]})`); + await client.query("begin"); + try { + await client.query( + ` + insert into public.self_host_app_resource_estimates (${columns.join(", ")}) + select ${columns.map((column) => `item.${column}`).join(", ")} + from jsonb_to_recordset($1::jsonb) as item(${columns.map((column, index) => `${column} ${recordTypes[index]}`).join(", ")}) + on conflict (app_id) do update set + ram_estimate_min_mb = coalesce(excluded.ram_estimate_min_mb, self_host_app_resource_estimates.ram_estimate_min_mb), + ram_estimate_max_mb = coalesce(excluded.ram_estimate_max_mb, self_host_app_resource_estimates.ram_estimate_max_mb), + ram_estimate_kind = coalesce(excluded.ram_estimate_kind, self_host_app_resource_estimates.ram_estimate_kind), + ram_estimate_confidence = coalesce(excluded.ram_estimate_confidence, self_host_app_resource_estimates.ram_estimate_confidence), + ram_estimate_basis = coalesce(excluded.ram_estimate_basis, self_host_app_resource_estimates.ram_estimate_basis), + ram_source_urls = case when cardinality(excluded.ram_source_urls) > 0 then excluded.ram_source_urls else self_host_app_resource_estimates.ram_source_urls end, + ram_note = coalesce(excluded.ram_note, self_host_app_resource_estimates.ram_note), + storage_estimate_min_gb = coalesce(excluded.storage_estimate_min_gb, self_host_app_resource_estimates.storage_estimate_min_gb), + storage_estimate_max_gb = coalesce(excluded.storage_estimate_max_gb, self_host_app_resource_estimates.storage_estimate_max_gb), + storage_estimate_kind = coalesce(excluded.storage_estimate_kind, self_host_app_resource_estimates.storage_estimate_kind), + storage_estimate_confidence = coalesce(excluded.storage_estimate_confidence, self_host_app_resource_estimates.storage_estimate_confidence), + storage_estimate_basis = coalesce(excluded.storage_estimate_basis, self_host_app_resource_estimates.storage_estimate_basis), + storage_source_urls = case when cardinality(excluded.storage_source_urls) > 0 then excluded.storage_source_urls else self_host_app_resource_estimates.storage_source_urls end, + storage_note = coalesce(excluded.storage_note, self_host_app_resource_estimates.storage_note), + checked_at = excluded.checked_at, + updated_at = now() + where row(${currentFacts.join(', ')}) is distinct from row(${incomingFacts.join(', ')}) + `, + [JSON.stringify(storedRows)], + ); + await client.query("commit"); + } catch (error) { try { - await client.query( - ` - insert into public.self_host_app_resource_estimates (${columns.join(", ")}) - select ${columns.map((column) => `item.${column}`).join(", ")} - from jsonb_to_recordset($1::jsonb) as item(${columns.map((column, index) => `${column} ${recordTypes[index]}`).join(", ")}) - on conflict (app_id) do update set - ram_estimate_min_mb = coalesce(excluded.ram_estimate_min_mb, self_host_app_resource_estimates.ram_estimate_min_mb), - ram_estimate_max_mb = coalesce(excluded.ram_estimate_max_mb, self_host_app_resource_estimates.ram_estimate_max_mb), - ram_estimate_kind = coalesce(excluded.ram_estimate_kind, self_host_app_resource_estimates.ram_estimate_kind), - ram_estimate_confidence = coalesce(excluded.ram_estimate_confidence, self_host_app_resource_estimates.ram_estimate_confidence), - ram_estimate_basis = coalesce(excluded.ram_estimate_basis, self_host_app_resource_estimates.ram_estimate_basis), - ram_source_urls = case when cardinality(excluded.ram_source_urls) > 0 then excluded.ram_source_urls else self_host_app_resource_estimates.ram_source_urls end, - ram_note = coalesce(excluded.ram_note, self_host_app_resource_estimates.ram_note), - storage_estimate_min_gb = coalesce(excluded.storage_estimate_min_gb, self_host_app_resource_estimates.storage_estimate_min_gb), - storage_estimate_max_gb = coalesce(excluded.storage_estimate_max_gb, self_host_app_resource_estimates.storage_estimate_max_gb), - storage_estimate_kind = coalesce(excluded.storage_estimate_kind, self_host_app_resource_estimates.storage_estimate_kind), - storage_estimate_confidence = coalesce(excluded.storage_estimate_confidence, self_host_app_resource_estimates.storage_estimate_confidence), - storage_estimate_basis = coalesce(excluded.storage_estimate_basis, self_host_app_resource_estimates.storage_estimate_basis), - storage_source_urls = case when cardinality(excluded.storage_source_urls) > 0 then excluded.storage_source_urls else self_host_app_resource_estimates.storage_source_urls end, - storage_note = coalesce(excluded.storage_note, self_host_app_resource_estimates.storage_note), - checked_at = excluded.checked_at, - updated_at = now() - where row(${currentFacts.join(', ')}) is distinct from row(${incomingFacts.join(', ')}) - `, - [JSON.stringify(storedRows)], - ); - await client.query("commit"); - } catch (error) { - try { - await client.query("rollback"); - } catch (rollbackError) { - error.message = `${error.message}; rollback failed: ${rollbackError.message}`; - } - throw error; + await client.query("rollback"); + } catch (rollbackError) { + error.message = `${error.message}; rollback failed: ${rollbackError.message}`; } + throw error; } + } - return { - mode: apply ? "apply" : "dry-run", - records: records.length, - apps: byApp.size, - resource_counts: Object.fromEntries( - [...records.reduce((counts, record) => counts.set(record.resource_kind, (counts.get(record.resource_kind) ?? 0) + 1), new Map()).entries()], - ), - }; - }, args["database-url"]); + return { + mode: apply ? "apply" : "dry-run", + records: records.length, + apps: byApp.size, + resource_counts: Object.fromEntries( + [...records.reduce((counts, record) => counts.set(record.resource_kind, (counts.get(record.resource_kind) ?? 0) + 1), new Map()).entries()], + ), + }; +} +async function main() { + const args = parseArgs(); + const databaseUrl = databaseOptions(args); + const records = validateRecords(readRecords(args.file)); + const report = await withClient((client) => applyResourceEstimates(client, records, args["apply-reviewed"] === true), databaseUrl); console.log(JSON.stringify(report, null, 2)); } @@ -151,7 +154,7 @@ function columnCast(column) { return ""; } -main().catch((error) => { +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => { console.error(error.stack ?? error.message); process.exit(1); }); diff --git a/scripts/self-host-apps/lib/environment.mjs b/scripts/self-host-apps/lib/environment.mjs new file mode 100644 index 00000000..b2cc1982 --- /dev/null +++ b/scripts/self-host-apps/lib/environment.mjs @@ -0,0 +1,14 @@ +import { resolveDatabaseUrl } from "../../stack127/tool-pages/lib.mjs"; + +// Editorial commands always resolve from the named env file. NODE_ENV and an +// inherited DATABASE_URL cannot silently send a manual claim/review to production. +export function databaseOptions(args) { + for (const flag of ["dry-run", "apply-reviewed"]) { + if (args[flag] !== undefined && args[flag] !== true) throw new Error(`--${flag} is a flag and does not accept a value`); + } + if (args["dry-run"] && (args["apply-reviewed"] || args.apply)) throw new Error("Use --dry-run or an apply flag, not both."); + if (args.environment === "production" && args.confirm !== "stack127-production") { + throw new Error("Production requires --confirm stack127-production, including previews."); + } + return resolveDatabaseUrl({ ...args, environment: args.environment ?? "development" }); +} diff --git a/scripts/self-host-apps/refresh-github.mjs b/scripts/self-host-apps/refresh-github.mjs index dfb1bb49..5ebb4a08 100644 --- a/scripts/self-host-apps/refresh-github.mjs +++ b/scripts/self-host-apps/refresh-github.mjs @@ -514,9 +514,12 @@ export async function refreshGithub(client, args) { return { selected: rows.length, refreshed, failed: failures.length, failures, retryMetadataAvailable }; } +import { databaseOptions } from "./lib/environment.mjs"; + async function main() { const args = parseArgs(); - const report = await withClient((client) => refreshGithub(client, args), args["database-url"]); + if (args.slug !== undefined && (typeof args.slug !== "string" || !args.slug.trim())) throw new Error("--slug needs an app slug"); + const report = await withClient((client) => refreshGithub(client, args), args.environment ? databaseOptions(args) : args["database-url"]); if (refreshFailed(report)) process.exitCode = 1; } diff --git a/scripts/self-host-apps/refresh-install-options.mjs b/scripts/self-host-apps/refresh-install-options.mjs index 9d7d5559..80a2779f 100644 --- a/scripts/self-host-apps/refresh-install-options.mjs +++ b/scripts/self-host-apps/refresh-install-options.mjs @@ -43,8 +43,11 @@ async function upsertRows(client, rows) { ); } +import { databaseOptions } from "./lib/environment.mjs"; + async function main() { const args = parseArgs(); + if (args.slug !== undefined && (typeof args.slug !== "string" || !args.slug.trim())) throw new Error("--slug needs an app slug"); const limit = numberArg(args, "limit", 5000); const apply = Boolean(args.apply); const checkedAt = new Date().toISOString(); @@ -72,10 +75,11 @@ async function main() { left join public.self_host_app_install_options install on install.app_id = app.id where app.published = true and app.self_hosting_status = 'self_hostable' + and ($2::text is null or app.slug = $2) order by app.popularity_score desc, app.name asc limit $1 `, - [limit], + [limit, args.slug ?? null], ); const prepared = result.rows.map((row) => ({ @@ -107,7 +111,7 @@ async function main() { docker_source_file_links_excluded: prepared.filter((row) => row.docker_url && /\/(?:blob|tree)\/[^/]+\/(?:.+\/)?(?:Dockerfile|docker-compose[^/]*\.ya?ml|compose[^/]*\.ya?ml)/i.test(row.docker_url) && !row.options.some((option) => option.method === "docker" && option.url === row.docker_url)).length, sample: options.slice(0, 20).map(({ slug, method, label, url, image, support_level }) => ({ slug, method, label, url, image, support_level })), }; - }, args["database-url"]); + }, args.environment ? databaseOptions(args) : args["database-url"]); console.log(JSON.stringify(report, null, 2)); jobSummary("Install destinations", { Mode: report.mode, Apps: report.apps, "Apps with options": report.apps_with_options, Options: report.options }); diff --git a/scripts/self-host-apps/refresh-source-host.mjs b/scripts/self-host-apps/refresh-source-host.mjs index 72e3b0d7..fb0c8d2a 100644 --- a/scripts/self-host-apps/refresh-source-host.mjs +++ b/scripts/self-host-apps/refresh-source-host.mjs @@ -159,9 +159,14 @@ async function codebergFacts(row) { }; } -async function rowsToRefresh(client, host, args) { +export async function rowsToRefresh(client, host, args) { const params = [host]; const conditions = ["source_code_host = $1", "source_code_url is not null"]; + if (args.slug !== undefined) { + if (typeof args.slug !== "string" || !args.slug.trim()) throw new Error("--slug needs an app slug"); + params.push(args.slug); + conditions.push(`slug = $${params.length}`); + } if (args["published-only"]) conditions.push("published = true"); if (args["max-age-hours"]) { params.push(numberArg(args, "max-age-hours", 24)); @@ -267,9 +272,11 @@ export async function refreshSourceHost(client, args) { return report; } +import { databaseOptions } from "./lib/environment.mjs"; + async function main() { const args = parseArgs(); - const report = await withClient((client) => refreshSourceHost(client, args), args["database-url"]); + const report = await withClient((client) => refreshSourceHost(client, args), args.environment ? databaseOptions(args) : args["database-url"]); if (refreshFailed(report)) process.exitCode = 1; } diff --git a/scripts/self-host-apps/scripts.json b/scripts/self-host-apps/scripts.json index 686097d3..5aec3245 100644 --- a/scripts/self-host-apps/scripts.json +++ b/scripts/self-host-apps/scripts.json @@ -93,7 +93,7 @@ { "name": "self-host-apps:refresh-install-options", "command": "npm run self-host-apps:refresh-install-options -- --limit 5000", - "purpose": "Normalize Docker registry, Compose, and supported appliance-catalog destinations without treating Dockerfiles as install links." + "purpose": "Normalize reviewed install destinations; --slug selects one app, --apply writes, and explicit --environment uses the shared production confirmation gate." }, { "name": "self-host-apps:propose-resource-estimates", @@ -103,7 +103,7 @@ { "name": "self-host-apps:apply-resource-estimates", "command": "npm run self-host-apps:apply-resource-estimates -- --file --dry-run", - "purpose": "Apply reviewed derived or estimated RAM/storage ranges with confidence, evidence URLs, and notes while preserving exact published requirements." + "purpose": "Preview reviewed RAM/storage ranges; --apply-reviewed writes. Production requires --environment production --confirm stack127-production, including previews." }, { "name": "self-host-apps:dedupe-candidates", @@ -113,7 +113,7 @@ { "name": "self-host-apps:select-candidate-batch", "command": "npm run self-host-apps:select-candidate-batch -- --limit 100 --out scripts/self-host-apps/batches/YYYY-MM-DD-candidate-enrichment.json", - "purpose": "Atomically claim up to 100 queued or name-review candidates for Codex research." + "purpose": "Atomically claim ranked queued/name-review candidates, or exact IDs with --ids / --ids-from reviewed.json (also new and enriching). --match performs read-only lookup. Use --environment production --confirm stack127-production for production, including previews." }, { "name": "self-host-apps:release-candidate-batch", @@ -128,7 +128,7 @@ { "name": "self-host-apps:apply-candidate-review", "command": "npm run self-host-apps:apply-candidate-review -- --file --dry-run", - "purpose": "Validate and transactionally apply Codex-reviewed publish, duplicate, reject, or needs-review decisions." + "purpose": "Validate and transactionally apply reviewed decisions; --dry-run uses a read-only transaction. Production requires --environment production --confirm stack127-production, including previews." }, { "name": "self-host-apps:reset-categories", @@ -148,12 +148,12 @@ { "name": "self-host-apps:refresh-gitlab", "command": "npm run self-host-apps:refresh-gitlab -- --published-only --limit 500 --dry-run", - "purpose": "Refresh provider-neutral GitLab repository facts without changing curated content." + "purpose": "Refresh GitLab repository facts; --slug selects one app. Explicit --environment uses the shared production confirmation gate." }, { "name": "self-host-apps:refresh-codeberg", "command": "npm run self-host-apps:refresh-codeberg -- --published-only --limit 500 --dry-run", - "purpose": "Refresh provider-neutral Codeberg repository facts without changing curated content." + "purpose": "Refresh Codeberg repository facts; --slug selects one app. Explicit --environment uses the shared production confirmation gate." }, { "name": "self-host-apps:validate", @@ -163,7 +163,7 @@ { "name": "self-host-apps:upsert", "command": "npm run self-host-apps:upsert -- --file app.json --dry-run", - "purpose": "Upsert one reviewed self-host app with category links and compact source metadata." + "purpose": "Upsert one reviewed app when no candidate exists. Use --environment development, or production with --confirm stack127-production, and preview with --dry-run." }, { "name": "self-host-apps:upsert-batch", diff --git a/scripts/self-host-apps/select-candidate-batch.mjs b/scripts/self-host-apps/select-candidate-batch.mjs index a08e7405..c97cf946 100644 --- a/scripts/self-host-apps/select-candidate-batch.mjs +++ b/scripts/self-host-apps/select-candidate-batch.mjs @@ -1,10 +1,13 @@ #!/usr/bin/env node -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, writeFile, readFile } from "node:fs/promises"; import path from "node:path"; import { parseArgs, numberArg } from "./lib/args.mjs"; import { withClient } from "./lib/db.mjs"; +import { pathToFileURL } from "node:url"; +import { databaseOptions } from "./lib/environment.mjs"; + const claimableStatuses = new Set(["queued", "needs_review"]); function requestedStatuses(value) { @@ -49,16 +52,75 @@ async function loadPotentialMatches(client, candidates) { })); } +export async function requestedIds(args) { + const hasIds = args.ids !== undefined; + const hasFile = args["ids-from"] !== undefined; + if (hasIds && hasFile) throw new Error("Use --ids or --ids-from, not both."); + if (!hasIds && !hasFile) return null; + if (args.status !== undefined || args.limit !== undefined || args.match !== undefined) throw new Error("Exact IDs cannot be combined with --status, --limit or --match."); + let ids; + if (hasFile) { + const input = JSON.parse(await readFile(args["ids-from"], "utf8")); + const reviews = Array.isArray(input) ? input : input.records ?? input.reviews ?? [input]; + ids = reviews.map((review) => review.candidate_id); + } else { + ids = String(args.ids).split(",").map((id) => id.trim()); + } + if (!ids.length || ids.some((id) => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))) { + throw new Error("Every selected record must have a valid candidate UUID."); + } + return [...new Set(ids.map((id) => id.toLowerCase()))]; +} + +// Called inside the caller's transaction. Lock all requested rows before validating +// any of them; a missing or terminal row must leave the whole batch unchanged. +export async function claimIds(client, ids, dryRun = false) { + const result = await client.query( + "select * from public.self_host_app_candidates where id = any($1::uuid[]) order by id" + (dryRun ? "" : " for update"), [ids], + ); + const byId = new Map(result.rows.map((row) => [row.id, row])); + const missing = ids.filter((id) => !byId.has(id)); + if (missing.length) throw new Error(`Missing candidate IDs: ${missing.join(", ")}`); + const invalid = result.rows.filter((row) => !["new", "queued", "needs_review", "enriching"].includes(row.status)); + if (invalid.length) throw new Error(`Cannot claim candidates: ${invalid.map((row) => `${row.id} (${row.status})`).join(", ")}`); + const changes = result.rows.filter((row) => row.status !== "enriching"); + if (!dryRun && changes.length) { + await client.query("update public.self_host_app_candidates set status = 'enriching', updated_at = now() where id = any($1::uuid[])", [changes.map((row) => row.id)]); + } + return result.rows.map((row) => ({ ...row, previous_status: row.status, already_enriching: row.status === "enriching", status: dryRun ? row.status : "enriching" })); +} + +export async function findMatches(client, match) { + if (typeof match !== "string" || !match.trim()) throw new Error("--match needs a name or repository."); + const term = match.trim(); + // strpos uses a literal substring, so URL underscores and percent signs are not wildcards. + const candidates = await client.query(`select id, status, name, source, source_url, source_code_url, website_url, + duplicate_app_id, duplicate_candidate_id, duplicate_reason, enriched_app_id + from public.self_host_app_candidates + where strpos(lower(concat_ws(' ', name, slug, source_key, source_url, source_code_url, website_url)), lower($1)) > 0 order by name, id`, [term]); + const apps = await client.query(`select id, slug, name, published, self_hosting_status, source_code_url, website_url + from public.self_host_apps + where strpos(lower(concat_ws(' ', name, slug, source_code_url, website_url)), lower($1)) > 0 order by slug`, [term]); + return { candidates: candidates.rows, apps: apps.rows }; +} + async function main() { const args = parseArgs(); + const ids = await requestedIds(args); + const databaseUrl = databaseOptions(args); const limit = numberArg(args, "limit", 100); const statuses = requestedStatuses(args.status); const outPath = args.out ?? defaultOutputPath(); await withClient(async (client) => { - await client.query("begin"); + if (args.match !== undefined) { + if (args.ids !== undefined || args["ids-from"] !== undefined || args.status !== undefined || args.limit !== undefined) throw new Error("--match is a separate read-only lookup."); + console.log(JSON.stringify(await findMatches(client, args.match), null, 2)); + return; + } + await client.query(args["dry-run"] ? "begin read only" : "begin"); try { - const result = args["dry-run"] + const result = ids ? { rows: await claimIds(client, ids, Boolean(args["dry-run"])) } : args["dry-run"] ? await client.query( ` select * @@ -92,7 +154,9 @@ async function main() { source: "self_host_app_candidates", selected_at: new Date().toISOString(), claimed: !args["dry-run"], - selected_statuses: statuses, + selected_statuses: ids ? ["new", "queued", "needs_review", "enriching"] : statuses, + already_enriching: candidates.filter((row) => row.already_enriching).length, + would_claim: candidates.filter((row) => !row.already_enriching).length, candidate_count: candidates.length, candidates, }; @@ -103,15 +167,15 @@ async function main() { if (args["dry-run"]) await client.query("rollback"); else await client.query("commit"); - console.log(`${args["dry-run"] ? "Selected" : "Claimed"} ${candidates.length} candidate(s) in ${outPath}`); + console.log(`${args["dry-run"] ? "Selected" : "Claimed"} ${candidates.length} candidate(s) in ${outPath}; ${batch.already_enriching} already enriching`); } catch (error) { await client.query("rollback"); throw error; } - }, args["database-url"]); + }, databaseUrl); } -main().catch((error) => { +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => { console.error(error); process.exit(1); }); diff --git a/scripts/self-host-apps/upsert-app.mjs b/scripts/self-host-apps/upsert-app.mjs index 0217bda3..17ab2d9e 100644 --- a/scripts/self-host-apps/upsert-app.mjs +++ b/scripts/self-host-apps/upsert-app.mjs @@ -105,8 +105,11 @@ async function replaceCategoryLinks(client, appId, categories) { } } +import { databaseOptions } from "./lib/environment.mjs"; + async function main() { const args = parseArgs(); + const databaseUrl = databaseOptions(args); if (!args.file) { console.error("Usage: npm run self-host-apps:upsert -- --file app.json [--dry-run]"); process.exit(1); @@ -159,7 +162,7 @@ async function main() { await client.query("commit"); console.log(`Upserted: ${record.slug}`); } - }, args["database-url"]); + }, databaseUrl); } main().catch((error) => { diff --git a/skills/self-host-apps/enrichment/SKILL.md b/skills/self-host-apps/enrichment/SKILL.md index f59b5857..c238c5c9 100644 --- a/skills/self-host-apps/enrichment/SKILL.md +++ b/skills/self-host-apps/enrichment/SKILL.md @@ -22,7 +22,7 @@ Scripts move data. Codex makes editorial and factual decisions. Never generate s 6. Dry-run, then apply: `npm run self-host-apps:apply-candidate-review -- --file --dry-run` `npm run self-host-apps:apply-candidate-review -- --file ` -7. Run QA, summary, link, icon, typecheck, and build checks. +7. Run the relevant QA, summary, link, icon and typecheck checks. Do not build locally. 8. Repeat until no queued or review candidates remain. Release an abandoned claim with `self-host-apps:release-candidate-batch`. ## Decisions @@ -61,3 +61,18 @@ Don't build locally. If the batch changes code, the pull request CI runs the bui Report claimed, published, duplicate, rejected, needs-review, and failed counts. Published record changes move the affected directory dates and queue cache refreshes. Identical imports keep dates alone. App license-count changes also move the software license directory date. Use the page touch command for meaningful code facts outside the database triggers. See `stack127_docs/directory-pages.md`. + +## Add one app on request + +Follow [Add one app on request](../../../stack127_docs/self-host-apps-data.md#add-one-app-on-request) for the full commands. + +1. Use `self-host-apps:select-candidate-batch -- --environment development --match ` to find candidates and app records. Check production IDs too before preparing its review file. +2. Research official sources. Write the reviewed decisions and RAM/storage estimates, with source links and a managed icon. Review possible duplicates yourself. +3. Preview then claim exactly the review file's IDs with `--ids-from `; use `--dry-run` for the preview and `--out /tmp/app-claim.json` for the claim. Exact selection accepts `new`, `queued`, `needs_review` and already `enriching`. `new` has not passed dedupe; do not skip duplicate research. Top-N selection remains unchanged. +4. Dry-run and apply the review in development. Dry-run and apply estimates with `--apply-reviewed`. Check the app on the real directory at desktop and mobile width. +5. Refresh only its repository with `--slug ` on `refresh-github`, `refresh-gitlab` or `refresh-codeberg`. +6. After production authorization, repeat previews and applies from homelab with `--environment production --confirm stack127-production`. MacBook uploads to production are very slow. Check the live directory after its cache refresh. + +If no candidate exists, use the existing `self-host-apps:upsert -- --file ` path, first with `--dry-run`. Supply the complete reviewed app object with `published: true`, not a review wrapper. Apply estimates, then preview/apply `self-host-apps:refresh-install-options -- --slug ` (`--apply` writes). Pass the chosen environment on every command. No new candidate or ad hoc SQL is needed. + +Do not use `sync-local-to-prod`: it mirrors whole tables and would push about 5,000 unrelated development differences. Editorial commands default to development regardless of `NODE_ENV`; production previews also require the confirmation. Candidate-review dry runs use a read-only transaction. Estimates require `--apply-reviewed`; never combine it with `--dry-run`. diff --git a/stack127_docs/self-host-apps-data.md b/stack127_docs/self-host-apps-data.md index 38200740..c6aeaa5f 100644 --- a/stack127_docs/self-host-apps-data.md +++ b/stack127_docs/self-host-apps-data.md @@ -134,3 +134,31 @@ npm run self-host-apps:apply-artifact-cleanup -- --environment production --conf Review the production preview before running the apply command. The migration `20260901060529_streamline_self_host_apps_data_model.sql` folds the old rows into these tables. It removes pipeline-only tables and compatibility views and keeps the candidate review queue. + +## Add one app on request + +This adds records to `/self-host-apps`. It does not create a route or page. + +1. Find existing records first: `npm run self-host-apps:select-candidate-batch -- --environment development --match forgejo`. A name or repository substring lists candidate IDs, statuses, source and duplicate links, plus app IDs, slugs, publication and self-hosting status. Repeat the lookup in production before preparing that batch; app IDs must match that database. +2. Research the official website, repository, license, install docs and resource guidance. Write reviewed decisions and RAM/storage estimates using the enrichment skill formats. Include a managed icon, official evidence links, verified self-hosting and source model, and one to three controlled categories. Review every possible duplicate. Do not copy directory claims as verified facts. +3. Claim only the candidate IDs in the review file, preview first: + + ```bash + npm run self-host-apps:select-candidate-batch -- --environment development --ids-from --dry-run --out /tmp/app-claim-preview.json + npm run self-host-apps:select-candidate-batch -- --environment development --ids-from --out /tmp/app-claim.json + npm run self-host-apps:apply-candidate-review -- --environment development --file --dry-run + npm run self-host-apps:apply-candidate-review -- --environment development --file + npm run self-host-apps:apply-resource-estimates -- --environment development --file --dry-run + npm run self-host-apps:apply-resource-estimates -- --environment development --file --apply-reviewed + ``` + +4. Refresh only that app's repository facts: `self-host-apps:refresh-github -- --slug ` for GitHub, `self-host-apps:refresh-gitlab -- --slug ` for GitLab, or `self-host-apps:refresh-codeberg -- --slug ` for Codeberg. Pass `--environment development`. Check the real directory at desktop and mobile width, including the app's icon, install links, estimates and filters. +5. After the owner authorizes production changes, run the same claim preview, claim, review preview, review apply, estimate preview, estimate apply and per-app refresh on homelab. Replace `--environment development` with `--environment production --confirm stack127-production` on every command. Use homelab because uploading to production from the MacBook is very slow. Check the live directory after cache refresh completes. + +`new` means discovery has not run duplicate checks. `queued` means dedupe found no match; `needs_review` often means a possible name match. Exact-ID manual selection accepts all three because the reviewer checks duplicates. It also reports already `enriching` rows and leaves them alone. Missing, published, rejected or duplicate IDs stop the whole claim. `--ids ` is also supported. Claiming uses one transaction and touches no other candidates. Top-N selection still only claims `queued` and `needs_review`; it does not bypass dedupe. Re-running a claim is safe while rows are enriching. After reviews finish, use lookup rather than reclaiming terminal rows. + +Claims, candidate reviews, resource estimates and single-app upserts default to development and use the shared environment resolver. They ignore `NODE_ENV` and inherited `DATABASE_URL`, reject `--database-url`, and require the production confirmation even for previews. Review application still writes unless `--dry-run` is present; estimates still need `--apply-reviewed` to write. Those flags cannot be combined. Candidate review previews use a read-only transaction; claim and estimate previews issue no data writes. A preview validates inputs and references but cannot prove that every write-time constraint or trigger will pass. + +If there is no candidate, use the existing single-record path: put the same complete, officially researched app facts in `` with `published: true`, then run `npm run self-host-apps:upsert -- --environment development --file --dry-run` and repeat without `--dry-run`. This file is the app object, not a candidate-review wrapper. Find first: upsert merges by slug. No candidate ID or raw SQL is needed. Fill estimates with the commands above. Preview install destinations with `npm run self-host-apps:refresh-install-options -- --environment development --slug ` and repeat with `--apply`. Then check the directory and repeat this single-app sequence on homelab with the production flags after authorization. + +Do not use `sync-local-to-prod` for this. It mirrors whole tables and would send about 5,000 unrelated development differences along with the requested apps. diff --git a/tests/self-host-apps/chosen-candidates.test.mjs b/tests/self-host-apps/chosen-candidates.test.mjs new file mode 100644 index 00000000..04b759ea --- /dev/null +++ b/tests/self-host-apps/chosen-candidates.test.mjs @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { claimIds, requestedIds, findMatches } from "../../scripts/self-host-apps/select-candidate-batch.mjs"; +import { applyReviews } from "../../scripts/self-host-apps/apply-candidate-review.mjs"; +import { applyResourceEstimates } from "../../scripts/self-host-apps/apply-resource-estimates.mjs"; +import { databaseOptions } from "../../scripts/self-host-apps/lib/environment.mjs"; +import { rowsToRefresh } from "../../scripts/self-host-apps/refresh-source-host.mjs"; + +const id = (n) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +function candidateClient(rows) { + const writes = []; + return { writes, async query(sql, params) { + if (sql.startsWith("select")) return { rows: rows.filter((row) => params[0].includes(row.id)).map((row) => ({ ...row })) }; + writes.push(params[0]); + for (const row of rows) if (params[0].includes(row.id)) row.status = "enriching"; + return { rows: [] }; + } }; +} + +test("missing ID stops the entire claim before any write", async () => { + const rows = [{ id: id(1), status: "queued" }]; + const client = candidateClient(rows); + await assert.rejects(claimIds(client, [id(1), id(2)]), /Missing candidate/); + assert.deepEqual(client.writes, []); + assert.equal(rows[0].status, "queued"); +}); +for (const status of ["published", "rejected", "duplicate", "unexpected"]) { + test(`${status} stops the entire claim`, async () => { + const client = candidateClient([{ id: id(1), status: "new" }, { id: id(2), status }]); + await assert.rejects(claimIds(client, [id(1), id(2)]), /Cannot claim/); + assert.deepEqual(client.writes, []); + }); +} + +test("exact claim accepts new and already enriching, never touches unrelated rows, and can repeat", async () => { + const rows = ["new", "needs_review", "enriching", "queued"].map((status, n) => ({ id: id(n + 1), status })); + const client = candidateClient(rows); + const result = await claimIds(client, [id(1), id(2), id(3)]); + assert.equal(result.filter((row) => row.already_enriching).length, 1); + assert.deepEqual(client.writes, [[id(1), id(2)]]); + assert.equal(rows[3].status, "queued"); + assert.equal((await claimIds(client, [id(1), id(2), id(3)])).every((row) => row.already_enriching), true); + assert.equal(client.writes.length, 1); +}); + +test("claim dry run reports original statuses and writes nothing", async () => { + const client = candidateClient([{ id: id(1), status: "new" }]); + assert.equal((await claimIds(client, [id(1)], true))[0].previous_status, "new"); + assert.deepEqual(client.writes, []); +}); + +test("IDs come from every review decision, with strict validation", async () => { + const file = new URL("../../scripts/self-host-apps/batches/2026-11-01-forge-error-tracking-reviewed.json", import.meta.url); + const ids = await requestedIds({ "ids-from": file }); + assert.equal(ids.length, 10); + await assert.rejects(requestedIds({ ids: id(1), "ids-from": file }), /not both/); + await assert.rejects(requestedIds({ ids: id(1), limit: "1" }), /cannot be combined/); + await assert.rejects(requestedIds({ ids: "not-an-id" }), /UUID/); + await assert.rejects(requestedIds({ ids: "" }), /UUID/); + assert.deepEqual(await requestedIds({ ids: `${id(1)},${id(1)}` }), [id(1)]); +}); + +test("production gate ignores inherited NODE_ENV and DATABASE_URL and requires confirmation even for previews", () => { + const previous = process.cwd(); + const dir = mkdtempSync(join(tmpdir(), "self-host-target-")); + const oldNodeEnv = process.env.NODE_ENV; + const oldUrl = process.env.DATABASE_URL; + try { + writeFileSync(join(dir, ".env.local"), "DATABASE_URL=postgres://development/db\n"); + writeFileSync(join(dir, ".env.database"), "DATABASE_URL=postgres://production/db\n"); + process.chdir(dir); + process.env.NODE_ENV = "production"; + process.env.DATABASE_URL = "postgres://unintended/db"; + assert.equal(databaseOptions({}), "postgres://development/db"); + for (const flags of [{}, { "dry-run": true }, { "apply-reviewed": true }]) { + assert.throws(() => databaseOptions({ environment: "production", ...flags }), /stack127-production/); + assert.equal(databaseOptions({ environment: "production", confirm: "stack127-production", ...flags }), "postgres://production/db"); + } + assert.throws(() => databaseOptions({ "database-url": "postgres://override/db" }), /not supported/); + assert.throws(() => databaseOptions({ "dry-run": true, "apply-reviewed": true }), /not both/); + assert.throws(() => databaseOptions({ "dry-run": "false" }), /flag/); + } finally { + process.chdir(previous); + if (oldNodeEnv === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = oldNodeEnv; + if (oldUrl === undefined) delete process.env.DATABASE_URL; else process.env.DATABASE_URL = oldUrl; + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("review dry run checks all ten reviewed records using only reads", async () => { + const reviews = JSON.parse(readFileSync(new URL("../../scripts/self-host-apps/batches/2026-11-01-forge-error-tracking-reviewed.json", import.meta.url))).records; + const queries = []; + const client = { async query(sql, params) { + queries.push(sql); + if (["begin read only", "rollback"].includes(sql)) return { rows: [] }; + assert.match(sql, /^select/i); + assert.doesNotMatch(sql, /for update/i); + if (sql.includes("self_host_app_candidates")) return { rows: [{ id: params[0], status: "enriching" }] }; + const review = reviews.find((row) => row.app?.slug === params[0]); + return { rows: [{ id: review?.app_id ?? params[0] }] }; + } }; + assert.equal((await applyReviews(client, reviews, true)).length, 10); + assert.equal(queries[0], "begin read only"); + assert.equal(queries.at(-1), "rollback"); +}); + +test("review preview rejects unclaimed candidates and rolls back", async () => { + const queries = []; + const client = { async query(sql) { + queries.push(sql); + return { rows: [{ id: id(1), status: "new" }] }; + } }; + await assert.rejects(applyReviews(client, [{ candidate_id: id(1), decision: "reject", reason: "test" }], true), /claim it/); + assert.equal(queries.at(-1), "rollback"); +}); + +test("estimate preview performs only a lookup", async () => { + const records = [{ slug: "example", resource_kind: "ram" }]; + const client = { async query(sql) { + assert.match(sql.trim(), /^select/i); + return { rows: [{ id: id(1), slug: "example" }] }; + } }; + assert.equal((await applyResourceEstimates(client, records, false)).mode, "dry-run"); +}); + +test("find is read only and matches names and repositories literally", async () => { + const queries = []; + const client = { async query(sql, params) { + assert.match(sql.trim(), /^select/i); + assert.deepEqual(params, ["org/a_b"]); + queries.push(sql); + return { rows: [{ id: id(1) }] }; + } }; + const matches = await findMatches(client, "org/a_b"); + assert.equal(matches.candidates.length, 1); + assert.equal(matches.apps.length, 1); + assert.equal(queries.length, 2); +}); + +test("source-host refresh scopes a single slug with a bound parameter", async () => { + const client = { async query(sql, params) { + assert.match(sql, /slug = \$2/); + assert.deepEqual(params, ["gitlab", "glitchtip", 500]); + return { rows: [] }; + } }; + await rowsToRefresh(client, "gitlab", { slug: "glitchtip" }); +}); + + test("empty lookup and slug flags cannot fall back to broad operations", async () => { + const client = { query() { throw new Error("Must reject before querying"); } }; + await assert.rejects(findMatches(client, ""), /needs a name/); + await assert.rejects(rowsToRefresh(client, "codeberg", { slug: "" }), /needs an app slug/); +}); From c25c52953d4d9b5e293f9e8d31f520db9da471b1 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Mon, 2 Nov 2026 15:11:13 +0530 Subject: [PATCH 2/4] Protect manual claims from dedupe and fail clearly on a wrong-host refresh Dedupe rechecks and locks eligible rows before writing, so it can't overwrite a claimed or published candidate. A targeted source refresh now fails for a missing app or the wrong host. Releasing a claim uses the shared environment gate. Co-Authored-By: Claude Opus 5.5 --- content_docs/self-host-apps/data-workflow.md | 8 +- scripts/self-host-apps/README.md | 4 +- scripts/self-host-apps/dedupe-candidates.mjs | 38 ++++++++- .../self-host-apps/refresh-source-host.mjs | 6 ++ .../release-candidate-batch.mjs | 4 +- scripts/self-host-apps/scripts.json | 12 +-- skills/self-host-apps/enrichment/SKILL.md | 14 ++-- stack127_docs/self-host-apps-data.md | 6 +- .../self-host-apps/chosen-candidates.test.mjs | 46 ++++++++++ .../self-host-apps/dedupe-claim-race.test.mjs | 83 +++++++++++++++++++ 10 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 tests/self-host-apps/dedupe-claim-race.test.mjs diff --git a/content_docs/self-host-apps/data-workflow.md b/content_docs/self-host-apps/data-workflow.md index 9f75a367..3c7cd9ca 100644 --- a/content_docs/self-host-apps/data-workflow.md +++ b/content_docs/self-host-apps/data-workflow.md @@ -12,7 +12,7 @@ recurring discovery -> provider refresh ``` -Production is canonical. Local is a disposable mirror for development and QA. +Production is canonical. Local is a disposable mirror for development and QA. Claim, review, release, estimate and single-app upsert examples target development explicitly. For authorized production work on homelab, replace `--environment development` with `--environment production --confirm stack127-production`. These commands ignore `NODE_ENV` and inherited `DATABASE_URL` and reject `--database-url`. Discovery and scheduled refresh commands keep their existing environment behavior. ## Discovery @@ -31,12 +31,14 @@ npm run self-host-apps:discover-app-catalogs -- --catalog all --limit 5000 --wri ```bash npm run self-host-apps:dedupe-candidates -- --dry-run npm run self-host-apps:dedupe-candidates -npm run self-host-apps:select-candidate-batch -- --limit 100 --out +npm run self-host-apps:select-candidate-batch -- --environment development --limit 100 --out npm run self-host-apps:collect-github-evidence -- --file ``` Selection uses `FOR UPDATE SKIP LOCKED` and marks claimed rows `enriching`, so two workers do not take the same work. Reviewers use `references/review-format.md`. Put supporting links in `sources` and `data_source_urls`; use `technical_fact_status` for checked-but-unpublished technical facts. There is no field-source table. +Before a review commits, claiming the same IDs again is safe. Release an abandoned claim with `npm run self-host-apps:release-candidate-batch -- --environment development --file --to needs_review`; production requires the production environment and confirmation above. After the review commits, resume at estimates without reclaiming or reapplying the review. Estimates and repository refreshes can be re-run. + The legacy `self-host-apps:reset-categories` command is retired and exits without database access. It previously replaced all category links using imported labels and keyword matches. Existing categories need source review too. Correct them per app with `apply-qa-overrides.mjs`, `review_scope: "partial"`, expected categories, and official evidence URLs. Do not infer purpose from incidental words such as image, model, or server. The review applier validates and moves data. Publication requires an official identity, a managed icon in `stack127-media`, one to three controlled categories, a verified self-hosting classification, install documentation, and at least one source URL. Open and source-available apps need a verified repository and license. Closed-source apps need an official website and publisher identity, but no repository. @@ -59,7 +61,7 @@ Estimate RAM and storage for every published app for a typical small self-hosted ```bash npm run self-host-apps:propose-resource-estimates -- --limit 5000 --out -npm run self-host-apps:apply-resource-estimates -- --file --apply-reviewed +npm run self-host-apps:apply-resource-estimates -- --environment development --file --apply-reviewed ``` An estimate record needs a range, kind, confidence, a short internal basis, and a link to the official project or publisher page. Use `review.method: "app_estimate"` for a practical estimate. Keep published requirements as source evidence for the estimate, not as a separate public RAM or storage label. Do not use a package memory cap as an app requirement. diff --git a/scripts/self-host-apps/README.md b/scripts/self-host-apps/README.md index f0cceee2..ad68d75b 100644 --- a/scripts/self-host-apps/README.md +++ b/scripts/self-host-apps/README.md @@ -24,7 +24,7 @@ Discovery stores raw evidence. It cannot publish or assign final categories, sum - `select-candidate-batch.mjs` atomically claims queued and name-review rows by rank, or exact IDs with `--ids` / `--ids-from `. `--match ` is read-only lookup. - `collect-github-evidence.mjs` collects official API evidence without editing apps. - `apply-candidate-review.mjs` validates reviewed decisions and writes them transactionally. -- `release-candidate-batch.mjs` releases an abandoned claim. +- `release-candidate-batch.mjs` releases only abandoned enriching claims, through the shared environment and production confirmation gate. `apply-candidate-review.mjs` requires source links for published records. Unknown values stay `null` or `unknown`; checked-but-unpublished facts use `technical_fact_status`. @@ -39,6 +39,6 @@ Discovery stores raw evidence. It cannot publish or assign final categories, sum Production is canonical. `sync-prod-to-local.mjs` rebuilds the disposable local mirror. `sync-local-to-prod.mjs` is an exceptional guarded upsert, requires `--confirm-reviewed-push`, and does not draft missing production rows unless `--draft-missing` is also explicit. -For targeted additions, follow “Add one app on request” in `stack127_docs/self-host-apps-data.md`. Do not use `sync-local-to-prod` for single apps. Editorial commands use `--environment development` (default) or `--environment production --confirm stack127-production`, including production previews. GitLab/Codeberg and install-option refreshes accept `--slug`; scheduled refresh callers keep their existing environment behavior. +For targeted additions, follow “Add one app on request” in `stack127_docs/self-host-apps-data.md`. Do not use `sync-local-to-prod` for single apps. Editorial commands (including claim release) use `--environment development` (default) or `--environment production --confirm stack127-production`, including production previews. GitLab/Codeberg and install-option refreshes accept `--slug`; scheduled refresh callers keep their existing environment behavior. See `content_docs/self-host-apps/data-workflow.md` and `scripts/self-host-apps/scripts.json`. diff --git a/scripts/self-host-apps/dedupe-candidates.mjs b/scripts/self-host-apps/dedupe-candidates.mjs index 606e0f93..fce847c6 100644 --- a/scripts/self-host-apps/dedupe-candidates.mjs +++ b/scripts/self-host-apps/dedupe-candidates.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { pathToFileURL } from "node:url"; import { parseArgs, numberArg } from "./lib/args.mjs"; import { withClient } from "./lib/db.mjs"; import { @@ -54,12 +55,35 @@ async function preserveExactAppSources(client, decisions) { ); } -async function applyDecisions(client, { +export async function applyDecisions(client, { appDuplicates, candidateDuplicates, nameReviews, queuedIds, + expectedCandidates, }) { + // Discovery reads can become stale while a reviewer claims/publishes a row. + // Recheck the original state under locks before *any* write, including source + // preservation, and hold these locks until the caller commits or rolls back. + const result = await client.query( + `select candidate.id + from public.self_host_app_candidates candidate + join jsonb_to_recordset($1::jsonb) as expected(id uuid, status text, duplicate_reason text) + on candidate.id = expected.id + where candidate.status = expected.status + and candidate.duplicate_reason is not distinct from expected.duplicate_reason + and (candidate.status = 'new' or + (candidate.status = 'needs_review' and candidate.duplicate_reason = 'possible_name_match')) + order by candidate.id + for update of candidate`, + [JSON.stringify(expectedCandidates.map(({ id, status, duplicate_reason }) => ({ id, status, duplicate_reason: duplicate_reason ?? null })))], + ); + const eligible = new Set(result.rows.map((row) => row.id)); + const skipped = expectedCandidates.length - eligible.size; + appDuplicates = appDuplicates.filter((decision) => eligible.has(decision.target_id)); + candidateDuplicates = candidateDuplicates.filter((decision) => eligible.has(decision.target_id)); + nameReviews = nameReviews.filter((decision) => eligible.has(decision.target_id)); + queuedIds = queuedIds.filter((id) => eligible.has(id)); await preserveExactAppSources(client, appDuplicates); if (appDuplicates.length > 0) { @@ -136,6 +160,14 @@ async function applyDecisions(client, { [queuedIds], ); } + return { + skipped_changed: skipped, + queued: queuedIds.length, + app_duplicates: appDuplicates.length, + candidate_duplicates: candidateDuplicates.length, + name_reviews: nameReviews.length, + sources_preserved: appDuplicates.length, + }; } async function main() { @@ -240,7 +272,7 @@ async function main() { if (args["dry-run"]) { await client.query("rollback"); } else { - await applyDecisions(client, { appDuplicates, candidateDuplicates, nameReviews, queuedIds }); + Object.assign(counts, await applyDecisions(client, { appDuplicates, candidateDuplicates, nameReviews, queuedIds, expectedCandidates: targetsResult.rows })); await client.query("commit"); } } catch (error) { @@ -252,7 +284,7 @@ async function main() { }, args["database-url"]); } -main().catch((error) => { +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => { console.error(error); process.exit(1); }); diff --git a/scripts/self-host-apps/refresh-source-host.mjs b/scripts/self-host-apps/refresh-source-host.mjs index fb0c8d2a..29d24ac2 100644 --- a/scripts/self-host-apps/refresh-source-host.mjs +++ b/scripts/self-host-apps/refresh-source-host.mjs @@ -164,6 +164,12 @@ export async function rowsToRefresh(client, host, args) { const conditions = ["source_code_host = $1", "source_code_url is not null"]; if (args.slug !== undefined) { if (typeof args.slug !== "string" || !args.slug.trim()) throw new Error("--slug needs an app slug"); + const app = await client.query("select source_code_host, source_code_url from public.self_host_apps where slug = $1", [args.slug]); + if (!app.rows[0]) throw new Error(`App not found: ${args.slug}`); + if (app.rows[0].source_code_host !== host) { + throw new Error(`App ${args.slug} belongs to host ${app.rows[0].source_code_host}, not ${host}`); + } + if (!app.rows[0].source_code_url) throw new Error(`App ${args.slug} has no repository URL`); params.push(args.slug); conditions.push(`slug = $${params.length}`); } diff --git a/scripts/self-host-apps/release-candidate-batch.mjs b/scripts/self-host-apps/release-candidate-batch.mjs index 837cdda1..7262e7cb 100644 --- a/scripts/self-host-apps/release-candidate-batch.mjs +++ b/scripts/self-host-apps/release-candidate-batch.mjs @@ -1,11 +1,13 @@ #!/usr/bin/env node +import { databaseOptions } from "./lib/environment.mjs"; import { parseArgs } from "./lib/args.mjs"; import { withClient } from "./lib/db.mjs"; import { readJson } from "./lib/schema.mjs"; async function main() { const args = parseArgs(); + const databaseUrl = databaseOptions(args); if (!args.file) throw new Error("Pass --file with a claimed candidate batch."); const targetStatus = args.to ?? "needs_review"; if (!new Set(["queued", "needs_review"]).has(targetStatus)) { @@ -28,7 +30,7 @@ async function main() { [ids, targetStatus], ); console.log(`Released ${result.rowCount} candidate(s) to ${targetStatus}.`); - }, args["database-url"]); + }, databaseUrl); } main().catch((error) => { diff --git a/scripts/self-host-apps/scripts.json b/scripts/self-host-apps/scripts.json index 5aec3245..df918c92 100644 --- a/scripts/self-host-apps/scripts.json +++ b/scripts/self-host-apps/scripts.json @@ -102,7 +102,7 @@ }, { "name": "self-host-apps:apply-resource-estimates", - "command": "npm run self-host-apps:apply-resource-estimates -- --file --dry-run", + "command": "npm run self-host-apps:apply-resource-estimates -- --environment development --file --dry-run", "purpose": "Preview reviewed RAM/storage ranges; --apply-reviewed writes. Production requires --environment production --confirm stack127-production, including previews." }, { @@ -112,13 +112,13 @@ }, { "name": "self-host-apps:select-candidate-batch", - "command": "npm run self-host-apps:select-candidate-batch -- --limit 100 --out scripts/self-host-apps/batches/YYYY-MM-DD-candidate-enrichment.json", + "command": "npm run self-host-apps:select-candidate-batch -- --environment development --limit 100 --out scripts/self-host-apps/batches/YYYY-MM-DD-candidate-enrichment.json", "purpose": "Atomically claim ranked queued/name-review candidates, or exact IDs with --ids / --ids-from reviewed.json (also new and enriching). --match performs read-only lookup. Use --environment production --confirm stack127-production for production, including previews." }, { "name": "self-host-apps:release-candidate-batch", - "command": "npm run self-host-apps:release-candidate-batch -- --file --to needs_review", - "purpose": "Release an abandoned enriching batch without losing candidate evidence." + "command": "npm run self-host-apps:release-candidate-batch -- --environment development --file --to needs_review", + "purpose": "Release an abandoned enriching batch without losing evidence; completed rows stay unchanged. Defaults to development. Production requires --environment production --confirm stack127-production." }, { "name": "self-host-apps:collect-github-evidence", @@ -127,7 +127,7 @@ }, { "name": "self-host-apps:apply-candidate-review", - "command": "npm run self-host-apps:apply-candidate-review -- --file --dry-run", + "command": "npm run self-host-apps:apply-candidate-review -- --environment development --file --dry-run", "purpose": "Validate and transactionally apply reviewed decisions; --dry-run uses a read-only transaction. Production requires --environment production --confirm stack127-production, including previews." }, { @@ -162,7 +162,7 @@ }, { "name": "self-host-apps:upsert", - "command": "npm run self-host-apps:upsert -- --file app.json --dry-run", + "command": "npm run self-host-apps:upsert -- --environment development --file app.json --dry-run", "purpose": "Upsert one reviewed app when no candidate exists. Use --environment development, or production with --confirm stack127-production, and preview with --dry-run." }, { diff --git a/skills/self-host-apps/enrichment/SKILL.md b/skills/self-host-apps/enrichment/SKILL.md index c238c5c9..3df98b68 100644 --- a/skills/self-host-apps/enrichment/SKILL.md +++ b/skills/self-host-apps/enrichment/SKILL.md @@ -5,7 +5,7 @@ description: Research and review stack127 self-host app candidates in claimed ba # Stack127 self-host enrichment -Work in `/home/teja/projects/stack127`. Production is canonical; refresh development from production before a long run. +Work in the assigned checkout. Production is canonical; refresh development from production before a long run. The commands below explicitly target development. For authorized production work on homelab, replace `--environment development` with `--environment production --confirm stack127-production`; `NODE_ENV=production` no longer chooses production for these editorial commands. Scripts move data. Codex makes editorial and factual decisions. Never generate summaries, categories, self-hosting status, surfaces, or resource facts from keywords. @@ -14,16 +14,16 @@ Scripts move data. Codex makes editorial and factual decisions. Never generate s 1. Read `content_docs/self-host-apps/data-workflow.md`, `content-rules.md`, and `schema.md`. Read `stack127_docs/writing-style.md` for app summaries and other public wording. 2. Claim up to 100 candidates: - `npm run self-host-apps:select-candidate-batch -- --limit 100 --out ` + `npm run self-host-apps:select-candidate-batch -- --environment development --limit 100 --out ` 3. For GitHub candidates, collect API evidence: `npm run self-host-apps:collect-github-evidence -- --file ` 4. Research every candidate from official source code, website, docs, release, package, and container pages. Treat directory copy as a lead only. 5. Write reviewed decisions using [review-format.md](references/review-format.md). 6. Dry-run, then apply: - `npm run self-host-apps:apply-candidate-review -- --file --dry-run` - `npm run self-host-apps:apply-candidate-review -- --file ` + `npm run self-host-apps:apply-candidate-review -- --environment development --file --dry-run` + `npm run self-host-apps:apply-candidate-review -- --environment development --file ` 7. Run the relevant QA, summary, link, icon and typecheck checks. Do not build locally. -8. Repeat until no queued or review candidates remain. Release an abandoned claim with `self-host-apps:release-candidate-batch`. +8. Repeat until no queued or review candidates remain. Before the review commits, release an abandoned claim with `npm run self-host-apps:release-candidate-batch -- --environment development --file --to needs_review`. Use the production environment and confirmation for a production claim. ## Decisions @@ -73,6 +73,8 @@ Follow [Add one app on request](../../../stack127_docs/self-host-apps-data.md#ad 5. Refresh only its repository with `--slug ` on `refresh-github`, `refresh-gitlab` or `refresh-codeberg`. 6. After production authorization, repeat previews and applies from homelab with `--environment production --confirm stack127-production`. MacBook uploads to production are very slow. Check the live directory after its cache refresh. -If no candidate exists, use the existing `self-host-apps:upsert -- --file ` path, first with `--dry-run`. Supply the complete reviewed app object with `published: true`, not a review wrapper. Apply estimates, then preview/apply `self-host-apps:refresh-install-options -- --slug ` (`--apply` writes). Pass the chosen environment on every command. No new candidate or ad hoc SQL is needed. +If no candidate exists, use the existing `self-host-apps:upsert -- --environment development --file ` path, first with `--dry-run`. Supply the complete reviewed app object with `published: true`, not a review wrapper. Apply estimates, then preview/apply `self-host-apps:refresh-install-options -- --slug ` (`--apply` writes). Pass the chosen environment on every command. No new candidate or ad hoc SQL is needed. Do not use `sync-local-to-prod`: it mirrors whole tables and would push about 5,000 unrelated development differences. Editorial commands default to development regardless of `NODE_ENV`; production previews also require the confirmation. Candidate-review dry runs use a read-only transaction. Estimates require `--apply-reviewed`; never combine it with `--dry-run`. + +Recovery: before the review commits, claiming the same IDs again is safe. After the review commits, resume at estimates; do not reclaim or reapply that review. Estimates and repository refreshes can be re-run. Release only abandoned enriching claims; release leaves completed candidates alone. diff --git a/stack127_docs/self-host-apps-data.md b/stack127_docs/self-host-apps-data.md index c6aeaa5f..af9660bf 100644 --- a/stack127_docs/self-host-apps-data.md +++ b/stack127_docs/self-host-apps-data.md @@ -155,10 +155,12 @@ This adds records to `/self-host-apps`. It does not create a route or page. 4. Refresh only that app's repository facts: `self-host-apps:refresh-github -- --slug ` for GitHub, `self-host-apps:refresh-gitlab -- --slug ` for GitLab, or `self-host-apps:refresh-codeberg -- --slug ` for Codeberg. Pass `--environment development`. Check the real directory at desktop and mobile width, including the app's icon, install links, estimates and filters. 5. After the owner authorizes production changes, run the same claim preview, claim, review preview, review apply, estimate preview, estimate apply and per-app refresh on homelab. Replace `--environment development` with `--environment production --confirm stack127-production` on every command. Use homelab because uploading to production from the MacBook is very slow. Check the live directory after cache refresh completes. -`new` means discovery has not run duplicate checks. `queued` means dedupe found no match; `needs_review` often means a possible name match. Exact-ID manual selection accepts all three because the reviewer checks duplicates. It also reports already `enriching` rows and leaves them alone. Missing, published, rejected or duplicate IDs stop the whole claim. `--ids ` is also supported. Claiming uses one transaction and touches no other candidates. Top-N selection still only claims `queued` and `needs_review`; it does not bypass dedupe. Re-running a claim is safe while rows are enriching. After reviews finish, use lookup rather than reclaiming terminal rows. +`new` means discovery has not run duplicate checks. `queued` means dedupe found no match; `needs_review` often means a possible name match. Exact-ID manual selection accepts all three because the reviewer checks duplicates. It also reports already `enriching` rows and leaves them alone. Missing, published, rejected or duplicate IDs stop the whole claim. `--ids ` is also supported. Claiming uses one transaction and touches no other candidates. Top-N selection still only claims `queued` and `needs_review`; it does not bypass dedupe. Re-running a claim is safe while rows are enriching. Before the review commits, claiming the same IDs again is safe. After the review commits, resume at estimates; do not reclaim or reapply that review. Estimates and repository refreshes can be re-run. Release an abandoned enriching claim with `npm run self-host-apps:release-candidate-batch -- --environment development --file --to needs_review`; use `--environment production --confirm stack127-production` for production. Release leaves completed candidates alone. -Claims, candidate reviews, resource estimates and single-app upserts default to development and use the shared environment resolver. They ignore `NODE_ENV` and inherited `DATABASE_URL`, reject `--database-url`, and require the production confirmation even for previews. Review application still writes unless `--dry-run` is present; estimates still need `--apply-reviewed` to write. Those flags cannot be combined. Candidate review previews use a read-only transaction; claim and estimate previews issue no data writes. A preview validates inputs and references but cannot prove that every write-time constraint or trigger will pass. +Claims, claim releases, candidate reviews, resource estimates and single-app upserts default to development and use the shared environment resolver. They ignore `NODE_ENV` and inherited `DATABASE_URL`, reject `--database-url`, and require the production confirmation even for previews. Review application still writes unless `--dry-run` is present; estimates still need `--apply-reviewed` to write. Those flags cannot be combined. Candidate review previews use a read-only transaction; claim and estimate previews issue no data writes. A preview validates inputs and references but cannot prove that every write-time constraint or trigger will pass. If there is no candidate, use the existing single-record path: put the same complete, officially researched app facts in `` with `published: true`, then run `npm run self-host-apps:upsert -- --environment development --file --dry-run` and repeat without `--dry-run`. This file is the app object, not a candidate-review wrapper. Find first: upsert merges by slug. No candidate ID or raw SQL is needed. Fill estimates with the commands above. Preview install destinations with `npm run self-host-apps:refresh-install-options -- --environment development --slug ` and repeat with `--apply`. Then check the directory and repeat this single-app sequence on homelab with the production flags after authorization. Do not use `sync-local-to-prod` for this. It mirrors whole tables and would send about 5,000 unrelated development differences along with the requested apps. + +Caller check: the repo has npm wrappers in `package.json` and command examples in `scripts/self-host-apps/scripts.json`, `skills/self-host-apps/enrichment/SKILL.md`, this document and `content_docs/self-host-apps/data-workflow.md`. Examples now name development explicitly. No workflow, monitoring entry, checked-in OVH job or homelab job calls the gated editorial commands or supplies `NODE_ENV=production` / `--database-url` to them. Installed timers and scripts outside this repo were not checked. `upsert-batch` is a separate legacy command; its environment behavior was not changed by this work. diff --git a/tests/self-host-apps/chosen-candidates.test.mjs b/tests/self-host-apps/chosen-candidates.test.mjs index 04b759ea..79f6bff1 100644 --- a/tests/self-host-apps/chosen-candidates.test.mjs +++ b/tests/self-host-apps/chosen-candidates.test.mjs @@ -1,3 +1,4 @@ +import { spawnSync } from "node:child_process"; import assert from "node:assert/strict"; import test from "node:test"; import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; @@ -142,6 +143,10 @@ test("find is read only and matches names and repositories literally", async () test("source-host refresh scopes a single slug with a bound parameter", async () => { const client = { async query(sql, params) { + if (sql.includes("select source_code_host")) { + assert.deepEqual(params, ["glitchtip"]); + return { rows: [{ source_code_host: "gitlab", source_code_url: "https://gitlab.com/glitchtip/glitchtip" }] }; + } assert.match(sql, /slug = \$2/); assert.deepEqual(params, ["gitlab", "glitchtip", 500]); return { rows: [] }; @@ -154,3 +159,44 @@ test("source-host refresh scopes a single slug with a bound parameter", async () await assert.rejects(findMatches(client, ""), /needs a name/); await assert.rejects(rowsToRefresh(client, "codeberg", { slug: "" }), /needs an app slug/); }); + +for (const [label, rows, message] of [ + ["missing app", [], /App not found: gitlab/], + ["wrong host", [{ source_code_host: "gitlab", source_code_url: "https://gitlab.com/gitlab-org/gitlab" }], /belongs to host gitlab, not codeberg/], + ["missing repository", [{ source_code_host: "codeberg", source_code_url: null }], /has no repository URL/], +]) { + test(`targeted source refresh rejects ${label}`, async () => { + let queries = 0; + const client = { async query() { queries++; return { rows }; } }; + await assert.rejects(rowsToRefresh(client, "codeberg", { slug: "gitlab" }), message); + assert.equal(queries, 1); + }); +} + +test("an empty scheduled source-host queue is successful", async () => { + const client = { async query() { return { rows: [] }; } }; + assert.deepEqual(await rowsToRefresh(client, "codeberg", {}), []); +}); + +test("release CLI requires production confirmation and ignores inherited production defaults", () => { + const script = new URL("../../scripts/self-host-apps/release-candidate-batch.mjs", import.meta.url).pathname; + const dir = mkdtempSync(join(tmpdir(), "self-host-release-")); + try { + writeFileSync(join(dir, ".env.local"), "DATABASE_URL=postgres://development/db\n"); + writeFileSync(join(dir, ".env.database"), "DATABASE_URL=postgres://production/db\n"); + const run = (...args) => spawnSync(process.execPath, [script, ...args], { + cwd: dir, encoding: "utf8", env: { ...process.env, NODE_ENV: "production", DATABASE_URL: "postgres://unintended/db" }, + }); + const missingGate = run("--environment", "production"); + assert.equal(missingGate.status, 1); + assert.match(missingGate.stderr, /Production requires --confirm stack127-production/); + const explicitProduction = run("--environment", "production", "--confirm", "stack127-production"); + assert.match(explicitProduction.stderr, /Pass --file/); + const development = run(); + assert.match(development.stderr, /Pass --file/); + assert.doesNotMatch(development.stderr, /Production requires/); + assert.match(run("--database-url", "postgres://override/db").stderr, /not supported/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/self-host-apps/dedupe-claim-race.test.mjs b/tests/self-host-apps/dedupe-claim-race.test.mjs new file mode 100644 index 00000000..b0283e48 --- /dev/null +++ b/tests/self-host-apps/dedupe-claim-race.test.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import pg from "pg"; +import { applyDecisions } from "../../scripts/self-host-apps/dedupe-candidates.mjs"; +import { claimIds } from "../../scripts/self-host-apps/select-candidate-batch.mjs"; +import { resolveDatabaseUrl } from "../../scripts/stack127/tool-pages/lib.mjs"; + +const id = (n) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const expectedCandidates = [1, 2, 3, 4].map((n) => ({ id: id(n), status: "new", duplicate_reason: null })); +function staleDecisions() { + return { + expectedCandidates, + appDuplicates: [{ target_id: id(1), app_id: id(10), reason: "exact_source_code_match" }], + candidateDuplicates: [{ target_id: id(2), duplicate_candidate_id: id(10), reason: "exact_source_code_match" }], + nameReviews: [{ target_id: id(3), app_id: id(10) }], + queuedIds: [id(4)], + }; +} + +for (const status of ["enriching", "published"]) { + test(`stale dedupe skips every write path and source preservation after a row becomes ${status}`, async () => { + let queries = 0; + const client = { async query(sql, params) { + queries++; + // A claim/review completed after dedupe read its original targets. None + // still match that snapshot when the guarded locking query runs. + assert.match(sql, /candidate.status = expected.status/); + assert.match(sql, /candidate.duplicate_reason is not distinct from expected.duplicate_reason/); + assert.match(sql, /for update of candidate/); + const expected = JSON.parse(params[0]); + const current = expected.map((row) => ({ ...row, status })); + return { rows: current.filter((row) => expected.some((old) => old.id === row.id && old.status === row.status)) }; + } }; + const report = await applyDecisions(client, staleDecisions()); + assert.equal(queries, 1); // No app evidence writes or candidate updates. + assert.deepEqual(report, { skipped_changed: 4, queued: 0, app_duplicates: 0, candidate_duplicates: 0, name_reviews: 0, sources_preserved: 0 }); + }); +} + +test("unchanged dedupe targets still move forward while changed targets are skipped", async () => { + const writes = []; + const client = { async query(sql, params) { + if (sql.includes("for update of candidate")) return { rows: [{ id: id(4) }] }; + writes.push(params); + assert.match(sql, /set status = 'queued'/); + return { rows: [] }; + } }; + const report = await applyDecisions(client, staleDecisions()); + assert.equal(report.skipped_changed, 3); + assert.equal(report.queued, 1); + assert.deepEqual(writes, [[[id(4)]]]); +}); + +// Opt-in real SQL regression: original dedupe read -> manual claim -> stale +// dedupe write. Temporary tables only; no public rows or schema are changed. +test("database: a manual claim between dedupe read and write survives all stale decisions", { + skip: process.env.SELF_HOST_APPS_TEST_ENVIRONMENT !== "development", +}, async () => { + const client = new pg.Client({ connectionString: resolveDatabaseUrl({ environment: "development" }), connectionTimeoutMillis: 15000, query_timeout: 30000 }); + try { + await client.connect(); + await client.query("begin"); + await client.query(`create temporary table dedupe_candidates_fixture ( + id uuid primary key, status text, duplicate_reason text, updated_at timestamptz + ) on commit drop`); + await client.query("insert into dedupe_candidates_fixture (id, status) select id, 'new' from unnest($1::uuid[]) id", [expectedCandidates.map((row) => row.id)]); + const isolated = { query(sql, params) { + // Source preservation must not run for claimed/published rows. + assert.doesNotMatch(sql, /public.self_host_apps/); + return client.query(sql.replaceAll("public.self_host_app_candidates", "pg_temp.dedupe_candidates_fixture"), params); + } }; + const snapshot = (await isolated.query("select * from public.self_host_app_candidates order by id")).rows; + await claimIds(isolated, snapshot.map((row) => row.id)); + await client.query("update dedupe_candidates_fixture set status = 'published' where id = $1", [id(1)]); + const report = await applyDecisions(isolated, { ...staleDecisions(), expectedCandidates: snapshot }); + assert.equal(report.skipped_changed, 4); + const after = await client.query("select status from dedupe_candidates_fixture order by id"); + assert.deepEqual(after.rows.map((row) => row.status), ["published", "enriching", "enriching", "enriching"]); + } finally { + await client.query("rollback").catch(() => {}); + await client.end(); + } +}); From 8c2ab44afd37cd2a321b13701b3012df61ae522a Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Mon, 2 Nov 2026 15:14:13 +0530 Subject: [PATCH 3/4] Use no-key-update locks so claims and dedupe can't deadlock A status update doesn't change a key, so the claim, review, and dedupe locks don't need to block foreign-key checks on the row. Adds a two-connection test that fails with the old lock. Co-Authored-By: Claude Opus 5.5 --- .../self-host-apps/apply-candidate-review.mjs | 2 +- scripts/self-host-apps/dedupe-candidates.mjs | 2 +- .../self-host-apps/select-candidate-batch.mjs | 4 +-- .../self-host-apps/chosen-candidates.test.mjs | 2 +- .../self-host-apps/dedupe-claim-race.test.mjs | 32 +++++++++++++++++-- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/scripts/self-host-apps/apply-candidate-review.mjs b/scripts/self-host-apps/apply-candidate-review.mjs index b44ef0f7..5061a5b5 100644 --- a/scripts/self-host-apps/apply-candidate-review.mjs +++ b/scripts/self-host-apps/apply-candidate-review.mjs @@ -286,7 +286,7 @@ export async function upsertInstallOptions(client, appId, options, catalogSource } async function candidateById(client, id, dryRun) { - const result = await client.query("select * from public.self_host_app_candidates where id = $1" + (dryRun ? "" : " for update"), [id]); + const result = await client.query("select * from public.self_host_app_candidates where id = $1" + (dryRun ? "" : " for no key update"), [id]); if (!result.rows[0]) throw new Error(`Missing candidate: ${id}`); return result.rows[0]; } diff --git a/scripts/self-host-apps/dedupe-candidates.mjs b/scripts/self-host-apps/dedupe-candidates.mjs index fce847c6..379cbe04 100644 --- a/scripts/self-host-apps/dedupe-candidates.mjs +++ b/scripts/self-host-apps/dedupe-candidates.mjs @@ -75,7 +75,7 @@ export async function applyDecisions(client, { and (candidate.status = 'new' or (candidate.status = 'needs_review' and candidate.duplicate_reason = 'possible_name_match')) order by candidate.id - for update of candidate`, + for no key update of candidate`, [JSON.stringify(expectedCandidates.map(({ id, status, duplicate_reason }) => ({ id, status, duplicate_reason: duplicate_reason ?? null })))], ); const eligible = new Set(result.rows.map((row) => row.id)); diff --git a/scripts/self-host-apps/select-candidate-batch.mjs b/scripts/self-host-apps/select-candidate-batch.mjs index c97cf946..625a891d 100644 --- a/scripts/self-host-apps/select-candidate-batch.mjs +++ b/scripts/self-host-apps/select-candidate-batch.mjs @@ -76,7 +76,7 @@ export async function requestedIds(args) { // any of them; a missing or terminal row must leave the whole batch unchanged. export async function claimIds(client, ids, dryRun = false) { const result = await client.query( - "select * from public.self_host_app_candidates where id = any($1::uuid[]) order by id" + (dryRun ? "" : " for update"), [ids], + "select * from public.self_host_app_candidates where id = any($1::uuid[]) order by id" + (dryRun ? "" : " for no key update"), [ids], ); const byId = new Map(result.rows.map((row) => [row.id, row])); const missing = ids.filter((id) => !byId.has(id)); @@ -138,7 +138,7 @@ async function main() { from public.self_host_app_candidates where status = any($1::text[]) order by confidence_score desc, last_seen_at desc, name, id - for update skip locked + for no key update skip locked limit $2 ) update public.self_host_app_candidates candidate diff --git a/tests/self-host-apps/chosen-candidates.test.mjs b/tests/self-host-apps/chosen-candidates.test.mjs index 79f6bff1..d3c1f1e3 100644 --- a/tests/self-host-apps/chosen-candidates.test.mjs +++ b/tests/self-host-apps/chosen-candidates.test.mjs @@ -98,7 +98,7 @@ test("review dry run checks all ten reviewed records using only reads", async () queries.push(sql); if (["begin read only", "rollback"].includes(sql)) return { rows: [] }; assert.match(sql, /^select/i); - assert.doesNotMatch(sql, /for update/i); + assert.doesNotMatch(sql, /for (no key )?update/i); if (sql.includes("self_host_app_candidates")) return { rows: [{ id: params[0], status: "enriching" }] }; const review = reviews.find((row) => row.app?.slug === params[0]); return { rows: [{ id: review?.app_id ?? params[0] }] }; diff --git a/tests/self-host-apps/dedupe-claim-race.test.mjs b/tests/self-host-apps/dedupe-claim-race.test.mjs index b0283e48..77cae737 100644 --- a/tests/self-host-apps/dedupe-claim-race.test.mjs +++ b/tests/self-host-apps/dedupe-claim-race.test.mjs @@ -26,7 +26,7 @@ for (const status of ["enriching", "published"]) { // still match that snapshot when the guarded locking query runs. assert.match(sql, /candidate.status = expected.status/); assert.match(sql, /candidate.duplicate_reason is not distinct from expected.duplicate_reason/); - assert.match(sql, /for update of candidate/); + assert.match(sql, /for no key update of candidate/); const expected = JSON.parse(params[0]); const current = expected.map((row) => ({ ...row, status })); return { rows: current.filter((row) => expected.some((old) => old.id === row.id && old.status === row.status)) }; @@ -40,7 +40,7 @@ for (const status of ["enriching", "published"]) { test("unchanged dedupe targets still move forward while changed targets are skipped", async () => { const writes = []; const client = { async query(sql, params) { - if (sql.includes("for update of candidate")) return { rows: [{ id: id(4) }] }; + if (sql.includes("for no key update of candidate")) return { rows: [{ id: id(4) }] }; writes.push(params); assert.match(sql, /set status = 'queued'/); return { rows: [] }; @@ -81,3 +81,31 @@ test("database: a manual claim between dedupe read and write survives all stale await client.end(); } }); + +// A claim must not block dedupe's foreign-key check on the row it holds. Otherwise a +// claim waiting on dedupe's next row and dedupe waiting on the claimed row would deadlock. +test("database: a claimed candidate can still be referenced by a concurrent dedupe write", { + skip: process.env.SELF_HOST_APPS_TEST_ENVIRONMENT !== "development", +}, async () => { + const connect = () => new pg.Client({ connectionString: resolveDatabaseUrl({ environment: "development" }), connectionTimeoutMillis: 15000, query_timeout: 30000 }); + const claimer = connect(); + const deduper = connect(); + try { + await claimer.connect(); + await deduper.connect(); + const { rows } = await claimer.query("select id from public.self_host_app_candidates where status = 'queued' order by id limit 2"); + assert.equal(rows.length, 2, "needs two queued development candidates"); + const [claimed, referencing] = rows.map((row) => row.id); + await claimer.query("begin"); + await claimIds(claimer, [claimed], false); + await deduper.query("begin"); + await deduper.query("set local lock_timeout = '3s'"); + // Rolled back below. A blocking lock would fail here with a lock timeout. + await deduper.query("update public.self_host_app_candidates set duplicate_candidate_id = $1 where id = $2", [claimed, referencing]); + } finally { + await deduper.query("rollback").catch(() => {}); + await claimer.query("rollback").catch(() => {}); + await deduper.end().catch(() => {}); + await claimer.end().catch(() => {}); + } +}); From 81e6392bbc42e31308d0033df77b3a5d4df963e6 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Mon, 2 Nov 2026 15:16:13 +0530 Subject: [PATCH 4/4] Lock review and release rows in ID order to match claims Co-Authored-By: Claude Opus 5.5 --- .../self-host-apps/apply-candidate-review.mjs | 4 ++++ .../release-candidate-batch.mjs | 15 +++++++++++---- .../self-host-apps/chosen-candidates.test.mjs | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/self-host-apps/apply-candidate-review.mjs b/scripts/self-host-apps/apply-candidate-review.mjs index 5061a5b5..a8a85007 100644 --- a/scripts/self-host-apps/apply-candidate-review.mjs +++ b/scripts/self-host-apps/apply-candidate-review.mjs @@ -385,6 +385,10 @@ export async function applyReviews(client, reviews, dryRun = false) { if (new Set(ids).size !== ids.length) throw new Error("Repeated candidate_id in review batch."); await client.query(dryRun ? "begin read only" : "begin"); try { + // Lock in ID order, the same order claims and dedupe use, so batches in any order can't deadlock with them. + if (!dryRun) { + await client.query("select id from public.self_host_app_candidates where id = any($1::uuid[]) order by id for no key update", [ids.filter(Boolean)]); + } for (const review of reviews) results.push(await applyReview(client, review, dryRun)); await client.query(dryRun ? "rollback" : "commit"); return results; diff --git a/scripts/self-host-apps/release-candidate-batch.mjs b/scripts/self-host-apps/release-candidate-batch.mjs index 7262e7cb..c5a4c8c8 100644 --- a/scripts/self-host-apps/release-candidate-batch.mjs +++ b/scripts/self-host-apps/release-candidate-batch.mjs @@ -21,11 +21,18 @@ async function main() { await withClient(async (client) => { const result = await client.query( ` - update public.self_host_app_candidates + with locked as ( + select id from public.self_host_app_candidates + where id = any($1::uuid[]) and status = 'enriching' + order by id + for no key update + ) + update public.self_host_app_candidates candidate set status = $2, updated_at = now() - where id = any($1::uuid[]) - and status = 'enriching' - returning id + from locked + where candidate.id = locked.id + and candidate.status = 'enriching' + returning candidate.id `, [ids, targetStatus], ); diff --git a/tests/self-host-apps/chosen-candidates.test.mjs b/tests/self-host-apps/chosen-candidates.test.mjs index d3c1f1e3..dd21949c 100644 --- a/tests/self-host-apps/chosen-candidates.test.mjs +++ b/tests/self-host-apps/chosen-candidates.test.mjs @@ -108,6 +108,25 @@ test("review dry run checks all ten reviewed records using only reads", async () assert.equal(queries.at(-1), "rollback"); }); +test("review apply locks candidates in ID order before its first write", async () => { + const queries = []; + const later = "00000000-0000-4000-8000-000000000002"; + const earlier = "00000000-0000-4000-8000-000000000001"; + const client = { async query(sql, params) { + queries.push({ sql, params }); + if (sql.includes("self_host_app_candidates") && sql.startsWith("select *")) return { rows: [{ id: params[0], status: "enriching" }] }; + return { rows: [], rowCount: 1 }; + } }; + await applyReviews(client, [ + { candidate_id: later, decision: "reject", reason: "test" }, + { candidate_id: earlier, decision: "reject", reason: "test" }, + ]); + assert.equal(queries[0].sql, "begin"); + assert.match(queries[1].sql, /where id = any\(\$1::uuid\[\]\) order by id for no key update$/); + assert.deepEqual(queries[1].params[0], [later, earlier]); + assert.equal(queries.at(-1).sql, "commit"); +}); + test("review preview rejects unclaimed candidates and rolls back", async () => { const queries = []; const client = { async query(sql) {