Keep directory refreshes running past invalid repositories

scripts/self-host-apps/lib/refresh-health.mjs

@@ -0,0 +1,4 @@
+// Empty queues are healthy; a complete outage or more than 20% failed rows is not.
+export function refreshFailed({ selected, refreshed, failed }) {
+  return selected > 0 && (refreshed === 0 || failed * 100 > selected * 20);
+}

scripts/self-host-apps/refresh-github.mjs

@@ -8,6 +8,7 @@ import { licenseTypeFromSpdx, sourceModelFromSpdx } from "./lib/schema.mjs";
 import { pathToFileURL } from "node:url";
 import { jobSummary } from "./lib/job-summary.mjs";
 import { maintenanceFrom as cliMaintenanceFrom } from "../cli-apps/lib/maintenance.mjs";
+import { refreshFailed } from "./lib/refresh-health.mjs";
 
 const githubGraphqlUrl = "https://api.github.com/graphql";
 let cachedGithubToken;
@@ -221,7 +222,7 @@ export async function rowsToRefresh(client, args, retryMetadataAvailable = true)
           coalesce(linked.owner, existing.owner, lower(parsed.parts[1])) as github_owner,
           coalesce(linked.repo, existing.repo, lower(regexp_replace(parsed.parts[2], '[.]git$', '', 'i'))) as github_repo
         from memberships member
-        cross join lateral (select regexp_match(member.repository_url, '^https://github[.]com/([^/]+)/([^/#?]+)', 'i') as parts) parsed
+        cross join lateral (select regexp_match(member.repository_url, '^https://github[.]com/([a-z0-9][a-z0-9-]{0,38})/([a-z0-9_.-]+)(?:[/?#].*)?$', 'i') as parts) parsed
         left join public.github_projects linked on linked.id = member.github_project_id
         left join public.github_projects existing on linked.id is null
           and existing.owner = lower(parsed.parts[1])
@@ -231,6 +232,8 @@ export async function rowsToRefresh(client, args, retryMetadataAvailable = true)
           min(github_owner) as github_owner, min(github_repo) as github_repo,
           coalesce(array_agg(distinct self_host_slug) filter (where self_host_slug is not null), '{}') as self_host_slugs
         from resolved
+        where github_project_id is not null
+          or (github_owner is not null and github_repo is not null and github_repo not in ('', '.', '..'))
         group by github_project_id, case when github_project_id is null then coalesce(github_owner || '/' || github_repo, lower(source_code_url)) end
       )
       select app.*
@@ -416,24 +419,32 @@ export async function writeRepositoryFacts(client, row, repository, checkedAt, d
 
 export async function writeFailureAttempt(client, row, attemptedAt, dryRun, retryMetadataAvailable = true) {
   if (dryRun || !retryMetadataAvailable) return;
-  const owner = String(row.github_owner ?? "").trim().toLowerCase();
-  const repo = String(row.github_repo ?? "").trim().replace(/\.git$/i, "").toLowerCase();
-  if (!owner || !repo) {
-    if (!row.github_project_id) throw new Error(`Cannot record failed attempt for ${row.slug}: missing repository identity`);
-    await client.query(`update public.github_projects set last_attempt_at = $2, consecutive_failures = consecutive_failures + 1 where id = $1`, [row.github_project_id, attemptedAt]);
-    return;
+  try {
+    const owner = String(row.github_owner ?? "").trim().toLowerCase();
+    const repo = String(row.github_repo ?? "").trim().replace(/\.git$/i, "").toLowerCase();
+    if (!owner || !repo) {
+      if (!row.github_project_id) {
+        console.warn(`warning ${row.slug}: skipping failed-attempt write: missing repository identity`);
+        return;
+      }
+      await client.query(`update public.github_projects set last_attempt_at = $2, consecutive_failures = consecutive_failures + 1 where id = $1`, [row.github_project_id, attemptedAt]);
+      return;
+    }
+    const result = await client.query(`
+      insert into public.github_projects (owner, repo, full_name, repository_url, last_attempt_at, consecutive_failures)
+      values ($1, $2, $1 || '/' || $2, $3, $4, 1)
+      on conflict (owner, repo) do update set
+        last_attempt_at = excluded.last_attempt_at,
+        consecutive_failures = github_projects.consecutive_failures + 1
+      returning id
+    `, [owner, repo, `https://github.com/${owner}/${repo}`, attemptedAt]);
+    // Even a previously unlinked repository must move to the back of the queue.
+    // Do not change checked_at or any app fact/verification date on failure.
+    await client.query(`update public.self_host_apps set github_project_id = $2 where slug = any($1::text[]) and github_project_id is distinct from $2::uuid`, [row.self_host_slugs ?? [row.slug], result.rows[0].id]);
+  } catch (error) {
+    // Failure bookkeeping must not prevent the remaining repositories or summary.
+    console.warn(`warning ${row.slug}: unable to record failed attempt: ${error.message}`);
   }
-  const result = await client.query(`
-    insert into public.github_projects (owner, repo, full_name, repository_url, last_attempt_at, consecutive_failures)
-    values ($1, $2, $1 || '/' || $2, $3, $4, 1)
-    on conflict (owner, repo) do update set
-      last_attempt_at = excluded.last_attempt_at,
-      consecutive_failures = github_projects.consecutive_failures + 1
-    returning id
-  `, [owner, repo, `https://github.com/${owner}/${repo}`, attemptedAt]);
-  // Even a previously unlinked repository must move to the back of the queue.
-  // Do not change checked_at or any app fact/verification date on failure.
-  await client.query(`update public.self_host_apps set github_project_id = $2 where slug = any($1::text[]) and github_project_id is distinct from $2::uuid`, [row.self_host_slugs ?? [row.slug], result.rows[0].id]);
 }
 
 export async function refreshGithub(client, args) {
@@ -460,7 +471,7 @@ export async function refreshGithub(client, args) {
 
     for (const row of selectedBatch) {
       try {
-        const repo = parseRepo(row.source_code_url);
+        const repo = parseRepo(row.github_owner && row.github_repo ? `${row.github_owner}/${row.github_repo}` : row.source_code_url);
         row.github_owner = repo.owner;
         row.github_repo = repo.repo;
         batch.push(row);
@@ -499,14 +510,14 @@ export async function refreshGithub(client, args) {
 
   if (args["dry-run"]) console.log("Dry run OK. No GitHub facts committed.");
   console.log(`GitHub refresh complete: ${refreshed} refreshed, ${failures.length} failed.`);
-  jobSummary("GitHub facts", { Selected: rows.length, Refreshed: refreshed, Failed: failures.length, "Dry run": Boolean(args["dry-run"]), "Retry metadata available": retryMetadataAvailable }, failures);
+  jobSummary("GitHub facts", { Selected: rows.length, Refreshed: refreshed, Failed: failures.length, "Allowed failure percentage": 20, "Dry run": Boolean(args["dry-run"]), "Retry metadata available": retryMetadataAvailable }, failures);
   return { selected: rows.length, refreshed, failed: failures.length, failures, retryMetadataAvailable };
 }
 
 async function main() {
   const args = parseArgs();
   const report = await withClient((client) => refreshGithub(client, args), args["database-url"]);
-  if (report.failed) process.exitCode = 1;
+  if (refreshFailed(report)) process.exitCode = 1;
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => {

scripts/self-host-apps/refresh-source-host.mjs

@@ -5,6 +5,7 @@ import { withClient } from "./lib/db.mjs";
 import { licenseTypeFromSpdx } from "./lib/schema.mjs";
 import { pathToFileURL } from "node:url";
 import { jobSummary } from "./lib/job-summary.mjs";
+import { refreshFailed } from "./lib/refresh-health.mjs";
 
 const supportedHosts = new Set(["gitlab", "codeberg"]);
 
@@ -239,33 +240,37 @@ async function writeFacts(client, row, host, facts, checkedAt) {
   );
 }
 
-async function main() {
-  const args = parseArgs();
+export async function refreshSourceHost(client, args) {
   const host = args.host;
   if (!supportedHosts.has(host)) throw new Error("--host must be gitlab or codeberg");
   const checkedAt = new Date().toISOString();
 
-  await withClient(async (client) => {
-    const rows = await rowsToRefresh(client, host, args);
-    let refreshed = 0;
-    const failures = [];
+  const rows = await rowsToRefresh(client, host, args);
+  let refreshed = 0;
+  const failures = [];
 
-    for (const row of rows) {
-      try {
-        const facts = host === "gitlab" ? await gitlabFacts(row) : await codebergFacts(row);
-        if (!args["dry-run"]) await writeFacts(client, row, host, facts, checkedAt);
-        refreshed += 1;
-        console.log(`${args["dry-run"] ? "Checked" : "Refreshed"} ${host} facts: ${row.slug}`);
-      } catch (error) {
-        failures.push({ slug: row.slug, error: error.message });
-        console.warn(`warning ${row.slug}: ${error.message}`);
-      }
+  for (const row of rows) {
+    try {
+      const facts = host === "gitlab" ? await gitlabFacts(row) : await codebergFacts(row);
+      if (!args["dry-run"]) await writeFacts(client, row, host, facts, checkedAt);
+      refreshed += 1;
+      console.log(`${args["dry-run"] ? "Checked" : "Refreshed"} ${host} facts: ${row.slug}`);
+    } catch (error) {
+      failures.push({ slug: row.slug, error: error.message });
+      console.warn(`warning ${row.slug}: ${error.message}`);
     }
+  }
+
+  const report = { host, dry_run: Boolean(args["dry-run"]), selected: rows.length, refreshed, failed: failures.length, failures };
+  console.log(JSON.stringify(report, null, 2));
+  jobSummary(`${host} facts`, { Selected: rows.length, Refreshed: refreshed, Failed: failures.length, "Allowed failure percentage": 20, "Dry run": Boolean(args["dry-run"]) }, failures);
+  return report;
+}
 
-    console.log(JSON.stringify({ host, dry_run: Boolean(args["dry-run"]), selected: rows.length, refreshed, failed: failures.length, failures }, null, 2));
-    jobSummary(`${host} facts`, { Selected: rows.length, Refreshed: refreshed, Failed: failures.length, "Dry run": Boolean(args["dry-run"]) }, failures);
-    if (failures.length) process.exitCode = 1;
-  }, args["database-url"]);
+async function main() {
+  const args = parseArgs();
+  const report = await withClient((client) => refreshSourceHost(client, args), args["database-url"]);
+  if (refreshFailed(report)) process.exitCode = 1;
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => {

stack127_docs/monitoring.json

@@ -22,8 +22,8 @@
     {"key": "github/stack127labs/stack127/self-host-apps-discovery-directories.yml", "group": "Self-host apps", "name": "Find apps in directories", "about": "Once a month, looks for self-hostable apps in public directories."},
     {"key": "github/stack127labs/stack127/self-host-apps-discovery-github.yml", "group": "Self-host apps", "name": "Find apps on GitHub", "about": "Every day, searches GitHub for new self-hostable apps."},
     {"key": "github/stack127labs/stack127/self-host-apps-discovery-lists.yml", "group": "Self-host apps", "name": "Find apps in curated lists", "about": "Every Sunday, reads maintained awesome-lists for new self-hostable apps."},
-    {"key": "github/stack127labs/stack127/self-host-apps-refresh-github.yml", "group": "Self-host apps", "name": "Refresh GitHub facts", "about": "Twice a day, attempts up to 500 unique GitHub projects linked to self-host and published CLI apps (about a 5-day full cycle), updates CLI maintenance from activity, and lists failures without marking failed facts fresh."},
-    {"key": "github/stack127labs/stack127/self-host-apps-refresh-source-hosts.yml", "group": "Self-host apps", "name": "Refresh GitLab and Codeberg facts", "about": "Every day, refreshes facts from each app’s own GitLab server and from Codeberg, and lists partial failures in the job summary while marking the job failed."},
+    {"key": "github/stack127labs/stack127/self-host-apps-refresh-github.yml", "group": "Self-host apps", "name": "Refresh GitHub facts", "about": "Twice a day, attempts up to 500 unique GitHub projects linked to self-host and published CLI apps (about a 5-day full cycle), updates CLI maintenance from activity, and lists failures without marking failed facts fresh; fails above 20% failures or when a non-empty batch refreshes nothing."},
+    {"key": "github/stack127labs/stack127/self-host-apps-refresh-source-hosts.yml", "group": "Self-host apps", "name": "Refresh GitLab and Codeberg facts", "about": "Every day, refreshes facts from each app’s own GitLab server and from Codeberg, and lists every failure in the job summary; fails above 20% failures or when a non-empty batch refreshes nothing."},
     {"key": "github/stack127labs/stack127/self-host-apps-refresh-technical.yml", "group": "Self-host apps", "name": "Collect technical details", "about": "Every Monday, collects technical evidence for review, previews install destinations without saving, and runs publication QA; reports unreadable sources and fails above 5% (or if none are readable), without writing editorial facts."},
     {"key": "stack127-vps/stack127-cache-worker", "group": "Site cache", "name": "Page cache worker", "about": "Every 30 seconds on the OVH server, checks for changed pages, refreshes ad-network heading counts when their data changes, and starts a static page rebuild when edits are waiting.", "impact": "Edits do not reach the live site."},
     {"key": "github/stack127labs/stack127/stack127-cache-maintenance.yml", "group": "Site cache", "name": "Cache maintenance", "about": "Every hour at minute 17, checks cache health and starts a static page rebuild when page refreshes are overdue."},

stack127_docs/self-host-apps-data.md

@@ -83,6 +83,15 @@ so inaccessible repositories move behind older work without appearing fresh.
 A failed repository with no project row gets a linked placeholder with a null
 `checked_at`. Failed repository URLs appear in the Actions job summary.
 
+The GitHub queue excludes unlinked apps without a usable GitHub repository URL,
+including null and malformed URLs. A linked project can still supply the identity
+when the app URL is missing. Missing identity or a failed attempt-metadata write
+is logged and skipped without stopping the remaining refreshes.
+GitHub, GitLab and Codeberg list every selected failure in the job summary.
+They tolerate up to 20% failed rows, and fail above that limit or when a non-empty
+batch refreshes nothing. Empty queues succeed. Failed GitLab/Codeberg sources
+keep their successful check dates and are retried on later daily runs.
+
 Other providers keep using the existing `source_*` columns until a provider table is worth adding. That's a planned transition, not a second source of truth for self-host apps.
 
 ## Evidence and QA