From 0580ce192c3710611871478a2696f673b6fd322d Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Wed, 21 Oct 2026 15:48:56 +0530 Subject: [PATCH 1/2] Show the ad networks directory on the homepage with its own card art Co-Authored-By: Claude Opus 5.5 --- app/(site)/(directories)/ad-networks/card-counts.ts | 10 ++++++++++ app/(site)/(directories)/card-counts.ts | 2 ++ app/(site)/art/site--spot-ad-networks.spec.json | 11 +++++++++++ app/(site)/art/site--spot-ad-networks.svg | 11 +++++++++++ app/(site)/art/site--spot-ad-networks.ts | 13 +++++++++++++ lib/card-art.ts | 2 ++ lib/card-icons.ts | 1 + lib/page-registry-helpers.ts | 1 + scripts/stack127/cache/targets.mjs | 2 ++ scripts/stack127/marker-art/figures.mjs | 10 ++++++++++ tests/stack127/cache-targets.test.mjs | 3 ++- 11 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 app/(site)/(directories)/ad-networks/card-counts.ts create mode 100644 app/(site)/art/site--spot-ad-networks.spec.json create mode 100644 app/(site)/art/site--spot-ad-networks.svg create mode 100644 app/(site)/art/site--spot-ad-networks.ts diff --git a/app/(site)/(directories)/ad-networks/card-counts.ts b/app/(site)/(directories)/ad-networks/card-counts.ts new file mode 100644 index 00000000..086b36a3 --- /dev/null +++ b/app/(site)/(directories)/ad-networks/card-counts.ts @@ -0,0 +1,10 @@ +import type { PageCardCount } from "@/app/ui/cards/page-card"; +import { getAdNetworkDirectoryData } from "@/lib/ad-networks"; + +export async function cardCounts(): Promise { + const { networks_tracked, listed } = await getAdNetworkDirectoryData(); + return [ + { label: "networks", value: networks_tracked }, + { label: "sites", value: listed }, + ]; +} diff --git a/app/(site)/(directories)/card-counts.ts b/app/(site)/(directories)/card-counts.ts index 6462997c..c31a6a7e 100644 --- a/app/(site)/(directories)/card-counts.ts +++ b/app/(site)/(directories)/card-counts.ts @@ -1,4 +1,5 @@ import type { PageCardCount } from "@/app/ui/cards/page-card"; +import { cardCounts as adNetworks } from "./ad-networks/card-counts"; import { cardCounts as cliApps } from "./cli-apps/card-counts"; import { cardCounts as eppStatusCodes } from "./epp-status-codes/card-counts"; import { cardCounts as keyboardShortcuts } from "./keyboard-shortcuts/card-counts"; @@ -10,6 +11,7 @@ import { cardCounts as tldDirectory } from "./tld-directory/card-counts"; /** Live homepage card counts. Each directory computes its own from its records. */ const counts: Record Promise> = { + "ad-networks": adNetworks, "cli-apps": cliApps, "epp-status-codes": eppStatusCodes, "keyboard-shortcuts": keyboardShortcuts, diff --git a/app/(site)/art/site--spot-ad-networks.spec.json b/app/(site)/art/site--spot-ad-networks.spec.json new file mode 100644 index 00000000..bc485a07 --- /dev/null +++ b/app/(site)/art/site--spot-ad-networks.spec.json @@ -0,0 +1,11 @@ +{ + "id": "site--spot-ad-networks", + "type": "toolSpot", + "takeaway": "Website after website carries an ad slot, and an ad network fills each one.", + "source": "conceptual figure, no external data", + "dataAsOf": "2026-10-21", + "width": 140, + "height": 110, + "seed": 127, + "data": { "motif": "adSlotSites" } +} diff --git a/app/(site)/art/site--spot-ad-networks.svg b/app/(site)/art/site--spot-ad-networks.svg new file mode 100644 index 00000000..858fb3b4 --- /dev/null +++ b/app/(site)/art/site--spot-ad-networks.svg @@ -0,0 +1,11 @@ + +Website after website carries an ad slot, and an ad network fills each one. + + + + + + + + + diff --git a/app/(site)/art/site--spot-ad-networks.ts b/app/(site)/art/site--spot-ad-networks.ts new file mode 100644 index 00000000..59555f27 --- /dev/null +++ b/app/(site)/art/site--spot-ad-networks.ts @@ -0,0 +1,13 @@ +// generated by scripts/stack127/marker-art/generate.mjs. do not edit. +// edit the sibling .spec.json and rerun: npm run stack127:marker-art +export const siteSpotAdNetworksArt = ` +Website after website carries an ad slot, and an ad network fills each one. + + + + + + + + +`; diff --git a/lib/card-art.ts b/lib/card-art.ts index 2dd0060a..89dfc963 100644 --- a/lib/card-art.ts +++ b/lib/card-art.ts @@ -1,3 +1,4 @@ +import { siteSpotAdNetworksArt } from "@/app/(site)/art/site--spot-ad-networks"; import { siteSpotPlatformFeesArt } from "@/app/(site)/art/site--spot-platform-fees"; import { siteSpotLaunchDirectoriesArt } from "@/app/(site)/art/site--spot-launch-directories"; import { siteSpotLicensesArt } from "@/app/(site)/art/site--spot-licenses"; @@ -95,6 +96,7 @@ import { siteSpotStorageToolsArt } from "@/app/(site)/art/site--spot-storage-too /** Approved generated SVGs keyed by references stored in the content tables. */ const art: Record = { + "ad-networks": siteSpotAdNetworksArt, "cli-apps": siteSpotCliAppsArt, "epp-status-codes": eppStatusArt, "keyboard-shortcuts": siteSpotKeyboardShortcutsArt, diff --git a/lib/card-icons.ts b/lib/card-icons.ts index 559c2070..4dfa39ed 100644 --- a/lib/card-icons.ts +++ b/lib/card-icons.ts @@ -8,6 +8,7 @@ import { } from "lucide-react"; const icons: Record = { + "ad-networks": Network, "cli-apps": SquareTerminal, "epp-status-codes": ListChecks, "keyboard-shortcuts": Keyboard, diff --git a/lib/page-registry-helpers.ts b/lib/page-registry-helpers.ts index 059f186e..9c98f1a1 100644 --- a/lib/page-registry-helpers.ts +++ b/lib/page-registry-helpers.ts @@ -40,6 +40,7 @@ export async function resolvePagePublicationState { const targets = targetsForEvents([{ entity_type: "ad_networks", slug: "mediavine" }]); - for (const path of ["/ad-networks", "/ad-networks/mediavine", "/ad-networks/stats", "/ad-networks/niches", "/ad-networks/niches/food", "/ad-network-checker", "/api/ad-networks/sites", "/api/ad-networks/site"]) assert.ok(targets.paths.includes(path)); + for (const path of ["/ad-networks", "/ad-networks/mediavine", "/ad-networks/stats", "/ad-networks/niches", "/ad-networks/niches/food", "/ad-network-checker", "/api/ad-networks/sites", "/api/ad-networks/site", "/"]) assert.ok(targets.paths.includes(path)); assert.ok(targets.nextTags.includes("stack127-ad-networks")); assert.ok(targets.edgeTags.includes("ad-networks-catalog")); assert.ok(targets.edgeTags.includes("page:ad-networks:mediavine")); + assert.ok(targets.edgeTags.includes("page:home")); assert.deepEqual(targets.warmPaths, targets.paths.filter(path => !path.startsWith("/api/"))); }); From 46fc49e2c4fee45a1bb05fed89b535d7623ad52e Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Wed, 21 Oct 2026 16:22:51 +0530 Subject: [PATCH 2/2] Keep ad network SEO counts current through the refresh queue --- .../ad-networks/production-runbook.md | 20 +- scripts/ad-networks/refresh-copy-counts.mjs | 174 ++++++++++++++++++ scripts/ad-networks/revalidate.mjs | 10 +- scripts/stack127/cache/worker.mjs | 25 ++- stack127_docs/monitoring.json | 4 +- tests/pages/ad-networks/copy-counts.test.mjs | 174 ++++++++++++++++++ tests/pages/ad-networks/refresh.test.mjs | 26 ++- tests/stack127/cache-worker.test.mjs | 98 +++++++++- 8 files changed, 514 insertions(+), 17 deletions(-) create mode 100644 scripts/ad-networks/refresh-copy-counts.mjs create mode 100644 tests/pages/ad-networks/copy-counts.test.mjs diff --git a/content_docs/ad-networks/production-runbook.md b/content_docs/ad-networks/production-runbook.md index 205e51de..1f42f1f9 100644 --- a/content_docs/ad-networks/production-runbook.md +++ b/content_docs/ad-networks/production-runbook.md @@ -1,6 +1,6 @@ # Ad-network production runbook -Ad-network collectors, syncs, snapshots, and backups run as cron jobs on OVH. Weekly niche sorting runs on homelab. All ad-network pages stay unpublished and return 404. Resuming automation does not authorize publication. +Ad-network collectors, syncs, snapshots, and backups run as cron jobs on OVH. Weekly niche sorting runs on homelab. The directory, stats, 21 niche pages, 44 network pages, and checker were published on October 21, 2026. Snigel stays retired and returns 404. Automation updates existing pages; it does not publish new ones. Cron is running again since October 20, 2026, 15:00 UTC, on self-hosted Postgres 17. It was paused from October 19, 12:15 UTC, until the sync sent only changed rows. History: [OVH cron plan](../../dev-space/ad-networks-ovh-cron-plan.md). @@ -20,6 +20,18 @@ The env file also sets `GITHUB_ALERT_TOKEN`, `AD_NETWORK_DB_PATH`, and `AD_NETWO The wrapper is [`run-job.sh`](../../scripts/ad-networks/ovh/run-job.sh). The full schedule is [`crontab.txt`](../../scripts/ad-networks/ovh/crontab.txt). Homelab's `stack127-ad-network-niche.timer` runs Sunday at 06:00 UTC. +## Counts in titles and headings + +The approved titles and headings use rounded site counts. The existing revalidation step refreshes only those count tokens from the target database, including decreases. The cache worker repeats this before processing ad-network events, so a stopped sync or direct data import can recover through the durable queue. No separate schedule is needed. + +Editorial wording stays in the page tables. The count refresher recognizes only the approved page and phrase combinations. Descriptions use no site-count snapshots. Publir, EthicalAds, Concert, Sovrn, and Media.net keep their approved count-free titles and headings. The 20-niche taxonomy stays fixed. A refresh within the same rounding bucket makes no page-field update. + +Before the new copy is published, older wording is skipped. If a page matches only part of the approved count phrases, the refresh fails and rolls back rather than rewriting its words. Counts below ten lose their numeric prefix and can regain it when the count recovers. + +Before preparing a publication plan, render its desired count tokens from that target's current data. Keep the publisher's exact conflict checks: if a scheduled refresh changes a frozen row, prepare a new plan. Check the live page after publishing. A static rebuild still takes several minutes, so data and heading changes reach the public site through the normal verified refresh process. + +Restoring an older plan also refuses to overwrite newer count changes. To recover old wording after that conflict, prepare a new plan with the previous wording and current counts. Do not force an old numeric snapshot back into the page tables. + ## How a job runs Collectors use HTTP and SQLite without holding a production connection. Sync then opens one connection, takes the existing runtime and reviewed-sync locks, and pins a read-only SQLite view. It sends only rows whose compact contents differ from the last successful sync. Before each target batch, its old hashes are durably invalidated. New hashes are saved after that batch commits. An interrupted sync cannot reuse an obsolete hash, even if the source later returns to its old value. @@ -90,7 +102,7 @@ The coordinator performs these steps after review and release authorization. Kee ssh ovh '/srv/stack127/scripts/ad-networks/ovh/run-job.sh backup' ``` - Check the job logs and Node cache-worker service. The queue must drain without retries or permission errors. Check `/ad-networks`, `/ad-network-checker`, and an existing network child URL: all must return 404. + Check the job logs and Node cache-worker service. The queue must drain without retries or permission errors. Check `/ad-networks`, `/ad-network-checker`, and an existing network child URL: all must return 200. `/ad-networks/snigel` must return 404. 6. Install the new schedule and re-enable the niche timer: ```bash @@ -99,7 +111,7 @@ The coordinator performs these steps after review and release authorization. Kee ``` Confirm verification starts at minute 40. Do not install the saved five-minute schedule. -7. Check connections, table sizes, queue age, logs, and disk space after the first hour and first day. Collectors should hold no production session; each sync uses one. Compare the live revision and unpublished HTTP responses again. +7. Check connections, table sizes, queue age, logs, and disk space after the first hour and first day. Collectors should hold no production session; each sync uses one. Compare the live revision and publication responses again. A local synthetic benchmark scanned 107,146 publishers and 114,284 relationships in 12.4 seconds with no changed rows, zero row payloads, and 12 SQL calls. PostgreSQL responses were simulated; this measures the SQLite scan and hash work. Measure actual OVH sync time during step 4. The old no-change sync took about 12 minutes. @@ -125,4 +137,4 @@ ssh homelab 'cd /srv/data/projects/stack127 && node --experimental-strip-types s ssh ovh '/srv/stack127/scripts/ad-networks/ovh/run-job.sh rdap-import --source-sqlite /srv/stack127-data/state/rdap.sqlite && rm /srv/stack127-data/state/rdap.sqlite' ``` -The importer copies only newer RDAP checkpoints and dates, then syncs and refreshes. After its successful backup, disable homelab's old `stack127-ad-network-backup.timer`. This does not affect the niche timer. Pages stay unpublished. +The importer copies only newer RDAP checkpoints and dates, then syncs and refreshes. After its successful backup, disable homelab's old `stack127-ad-network-backup.timer`. This does not affect the niche timer or change which pages are published. diff --git a/scripts/ad-networks/refresh-copy-counts.mjs b/scripts/ad-networks/refresh-copy-counts.mjs new file mode 100644 index 00000000..4e1e81a3 --- /dev/null +++ b/scripts/ad-networks/refresh-copy-counts.mjs @@ -0,0 +1,174 @@ +// Only the count-bearing fields approved in the October 2026 title batch. +// Editorial wording and descriptions stay owned by the page publisher. +const networks = [ + ['google-adsense', 'Google AdSense'], ['mediavine', 'Mediavine'], ['ezoic', 'Ezoic'], ['raptive', 'Raptive'], + ['journey-by-mediavine', 'Journey by Mediavine'], ['publift', 'Publift'], ['mgid', 'MGID'], + ['the-moneytizer', 'The Moneytizer'], ['taboola', 'Taboola'], ['monumetric', 'Monumetric'], + ['playwire', 'Playwire'], ['freestar', 'Freestar'], ['anymind-anymanager', 'AnyMind'], + ['adpushup', 'AdPushup'], ['infolinks', 'Infolinks'], ['nitropay', 'NitroPay'], ['newor-media', 'Newor Media'], + ['snack-media', 'Snack Media'], ['refinery89', 'Refinery89'], ['setupad', 'Setupad'], ['venatus', 'Venatus'], + ['pubpower', 'PubPower'], ['pubgalaxy', 'PubGalaxy'], ['publisher-collective', 'Publisher Collective'], + ['clickio', 'Clickio'], ['adnimation', 'Adnimation'], ['teads', 'Teads'], + ['amazon-publisher-services', 'Amazon Publisher Services'], ['adapex', 'Adapex'], ['geniee-ssp', 'Geniee SSP'], + ['opti-digital', 'Opti Digital'], ['monetizemore-pubguru', 'MonetizeMore'], ['pubfuture', 'PubFuture'], + ['buysellads-carbon-ads', 'Carbon Ads'], ['pubnation', 'PubNation'], ['revcontent', 'Revcontent'], + ['aditude', 'Aditude'], ['sparteo', 'Sparteo'], ['she-media', 'SHE Media'], +]; +const niches = [ + ['tech', 'tech sites'], ['news', 'news sites'], ['food', 'food blogs'], ['education', 'education sites'], + ['entertainment', 'entertainment sites'], ['business', 'business sites'], ['lifestyle', 'lifestyle blogs'], + ['travel', 'travel blogs'], ['gaming', 'gaming sites'], ['personal-finance', 'personal finance sites'], + ['sports', 'sports sites'], ['health-fitness', 'health and fitness sites'], ['home-diy', 'home and DIY blogs'], + ['crafts', 'craft blogs'], ['automotive', 'car sites'], ['beauty-fashion', 'beauty and fashion blogs'], + ['parenting', 'parenting blogs'], ['outdoors', 'outdoor blogs'], ['pets', 'pet blogs'], ['gardening', 'gardening blogs'], +]; + +export function roundedCopyCount(count, { catalog = false } = {}) { + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`Invalid ad-network count: ${count}`); + if (count < 10) return ''; + const step = catalog ? 10 : count < 100 ? 5 : count < 1000 ? 50 : count < 10000 ? 100 : 1000; + return `${(Math.floor(count / step) * step).toLocaleString('en-US')}+`; +} + +const escape = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +// A slot owns its following space, so removing a prefix below ten is reversible. +// Count-only mark wrappers are accepted and retained while a count is present. +function phrase(template, metric, catalog = false, anchored = true) { + const [before, after] = template.split('{} '); + if (after === undefined) throw new Error('Count phrase needs one slot'); + const number = '(?:[0-9]+(?:,[0-9]{3})*)\\+'; + const pattern = new RegExp(`${anchored ? '^' : ''}${escape(before)}(?(?:(?:${number}|${number}) )?)${escape(after)}`, 'dg'); + return { pattern, metric, catalog }; +} +const rules = new Map([ + ['ad-networks', { + meta_title: [phrase('{} ad networks compared:', 'network_count', true), phrase('pay, {} sites', 'listed', false, false)], + title: [phrase('Compare {} ad networks:', 'network_count', true), phrase('and {} sites using them', 'listed', false, false)], + }], + ['ad-networks/stats', { + meta_title: [phrase('Ad network market share (2026): {} sites checked', 'listed')], + title: [phrase('Which ad networks do {} sites actually run?', 'listed')], + }], + ['ad-networks/niches', { + meta_title: [phrase('Ad networks by niche: what {} sites in 20 niches run', 'classified')], + title: [phrase('The ad networks {} sites run, sorted into 20 niches', 'classified')], + }], +]); +for (const [slug, name] of networks) { + const metric = `networks.${slug}`; + const seo = { + 'google-adsense': '{} sites using AdSense:', + 'journey-by-mediavine': 'Journey by Mediavine: {} sites, requirements, and pay', + 'publisher-collective': '{} Publisher Collective sites:', + 'amazon-publisher-services': 'Amazon Publisher Services: {} sites, fees, and pay terms', + }[slug] ?? `{} sites using ${name}${slug === 'pubgalaxy' ? ', now part of Improve Digital' : ':'}`; + const h1 = slug === 'pubgalaxy' ? '{} sites still running PubGalaxy, now part of Improve Digital' + : `{} sites running ${name}, plus its`; + rules.set(`ad-networks/${slug}`, { meta_title: [phrase(seo, metric)], title: [phrase(h1, metric)] }); +} +for (const [slug, noun] of niches) { + // The approved SEO titles have two forms, chosen by their original length. + const short = `Ad networks for ${noun}: what 1,000+ real sites use`.length > 60; + rules.set(`ad-networks/niches/${slug}`, { + meta_title: [phrase(`Ad networks for ${noun}: what {} ${short ? 'use' : 'real sites use'}`, `niches.${slug}`)], + title: [phrase(`The ad networks {} ${noun} run, and how to join them`, `niches.${slug}`)], + }); +} + +export const copyCountSlugs = Object.freeze([...rules.keys()].sort()); + +function matches(value, rule) { + return typeof value === 'string' ? [...value.matchAll(rule.pattern)] : []; +} +function metricValue(metrics, key) { + const [group, slug] = key.split('.'); + const value = slug ? metrics[group]?.[slug] : metrics[group]; + // A niche with no listed publishers has no GROUP BY row. + return slug && group === 'niches' && value === undefined ? 0 : value; +} + +// Pure: pass flat approved/current {slug, title, meta_title, ...} snapshots. +// Use metrics read separately from EACH target database when preparing a batch. +export function renderCopyCounts(pages, metrics) { + const changes = [], skipped = []; + const rendered = pages.map(page => { + const fields = rules.get(page.slug); + if (!fields) { skipped.push({ slug: page.slug, reason: 'not_allowlisted' }); return { ...page }; } + const found = Object.entries(fields).map(([field, phrases]) => ({ field, phrases, + hits: phrases.map(rule => matches(page[field], rule)) })); + if (found.every(({ hits }) => hits.every(list => list.length === 0))) { + skipped.push({ slug: page.slug, reason: 'unadopted_copy' }); return { ...page }; + } + const result = { ...page }; + for (const { field, phrases, hits } of found) { + if (hits.some(list => list.length !== 1)) throw new Error(`Ad-network count phrase missing or ambiguous: ${page.slug}.${field}`); + // Work from the end so the original offsets remain valid for two slots. + const replacements = phrases.map((rule, index) => { + const hit = hits[index][0]; + const count = roundedCopyCount(metricValue(metrics, rule.metric), { catalog: rule.catalog }); + const slot = hit.groups.slot; + const replacement = count ? `${slot.startsWith('') ? `${count}` : count} ` : ''; + const [start, end] = hit.indices.groups.slot; + return { start, end, replacement }; + }); + let value = page[field]; + for (const { start, end, replacement } of replacements.sort((a, b) => b.start - a.start)) { + value = value.slice(0, start) + replacement + value.slice(end); + } + if (value !== page[field]) changes.push({ slug: page.slug, field, before: page[field], after: value }); + result[field] = value; + } + return result; + }); + return { pages: rendered, changes, skipped }; +} + +// All populations share one statement snapshot. Networks include zero-site rows; +// the root/stats and classified populations count publishers once, across networks. +export async function readCopyCountMetrics(client) { + const { rows } = await client.query(`with listed as materialized ( + select ad_network_id, publisher_id from public.ad_network_publisher_relationships + where status in ('confirmed', 'stale') + ), population as materialized ( + select p.id, p.niche from public.ad_network_publishers p + join (select distinct publisher_id from listed) l on l.publisher_id = p.id + ), network_counts as ( + select n.slug, count(l.publisher_id)::int as count from public.ad_networks n + left join listed l on l.ad_network_id = n.id group by n.id, n.slug + ), niche_counts as ( + select niche, count(*)::int as count from population where niche is not null group by niche + ) select (select count(*)::int from network_counts) as network_count, + (select count(*)::int from population) as listed, + (select count(*)::int from population where niche is not null) as classified, + coalesce((select jsonb_object_agg(slug, count) from network_counts), '{}'::jsonb) as networks, + coalesce((select jsonb_object_agg(niche, count) from niche_counts), '{}'::jsonb) as niches`); + if (!rows[0]) throw new Error('Ad-network copy metrics missing'); + return rows[0]; +} + +export async function refreshCopyCounts(client, { apply = false } = {}) { + await client.query(apply ? 'begin' : 'begin read only'); + try { + await client.query('select pg_advisory_xact_lock(hashtextextended($1, 0))', ['stack127-page-publishing']); + const { rows: pages } = await client.query(`select id, slug, page_family, meta_title + from public.stack127_pages where slug = any($1::text[]) + order by slug ${apply ? 'for update' : ''}`, [copyCountSlugs]); + const { rows: directories } = await client.query(`select page_id, title from public.stack127_directory_pages + where page_id = any($1::bigint[]) order by page_id ${apply ? 'for update' : ''}`, [pages.map(page => page.id)]); + const byId = new Map(directories.map(row => [String(row.page_id), row])); + const metrics = await readCopyCountMetrics(client); + const result = renderCopyCounts(pages.map(page => ({ ...page, title: byId.get(String(page.id))?.title })), metrics); + for (const change of result.changes) { + const page = pages.find(row => row.slug === change.slug); + if (page.page_family !== 'directory') throw new Error(`Wrong page family for count refresh: ${page.slug}`); + if (!apply) continue; + if (change.field === 'meta_title') await client.query('update public.stack127_pages set meta_title = $2 where id = $1', [page.id, change.after]); + else await client.query('update public.stack127_directory_pages set title = $2 where page_id = $1', [page.id, change.after]); + } + await client.query('commit'); + return { ...result, metrics, applied: apply }; + } catch (error) { + await client.query('rollback'); + throw error; + } +} diff --git a/scripts/ad-networks/revalidate.mjs b/scripts/ad-networks/revalidate.mjs index 29dcf5a5..ebef84d6 100644 --- a/scripts/ad-networks/revalidate.mjs +++ b/scripts/ad-networks/revalidate.mjs @@ -3,6 +3,7 @@ import { parseArgs } from 'node:util'; import { pathToFileURL } from 'node:url'; import { targetPool } from './runtime-db.mjs'; import { targetsForEvents } from '../stack127/cache/targets.mjs'; +import { refreshCopyCounts } from './refresh-copy-counts.mjs'; export function revalidationSite(options) { const site = new URL(options.site || (options.environment === 'production' ? 'https://stack127.com' : 'http://127.0.0.1:3333')); @@ -17,8 +18,11 @@ export async function revalidateAdNetworks(options, dependencies = {}) { const targets = targetsForEvents([{ entity_type: 'ad_networks', slug: 'all' }]); if (!options.apply) return { applied: false, environment: options.environment, paths: targets.paths.length, tags: targets.nextTags }; const pool = dependencies.pool ?? targetPool(options); + let client; try { - const credential = (await pool.query('select signing_key from public.stack127_cache_credentials where id')).rows[0]; + client = await pool.connect(); + await (dependencies.refreshCopyCounts ?? refreshCopyCounts)(client, { apply: true }); + const credential = (await client.query('select signing_key from public.stack127_cache_credentials where id')).rows[0]; if (!credential?.signing_key) throw new Error('Cache signing key missing'); const token = createHmac('sha256', credential.signing_key).update('stack127-cache-revalidation-v1').digest('hex'); const response = await (dependencies.fetch ?? fetch)(new URL('/api/cache/revalidate', site), { @@ -29,7 +33,9 @@ export async function revalidateAdNetworks(options, dependencies = {}) { // Never acknowledge queue events here. The existing cache-worker verifies // origin/edge responses (or rebuilds Cloudflare's static export) before ack. return { applied: true, environment: options.environment, origin_revalidated: true, durable_cache_queue: true, paths: targets.paths.length }; - } finally { await pool.end(); } + } finally { + try { client?.release(); } finally { await pool.end(); } + } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/scripts/stack127/cache/worker.mjs b/scripts/stack127/cache/worker.mjs index 611be581..88ddda80 100644 --- a/scripts/stack127/cache/worker.mjs +++ b/scripts/stack127/cache/worker.mjs @@ -20,6 +20,7 @@ import { targetsForEvents } from "./targets.mjs"; import { refreshTargets, cacheBodyDigest, selectRevalidationBatch } from "./refresh.mjs"; import { launchDirectoryRevisionMatches } from "../../../lib/launch-directories-data.ts"; import { dispatchFrontendRefresh, frontendLatencyMinutes } from "../frontend/content-queue.mjs"; +import { refreshCopyCounts } from "../../ad-networks/refresh-copy-counts.mjs"; export const runBudgetMs = 90_000; export const leaseSeconds = 150; @@ -75,9 +76,13 @@ const processedRows = `(select id, created_at::timestamptz as created_at from unnest($1::bigint[], $2::text[]) as input(id, created_at)) done`; const eventColumns = events => [events.map(event => event.id), events.map(event => event.created_at)]; +export const isAdNetworkCopyEvent = event => event.entity_type === "ad_networks" + || (["page", "directory"].includes(event.entity_type) && /^ad-networks(?:\/|$)/.test(event.slug)); + export async function runCacheWorker(client, { env = process.env, fetchImplementation = fetch, now = Date.now, log = console, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + refreshCounts = refreshCopyCounts, } = {}) { const settings = workerSettings(env); const { dryRun, siteUrl, cloudflareFrontend } = settings; @@ -195,10 +200,12 @@ export async function runCacheWorker(client, { } async function processBatch(events, key) { - const targets = await targetsFor(events); - const versions = await fingerprints(events); - const token = revalidationToken(key); + let versions; try { + if (events.some(isAdNetworkCopyEvent)) await refreshCounts(client, { apply: true }); + const targets = await targetsFor(events); + versions = await fingerprints(events); + const token = revalidationToken(key); await refreshTargets(targets, { invalidate: () => invalidateNext(targets, token), rebuild: async paths => { @@ -368,14 +375,24 @@ export async function runCacheWorker(client, { if (cloudflareFrontend) { const events = await readyEvents(); if (events.length) { + let dispatchEvents = events; try { + // The full export captures all due edits, beyond readyEvents' sampled + // batch (and even edits still settling). Include their copy dependency. + const { rows: copyEvents } = await client.query(`select id::text, entity_type, slug, created_at::text as created_at + from public.stack127_revalidation_events where next_attempt_at <= now() + and (entity_type = 'ad_networks' or (entity_type in ('page', 'directory') + and (slug = 'ad-networks' or slug like 'ad-networks/%'))) + order by first_queued_at, id`); + dispatchEvents = [...new Map([...events, ...copyEvents].map(event => [event.id, event])).values()]; + if (dispatchEvents.some(isAdNetworkCopyEvent)) await refreshCounts(client, { apply: true }); const result = await dispatchFrontendRefresh({ repo: settings.frontendRepo, token: settings.frontendToken, fetchImplementation: budgetFetch }); keepDispatchLease = result.dispatched || result.active === true; log.log(JSON.stringify({ hosting: "cloudflare", ...result, acknowledged: 0 })); } catch (error) { // A failed dispatch never removes work or penalizes a newer edit. lastError = String(error).slice(0, 500); - await scheduleRetry(events, error); + await scheduleRetry(dispatchEvents, error); await checkQueueAge(lastError); throw error; } diff --git a/stack127_docs/monitoring.json b/stack127_docs/monitoring.json index 31a86386..45d0742a 100644 --- a/stack127_docs/monitoring.json +++ b/stack127_docs/monitoring.json @@ -3,7 +3,7 @@ "project": "stack127", "about": "Entries for the Automation Monitor dashboard (https://monitor.ravitejaknts.com). One entry per scheduled job, workflow or service. key: / (machines: homelab, bloxodes-vps, stack127-vps, macbook-pro, macbook-air; job = cron run-job.sh name, systemd unit without .timer/.service, launchd label or T3 schedule title), github///, supabase/, or northflank/. name: short plain name. group: the section it is listed under. about: one sentence on what it does, written for the owner. impact (optional): what breaks if it stops. records (optional): stats_job_runs.job_name values it writes. ignore_records: job names that are sub-steps of other jobs.", "automations": [ - {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages) to confirm which ad networks they really use, then sends only the changed rows to production. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, + {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, {"key": "stack127-vps/snapshot", "group": "Ad networks", "name": "Daily ad network counts", "about": "Saves today's publisher counts for each ad network."}, {"key": "stack127-vps/platform", "group": "Ad networks", "name": "Publisher platform and favicon", "about": "Fills in each publisher's website platform and icon, and refreshes facts older than 30 days."}, {"key": "stack127-vps/domain-age", "group": "Ad networks", "name": "Publisher domain age", "about": "Looks up when new publisher domains were registered."}, @@ -24,7 +24,7 @@ {"key": "github/stack127labs/stack127/self-host-apps-refresh-github.yml", "group": "Self-host apps", "name": "Refresh GitHub facts", "about": "Twice a day, refreshes stars, releases and activity for apps hosted on GitHub."}, {"key": "github/stack127labs/stack127/self-host-apps-refresh-source-hosts.yml", "group": "Self-host apps", "name": "Refresh GitLab and Codeberg facts", "about": "Every day, refreshes facts for apps hosted on GitLab and Codeberg."}, {"key": "github/stack127labs/stack127/self-host-apps-refresh-technical.yml", "group": "Self-host apps", "name": "Collect technical details", "about": "Every Monday, collects install methods, languages and other technical details for each app."}, - {"key": "stack127-vps/stack127-cache-worker", "group": "Site cache", "name": "Page cache worker", "about": "Every 30 seconds on the OVH server, checks the database for changed pages and starts a static page rebuild when edits are waiting.", "impact": "Edits do not reach the live site."}, + {"key": "stack127-vps/stack127-cache-worker", "group": "Site cache", "name": "Page cache worker", "about": "Every 30 seconds on the OVH server, checks for changed pages, refreshes ad-network heading counts when their data changes, and starts a static page rebuild when edits are waiting.", "impact": "Edits do not reach the live site."}, {"key": "github/stack127labs/stack127/stack127-cache-maintenance.yml", "group": "Site cache", "name": "Cache maintenance", "about": "Every hour at minute 17, checks cache health and starts a static page rebuild when page refreshes are overdue."}, {"key": "github/stack127labs/stack127/stack127-frontend-refresh.yml", "group": "Site cache", "name": "Rebuild static pages", "about": "Rebuilds the static Cloudflare pages when content edits are waiting. Started by the cache system or by hand."}, {"key": "github/stack127labs/stack127/stack127-ci.yml", "group": "Releases and checks", "name": "Pull request checks", "about": "Runs the core tests and one full build on every pull request."}, diff --git a/tests/pages/ad-networks/copy-counts.test.mjs b/tests/pages/ad-networks/copy-counts.test.mjs new file mode 100644 index 00000000..a5e4e347 --- /dev/null +++ b/tests/pages/ad-networks/copy-counts.test.mjs @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { roundedCopyCount, renderCopyCounts, readCopyCountMetrics, refreshCopyCounts } from '../../../scripts/ad-networks/refresh-copy-counts.mjs'; + +const network = (slug = 'the-moneytizer', name = 'The Moneytizer', count = '1,000+') => ({ + slug: `ad-networks/${slug}`, + meta_title: `${count} sites using ${name}: requirements, pay, stats`, + title: `${count} sites running ${name}, plus its requirements, pay, and traffic stats`, + meta_description: 'Policy checked in 2026: 50,000 sites required; fees of 10% and Net 30 pay.', +}); +const root = { + slug: 'ad-networks', meta_title: '40+ ad networks compared: requirements, pay, 108,000+ sites', + title: 'Compare 40+ ad networks: what they ask, what they pay, and 108,000+ sites using them', +}; +const stats = { slug: 'ad-networks/stats', meta_title: 'Ad network market share (2026): 108,000+ sites checked', + title: 'Which ad networks do 108,000+ sites actually run?' }; +const hub = { slug: 'ad-networks/niches', meta_title: 'Ad networks by niche: what 87,000+ sites in 20 niches run', + title: 'The ad networks 87,000+ sites run, sorted into 20 niches' }; +const niche = (slug, noun, shortened = false) => ({ slug: `ad-networks/niches/${slug}`, + meta_title: `Ad networks for ${noun}: what 8,300+ ${shortened ? 'use' : 'real sites use'}`, + title: `The ad networks 8,300+ ${noun} run, and how to join them` }); +const metrics = { network_count: 44, listed: 108399, classified: 87399, + networks: { 'the-moneytizer': 1015 }, niches: { food: 8399, 'personal-finance': 3478 } }; +const render = (page, counts = metrics) => renderCopyCounts([page], counts).pages[0]; + +test('rounding uses approved tiers on increases and decreases, with catalog tens', () => { + for (const [count, expected] of [[0, ''], [9, ''], [10, '10+'], [14, '10+'], [99, '95+'], + [100, '100+'], [999, '950+'], [1015, '1,000+'], [9999, '9,900+'], [10000, '10,000+'], [108399, '108,000+']]) { + assert.equal(roundedCopyCount(count), expected); + } + assert.equal(roundedCopyCount(44, { catalog: true }), '40+'); + assert.equal(roundedCopyCount(59, { catalog: true }), '50+'); + for (const count of [-1, NaN, undefined, 1.5, Number.MAX_SAFE_INTEGER + 1]) assert.throws(() => roundedCopyCount(count), /Invalid/); +}); + +test('only count slots change, including a decrease from 1,015 to 999', () => { + const page = network(); + const result = render(page, { ...metrics, networks: { 'the-moneytizer': 999 } }); + assert.equal(result.meta_title, page.meta_title.replace('1,000+', '950+')); + assert.equal(result.title, page.title.replace('1,000+', '950+')); + assert.equal(result.meta_description, page.meta_description); + assert.equal(renderCopyCounts([page], metrics).changes.length, 0); +}); + +test('special approved phrases preserve product names, fees, years, and mark wrappers', () => { + const cases = [ + ['journey-by-mediavine', 'Journey by Mediavine', 'Journey by Mediavine: 200+ sites, requirements, and pay'], + ['publisher-collective', 'Publisher Collective', '200+ Publisher Collective sites: requirements, pay, stats'], + ['amazon-publisher-services', 'Amazon Publisher Services', 'Amazon Publisher Services: 200+ sites, fees, and pay terms'], + ['google-adsense', 'Google AdSense', '200+ sites using AdSense: requirements, pay, and stats'], + ['pubgalaxy', 'PubGalaxy', '200+ sites using PubGalaxy, now part of Improve Digital'], + ]; + for (const [slug, name, meta_title] of cases) { + const page = { ...network(slug, name, '200+'), meta_title }; + if (slug === 'pubgalaxy') page.title = '200+ sites still running PubGalaxy, now part of Improve Digital'; + const result = render(page, { networks: { [slug]: 299 } }); + assert.equal(result.meta_title, page.meta_title.replace('200+', '250+')); + assert.equal(result.title, page.title.replace('200+', '250+')); + const empty = render(page, { networks: { [slug]: 0 } }); + assert.ok(!empty.meta_title.includes('200+')); + assert.equal(render(empty, { networks: { [slug]: 299 } }).meta_title, result.meta_title); + } + const marked = { ...network(), title: network().title.replace('1,000+', '1,000+') }; + assert.match(render(marked, { networks: { 'the-moneytizer': 999 } }).title, /^950\+<\/mark> sites/); + const noCount = render(marked, { networks: { 'the-moneytizer': 2 } }); + assert.ok(!noCount.title.includes('')); + assert.match(render(noCount, { networks: { 'the-moneytizer': 10 } }).title, /^10\+ sites/); +}); + +test('root, stats, classified hub, and niche children use separate metrics', () => { + const counts = { ...metrics, network_count: 59, listed: 1015, classified: 999, niches: { food: 78, 'personal-finance': 105 } }; + const result = renderCopyCounts([root, stats, hub, niche('food', 'food blogs'), niche('personal-finance', 'personal finance sites', true)], counts); + assert.match(result.pages[0].meta_title, /^50\+ ad networks.*1,000\+ sites$/); + assert.match(result.pages[1].meta_title, /\(2026\): 1,000\+ sites/); + assert.match(result.pages[2].title, /950\+ sites run, sorted into 20 niches/); + assert.match(result.pages[3].meta_title, /75\+ real sites use/); + assert.match(result.pages[3].title, /75\+ food blogs run/); + assert.match(result.pages[4].meta_title, /100\+ use$/); + const empty = renderCopyCounts([root, stats, hub, niche('food', 'food blogs')], { network_count: 4, listed: 3, classified: 0, niches: {} }); + assert.ok(empty.pages.every(page => !/\d[\d,]*\+/.test(page.title + page.meta_title))); + const recovered = renderCopyCounts(empty.pages, counts); + assert.deepEqual(recovered.pages, result.pages.slice(0, 4)); +}); + +test('old unadopted copy skips explicitly; partial adoption or ambiguous phrases fails', () => { + const old = { slug: 'ad-networks', title: 'Find an ad network for your site', meta_title: 'Ad networks: requirements and pay' }; + assert.deepEqual(renderCopyCounts([old], {}).skipped, [{ slug: old.slug, reason: 'unadopted_copy' }]); + assert.throws(() => render({ ...network(), meta_title: 'Old title' }), /the-moneytizer.meta_title/); + assert.throws(() => render({ ...network(), meta_title: network().meta_title.replace('1,000+', '1000') }), /missing or ambiguous/); + assert.throws(() => render({ ...root, meta_title: `${root.meta_title}; pay, 200+ sites` }), /ambiguous/); + for (const slug of ['publir', 'ethicalads', 'concert', 'sovrn', 'media-net']) { + const page = { slug: `ad-networks/${slug}`, title: 'Approved count-free title', meta_title: 'Requirements and pay (2026)' }; + assert.deepEqual(render(page, { networks: { [slug]: 99999 } }), page); + } +}); + +function database({ pages = [network()], counts = metrics, failUpdate = false } = {}) { + const calls = [], committed = [], pending = []; + return { calls, committed, async query(sql, values) { + const text = sql.replace(/\s+/g, ' ').trim(); calls.push({ text, values }); + if (text.startsWith('select id, slug')) return { rows: pages.map((page, index) => ({ ...page, id: index + 1, page_family: 'directory' })) }; + if (text.startsWith('select page_id')) return { rows: pages.map((page, index) => ({ page_id: index + 1, title: page.title })) }; + if (text.startsWith('with listed as materialized')) return { rows: [counts] }; + if (text.startsWith('update')) { + pending.push({ text, values }); + if (failUpdate && pending.length === 2) throw Error('write failed'); + } + if (text === 'commit') committed.push(...pending.splice(0)); + if (text === 'rollback') pending.length = 0; + return { rows: [] }; + } }; +} + +test('metrics are obtained in one statement with distinct listed publishers and all catalog networks', async () => { + const client = database(); + assert.deepEqual(await readCopyCountMetrics(client), metrics); + assert.equal(client.calls.length, 1); + // Protect the population boundaries: stale stays listed, multi-network sites + // count once, and the catalog keeps its zero-site networks. + const sql = client.calls[0].text; + assert.match(sql, /status in \('confirmed', 'stale'\)/); + assert.match(sql, /select distinct publisher_id from listed/); + assert.match(sql, /from public.ad_networks n left join listed/); +}); + +test('refresh defaults to read-only preview; apply locks in publisher order and same-bucket runs never update', async () => { + const counts = { ...metrics, networks: { 'the-moneytizer': 999 } }; + const preview = database({ counts }); + const plan = await refreshCopyCounts(preview); + assert.equal(plan.applied, false); assert.equal(plan.changes.length, 2); + assert.equal(preview.calls[0].text, 'begin read only'); + assert.ok(!preview.calls.some(call => /for update|^update/.test(call.text))); + const client = database({ counts }); + await refreshCopyCounts(client, { apply: true }); + assert.match(client.calls[1].text, /pg_advisory_xact_lock\(hashtextextended\(\$1, 0\)\)/); + assert.deepEqual(client.calls[1].values, ['stack127-page-publishing']); + assert.match(client.calls[2].text, /stack127_pages.*order by slug for update/); + assert.match(client.calls[3].text, /stack127_directory_pages.*order by page_id for update/); + assert.equal(client.committed.length, 2); + assert.ok(client.committed.every(call => !/description|updated_at|published_at/.test(call.text))); + const sameBucket = database(); + assert.equal((await refreshCopyCounts(sameBucket, { apply: true })).changes.length, 0); + assert.equal(sameBucket.committed.length, 0); +}); + +test('partial adoption and write failures roll back the entire refresh', async () => { + const partial = database({ pages: [network(), { ...network('mediavine', 'Mediavine'), meta_title: 'Old title' }] }); + await assert.rejects(refreshCopyCounts(partial, { apply: true }), /missing or ambiguous/); + assert.equal(partial.calls.at(-1).text, 'rollback'); + assert.equal(partial.committed.length, 0); + const failed = database({ counts: { networks: { 'the-moneytizer': 999 } }, failUpdate: true }); + await assert.rejects(refreshCopyCounts(failed, { apply: true }), /write failed/); + assert.equal(failed.calls.at(-1).text, 'rollback'); + assert.equal(failed.committed.length, 0); +}); + +test('the publisher lock must be acquired before reading or changing managed copy', async () => { + const client = database({ counts: { ...metrics, networks: { 'the-moneytizer': 999 } } }); + const query = client.query.bind(client); + let unlock, reached; + const blocked = new Promise(resolve => { unlock = resolve; }); + const lockReached = new Promise(resolve => { reached = resolve; }); + client.query = async (sql, values) => { + if (sql.includes('pg_advisory_xact_lock')) { reached(); await blocked; } + return query(sql, values); + }; + const work = refreshCopyCounts(client, { apply: true }); + await lockReached; + assert.deepEqual(client.calls.map(call => call.text), ['begin']); + assert.equal(client.committed.length, 0); + unlock(); + await work; + assert.equal(client.committed.length, 2); +}); diff --git a/tests/pages/ad-networks/refresh.test.mjs b/tests/pages/ad-networks/refresh.test.mjs index 73351115..4e2f09b4 100644 --- a/tests/pages/ad-networks/refresh.test.mjs +++ b/tests/pages/ad-networks/refresh.test.mjs @@ -62,17 +62,37 @@ test('sync failures release the shared lock', async () => { test('cache failures leave durable events for the existing worker', async () => { let query; - const pool = { query: async sql => { query = sql; return { rows: [{ signing_key: 'test-only' }] }; }, end: async () => {} }; + const calls = []; + const client = { query: async sql => { query = sql; calls.push('credential'); return { rows: [{ signing_key: 'test-only' }] }; }, + release: () => calls.push('released') }; + const pool = { connect: async () => client, end: async () => calls.push('ended') }; + const refreshCopyCounts = async (connection, options) => { + assert.equal(connection, client); assert.equal(options.apply, true); calls.push('counts'); + }; const options = { environment: 'development', apply: true }; - const result = await revalidateAdNetworks(options, { pool, fetch: async (_url, request) => { + const result = await revalidateAdNetworks(options, { pool, refreshCopyCounts, fetch: async (_url, request) => { + calls.push('http'); const body = JSON.parse(request.body); assert.ok(body.tags.includes('stack127-ad-networks')); return { ok: true, json: async () => ({ ok: true }) }; } }); assert.equal(result.durable_cache_queue, true); assert.match(query, /^select signing_key/); - await assert.rejects(revalidateAdNetworks(options, { pool, fetch: async () => ({ ok: false, status: 503 }) }), /503/); + assert.deepEqual(calls, ['counts', 'credential', 'http', 'released', 'ended']); + await assert.rejects(revalidateAdNetworks(options, { pool, refreshCopyCounts, fetch: async () => ({ ok: false, status: 503 }) }), /503/); + assert.deepEqual(calls.slice(-2), ['released', 'ended']); await assert.rejects(revalidateAdNetworks({ environment: 'development', site: 'https://stack127.com' }), /Invalid revalidation/); }); +test('copy refresh failures prevent HTTP revalidation and always release the session', async () => { + const calls = []; + const client = { release: () => calls.push('released') }; + const pool = { connect: async () => client, end: async () => calls.push('ended') }; + await assert.rejects(revalidateAdNetworks({ environment: 'development', apply: true }, { + pool, refreshCopyCounts: async () => { throw Error('copy mismatch'); }, fetch: () => assert.fail('HTTP after failure'), + }), /copy mismatch/); + assert.deepEqual(calls, ['released', 'ended']); + await revalidateAdNetworks({ environment: 'development' }, { pool: { connect: () => assert.fail('preview opens database') } }); +}); + test('publisher-name refresh checks existing names again', () => { assert.ok(refreshCommand(refreshArgs(['--job', 'publisher-names'])).args.includes('--refresh')); }); diff --git a/tests/stack127/cache-worker.test.mjs b/tests/stack127/cache-worker.test.mjs index 3a7dbd5f..1a63edf1 100644 --- a/tests/stack127/cache-worker.test.mjs +++ b/tests/stack127/cache-worker.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createHmac } from "node:crypto"; -import { edgeFunctionRetired, revalidationToken, runCacheWorker, workerSettings } from "../../scripts/stack127/cache/worker.mjs"; +import { edgeFunctionRetired, isAdNetworkCopyEvent, revalidationToken, runCacheWorker, workerSettings } from "../../scripts/stack127/cache/worker.mjs"; const key = "a".repeat(64); const token = createHmac("sha256", key).update("stack127-cache-revalidation-v1").digest("hex"); @@ -13,7 +13,7 @@ const legacyEnv = { CLOUDFLARE_API_TOKEN: "cf-token", CLOUDFLARE_ZONE_ID: "zone1 // Routes the worker's SQL by its distinctive text, like the content-queue tests. function fakeClient({ events = [], wiring = {}, cronJobs = 0, leaseTaken = false, queue = { pending: 0, minutes: null }, - alertIssue = null, lastReconcileAt = new Date().toISOString(), recovered = 0, history = true, deleted } = {}) { + alertIssue = null, lastReconcileAt = new Date().toISOString(), recovered = 0, history = true, deleted, copyEvents } = {}) { const calls = []; const rows = list => ({ rows: list, rowCount: list.length }); const client = { calls, async query(text, values = []) { @@ -26,6 +26,7 @@ function fakeClient({ events = [], wiring = {}, cronJobs = 0, leaseTaken = false if (sql.includes("select last_reconcile_at")) return rows([{ last_reconcile_at: lastReconcileAt }]); if (sql.includes("with sources as materialized")) return rows(Array.from({ length: recovered }, () => ({}))); if (sql.includes("from public.stack127_cache_processed limit 1")) return rows(history ? [{ 1: 1 }] : []); + if (sql.includes("and (entity_type = 'ad_networks'")) return rows(copyEvents ?? events.filter(isAdNetworkCopyEvent)); if (sql.includes("entity_type <> 'self_host_app'")) return rows(events.filter(event => event.entity_type !== "self_host_app")); if (sql.includes("entity_type = 'self_host_app' and")) return rows(events.filter(event => event.entity_type === "self_host_app")); if (sql.includes("where page.page_family = 'catalog'")) return rows([]); @@ -378,3 +379,96 @@ test("verified deletion responses are warmed as 404s and acknowledged as deleted const recorded = client.calls.find(call => call.text.startsWith("insert into public.stack127_cache_processed")); assert.deepEqual(recorded.values[2], ["deleted"]); }); + +test("only ad-network source and registry/directory events trigger copy refresh", () => { + for (const event of [ + { entity_type: "ad_networks", slug: "all" }, + { entity_type: "page", slug: "ad-networks" }, + { entity_type: "page", slug: "ad-networks/niches/food" }, + { entity_type: "directory", slug: "ad-networks/mediavine" }, + ]) assert.equal(isAdNetworkCopyEvent(event), true); + for (const event of [pageEvent, { entity_type: "page", slug: "ad-networks-other" }, + { entity_type: "self_host_app", slug: "ad-networks" }]) assert.equal(isAdNetworkCopyEvent(event), false); +}); + +for (const hosting of ["legacy", "cloudflare"]) { + const env = hosting === "legacy" ? legacyEnv : { STACK127_FRONTEND_HOSTING: "cloudflare", GITHUB_FRONTEND_TOKEN: "token" }; + test(`${hosting} refreshes counts before any ad-network HTTP processing or dispatch`, async () => { + for (const event of [{ ...pageEvent, entity_type: "ad_networks", slug: "all" }, + { ...pageEvent, slug: "ad-networks/niches/food" }]) { + const client = fakeClient({ events: [event] }); + let refreshed = 0; + const fetch = fakeFetch({ github: (_url, init) => init.method === "POST" + ? new Response(null, { status: 204 }) : json({ total_count: 0, workflow_runs: [] }) }); + const result = await runCacheWorker(client, { + env, log: quiet, + refreshCounts: async (connection, options) => { + assert.equal(connection, client); assert.deepEqual(options, { apply: true }); + assert.equal(fetch.calls.length, 0); refreshed++; + assert.ok(!statements(client).some(sql => sql.includes("stack127_cache_source_revisions($1"))); + return { changes: [] }; // a no-op adds no follow-up work + }, + fetchImplementation: (url, init) => { + assert.equal(refreshed, 1); return fetch.fetchImplementation(url, init); + }, + }); + assert.equal(refreshed, 1); + assert.equal(result.error, null); + assert.equal(result.processed, hosting === "legacy" ? 1 : 0); + } + }); + + test(`${hosting} copy refresh failure retains events, retries, and releases the lease`, async () => { + const event = { ...pageEvent, entity_type: "ad_networks", slug: "all" }; + const client = fakeClient({ events: [event] }); + const fetch = fakeFetch(); + const result = await runCacheWorker(client, { env, log: quiet, fetchImplementation: fetch.fetchImplementation, + refreshCounts: async () => { throw Error("count phrase missing"); }, + }); + assert.match(result.error, /count phrase missing/); + assert.equal(result.processed, 0); + assert.ok(!statements(client).some(sql => sql.startsWith("delete from"))); + assert.ok(statements(client).some(sql => sql.includes("attempts = event.attempts + 1"))); + assert.equal(fetch.calls.length, 0); + assert.equal(client.calls.at(-1).values[1], false); + }); +} + +test("copy refresh queue mutations preserve newer event versions for the next run", async () => { + const event = { ...pageEvent, slug: "ad-networks" }; + const client = fakeClient({ events: [event], deleted: [] }); + const result = await runCacheWorker(client, { env: legacyEnv, log: quiet, + fetchImplementation: fakeFetch().fetchImplementation, refreshCounts: async () => ({ changes: [{ slug: event.slug }] }), + }); + assert.equal(result.error, null); + assert.equal(result.processed, 0); + assert.ok(!statements(client).some(sql => sql.startsWith("insert into public.stack127_cache_processed"))); +}); + +test("unrelated and dry-run events never invoke the copy refresher", async () => { + for (const [events, env] of [[[pageEvent], legacyEnv], + [[{ ...pageEvent, slug: "ad-networks" }], { ...legacyEnv, CACHE_WORKER_DRY_RUN: "1" }]]) { + await runCacheWorker(fakeClient({ events }), { env, log: quiet, + fetchImplementation: fakeFetch().fetchImplementation, refreshCounts: () => assert.fail("unexpected copy refresh"), + }); + } +}); + +test("frontend exports refresh due ad-network events beyond the sampled batch and retry their exact versions", async () => { + const hidden = { ...pageEvent, id: "9007199254740994", slug: "ad-networks/niches" }; + const client = fakeClient({ events: [pageEvent], copyEvents: [hidden] }); + const fetch = fakeFetch(); + let refreshed = 0; + const result = await runCacheWorker(client, { + env: { STACK127_FRONTEND_HOSTING: "cloudflare", GITHUB_FRONTEND_TOKEN: "token" }, log: quiet, + fetchImplementation: fetch.fetchImplementation, + refreshCounts: async () => { refreshed++; throw Error("hidden copy mismatch"); }, + }); + assert.equal(refreshed, 1); + assert.match(result.error, /hidden copy mismatch/); + assert.equal(fetch.calls.length, 0); + const retry = client.calls.find(call => call.text.includes("attempts = event.attempts + 1")); + assert.deepEqual(retry.values[0], [pageEvent.id, hidden.id]); + assert.deepEqual(retry.values[1], [pageEvent.created_at, hidden.created_at]); + assert.ok(!statements(client).some(sql => sql.startsWith("delete from"))); +});