]*>[\s\S]*?<\/tr>/gi) ?? [];
const entries = [];
for (const row of rows) {
- const cells = [...row.matchAll(/| ]*>([\s\S]*?)<\/td>/g)];
- if (cells.length < 3) continue;
- const [nameCell, typeCell, organisationCell] = cells.map((match) => match[1]);
-
- const href = nameCell.match(/\/domains\/root\/db\/([^."]+)\.html/);
- const displayed = stripTags(nameCell)
- // Strip the bidi control characters IANA wraps RTL names in.
+ if (!row.includes("/domains/root/db/")) continue;
+ const cells = [...row.matchAll(/ | ]*>([\s\S]*?)<\/td>/gi)].map((match) => match[1]);
+ const [nameCell = "", statusCell = "", typeCell = "", organisationCell = ""] = cells;
+ const link = nameCell.match(/]*href=["']\/domains\/root\/db\/([a-z0-9-]+)(?:\.html)?["'][^>]*>([\s\S]*?)<\/a>/i);
+ if (!link || cells.length < 4) {
+ throw new Error("Malformed IANA root database row. Refusing to write a partial dataset.");
+ }
+ const tld = link[1].toLowerCase();
+ const status = stripTags(statusCell).toLowerCase();
+ if (status === "retired") {
+ if (required?.has(tld)) {
+ throw new Error(`IANA marks required ending .${tld} as retired. Refresh the TLD list before retrying. Refusing to write.`);
+ }
+ continue;
+ }
+ // IANA also lists retired and eligible names, sometimes without managers.
+ // Validate every current root-zone ending, not those historical records.
+ if (required && !required.has(tld)) continue;
+ const displayed = stripTags(link[2].split("<")[0])
.replace(/[--]/g, "")
- .replace(/^\./, "")
- .trim();
-
- const tld = (href?.[1] ?? displayed).toLowerCase();
- if (!tld) continue;
-
- const type = stripTags(typeCell);
+ .trim().replace(/^\.|\.$/g, "");
+ if (!displayed) throw new Error(`Missing IANA display name for .${tld}. Refusing to write.`);
+
+ const rawType = stripTags(typeCell).toLowerCase();
+ // The new table includes reserved names outside the root zone. They are not
+ // TLDs in this dataset. .arpa is the active Special Use (infrastructure) row.
+ if (rawType === "special use" && stripTags(statusCell) === "Reserved") continue;
+ const type = rawType === "country code" ? "country-code"
+ : rawType === "special use" && tld === "arpa" ? "infrastructure" : rawType;
const organisation = stripTags(organisationCell);
-
entries.push({
tld,
- displayed: displayed && displayed !== tld ? displayed : null,
+ displayed: displayed !== tld ? displayed : null,
type,
- // IANA writes "Not assigned" for delegated-but-unused endings.
- organisation: organisation && organisation !== "Not assigned" ? organisation : null,
+ organisation: organisation === "Not assigned" ? null : organisation,
assigned: organisation !== "Not assigned",
});
}
+ validateRootDb(entries);
return entries;
}
+/** Validate both parsed facts and coverage before the caller can write anything. */
+export function validateRootDb(entries, requiredTlds = []) {
+ const seen = new Set();
+ for (const entry of entries) {
+ if (seen.has(entry.tld)) throw new Error(`Duplicate IANA row for .${entry.tld}. Refusing to write.`);
+ seen.add(entry.tld);
+ if (!KNOWN_TYPES.has(entry.type)) {
+ throw new Error(`Invalid IANA type for .${entry.tld}: "${entry.type}". Refusing to write.`);
+ }
+ if (!entry.organisation && !(entry.organisation === null && entry.assigned === false)) {
+ throw new Error(`Missing IANA organisation for .${entry.tld}. Refusing to write.`);
+ }
+ }
+ const missing = requiredTlds.filter((tld) => !seen.has(tld));
+ if (missing.length) throw new Error(`Missing IANA rows for: ${missing.join(", ")}. Refusing to write a partial dataset.`);
+}
+
/** Every TLD with at least one DS record in the root is signed. */
function parseSignedTlds(zone) {
const signed = new Set();
@@ -125,7 +163,7 @@ async function main() {
const [rootDbHtml, rootZone] = await Promise.all([get(SOURCES.rootDb), get(SOURCES.rootZone)]);
- const iana = parseRootDb(rootDbHtml);
+ const iana = parseRootDb(rootDbHtml, tldData.tlds);
if (iana.length < MIN_EXPECTED_ROWS) {
throw new Error(
`Only parsed ${iana.length} rows from the IANA root database, expected at least ${MIN_EXPECTED_ROWS}. ` +
@@ -133,6 +171,8 @@ async function main() {
);
}
+ validateRootDb(iana, tldData.tlds);
+
const signed = parseSignedTlds(rootZone);
if (signed.size < 500) {
throw new Error(`Only found ${signed.size} signed endings in the root zone, which looks wrong. Refusing to write.`);
@@ -202,7 +242,9 @@ async function main() {
if (missingType) console.log(`No type from IANA: ${missingType}`);
}
-main().catch((error) => {
- console.error(error.message);
- process.exit(1);
-});
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ main().catch((error) => {
+ console.error(error.message);
+ process.exit(1);
+ });
+}
diff --git a/tests/pages/domain-tools/shared/fixtures/iana-root-db.html b/tests/pages/domain-tools/shared/fixtures/iana-root-db.html
new file mode 100644
index 00000000..0c4250ae
--- /dev/null
+++ b/tests/pages/domain-tools/shared/fixtures/iana-root-db.html
@@ -0,0 +1,197 @@
+
+| TLD | Status | Type | TLD Manager / Purpose | Script | Writing System | Sort |
+
+
+
+
+.aaa
+
+
+
+
+ |
+
+
+
+
+
+Active
+
+ |
+
+
+
+
+
+Generic
+
+ |
+
+American Automobile Association, Inc.
+ |
+
+Latn
+ |
+
+Latin
+ |
+
+en||aaa
+ |
+
+
+
+
+ישראל.
+
+
+
+Hebrew
+
+xn--4dbrk0ce
+
+ |
+
+
+
+
+
+Active
+
+ |
+
+
+
+
+
+Country Code
+
+ |
+
+The Israel Internet Association (RA)
+ |
+
+Hebr
+ |
+
+Hebrew
+ |
+
+he||ישראל
+ |
+
+
+
+
+.bl
+
+
+
+
+ |
+
+
+
+
+
+Eligible
+
+ |
+
+
+
+
+
+Country Code
+
+ |
+
+Not assigned
+ |
+
+Latn
+ |
+
+Latin
+ |
+
+en||bl
+ |
+
+
+
+
+.arpa
+
+
+
+
+ |
+
+
+
+
+
+Active
+
+ |
+
+
+
+
+
+Special Use
+
+ |
+
+Internet Architecture Board (IAB)
+ |
+
+Latn
+ |
+
+Latin
+ |
+
+en||arpa
+ |
+
+
+
+
+.example
+
+
+
+
+ |
+
+
+
+
+
+Reserved
+
+ |
+
+
+
+
+
+Special Use
+
+ |
+
+Reserved for use in documentation and examples.
+ |
+
+Latn
+ |
+
+Latin
+ |
+
+en||example
+ |
+
+
diff --git a/tests/pages/domain-tools/shared/tld-directory-refresh.test.mjs b/tests/pages/domain-tools/shared/tld-directory-refresh.test.mjs
new file mode 100644
index 00000000..8673cde4
--- /dev/null
+++ b/tests/pages/domain-tools/shared/tld-directory-refresh.test.mjs
@@ -0,0 +1,90 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFile, mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { parseRootDb, validateRootDb } from "../../../../scripts/domain-tools/refresh-tld-directory.mjs";
+
+const fixture = await readFile(new URL("./fixtures/iana-root-db.html", import.meta.url), "utf8");
+
+test("parses current columns, clean IDN names, and real unassigned managers", () => {
+ assert.deepEqual(parseRootDb(fixture), [
+ { tld: "aaa", displayed: null, type: "generic", organisation: "American Automobile Association, Inc.", assigned: true },
+ { tld: "xn--4dbrk0ce", displayed: "ישראל", type: "country-code", organisation: "The Israel Internet Association (RA)", assigned: true },
+ { tld: "bl", displayed: null, type: "country-code", organisation: null, assigned: false },
+ { tld: "arpa", displayed: null, type: "infrastructure", organisation: "Internet Architecture Board (IAB)", assigned: true },
+ ]);
+});
+
+test("accepts all supported types and legacy .html links", () => {
+ for (const type of ["generic", "country-code", "sponsored", "infrastructure", "generic-restricted", "test"]) {
+ assert.equal(parseRootDb(fixture.replace("Generic", type))[0].type, type);
+ }
+ assert.equal(parseRootDb(fixture.replace('/db/aaa"', '/db/aaa.html"'))[0].tld, "aaa");
+});
+
+test("rejects blank type, unknown type, and a status in the type column", () => {
+ for (const value of ["", "Active", "unexpected"]) {
+ assert.throws(() => parseRootDb(fixture.replace("Generic", value)), /Invalid IANA type for .aaa/);
+ }
+});
+
+test("rejects a blank manager rather than treating it as Not assigned", () => {
+ assert.throws(() => parseRootDb(fixture.replace("American Automobile Association, Inc.", "")), /Missing IANA organisation for .aaa/);
+ assert.throws(() => parseRootDb(fixture.replace("Not assigned", "")), /Missing IANA organisation for .bl/);
+});
+
+test("rejects changed columns and malformed TLD links", () => {
+ assert.throws(() => parseRootDb(fixture.replace(" | Status | ", "Other | ")), /columns changed/);
+ assert.throws(() => parseRootDb(fixture.replace('/db/aaa"', '/db/aaa/unknown"')), /Malformed IANA/);
+});
+
+test("requires every current root ending and rejects duplicates", () => {
+ const entries = parseRootDb(fixture);
+ assert.throws(() => validateRootDb(entries, ["aaa", "com"]), /Missing IANA rows for: com/);
+ assert.throws(() => validateRootDb([...entries, entries[0]]), /Duplicate IANA row/);
+});
+
+test("validates every current ending while ignoring historical rows outside the root", () => {
+ const brokenHistoricalRow = fixture.replace("Not assigned", "");
+ assert.deepEqual(parseRootDb(brokenHistoricalRow, ["aaa"]), parseRootDb(fixture).slice(0, 1));
+ assert.throws(() => parseRootDb(brokenHistoricalRow, ["aaa", "bl"]), /Missing IANA organisation for .bl/);
+});
+
+test("rejects a retired ending still required by the local TLD list", () => {
+ const retired = fixture.replace("Active", "Retired");
+ assert.throws(() => parseRootDb(retired, ["aaa"]), /required ending .aaa as retired/);
+});
+
+test("skips retired endings outside the local TLD list, even with a blank manager", () => {
+ const retired = fixture.replace("Active", "Retired").replace("American Automobile Association, Inc.", "");
+ const expected = parseRootDb(fixture).filter((entry) => entry.tld !== "aaa");
+ assert.deepEqual(parseRootDb(retired, expected.map((entry) => entry.tld)), expected);
+ assert.deepEqual(parseRootDb(retired), expected);
+});
+
+test("a failed refresh exits with an error and leaves the committed output untouched", async () => {
+ const directory = await mkdtemp(path.join(tmpdir(), "stack127-tld-refresh-"));
+ try {
+ const output = path.join(directory, "lib/domain/data/tld-directory.json");
+ await mkdir(path.dirname(output), { recursive: true });
+ await writeFile(output, "previous valid dataset\n");
+ for (const [invalidHtml, error] of [
+ [fixture.replace("Generic", "Active"), /Invalid IANA type for .aaa/],
+ [fixture.replace("Active", "Retired"), /required ending .aaa as retired/],
+ ]) {
+ const preload = `globalThis.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(invalidHtml)} });`;
+ const result = spawnSync(process.execPath, [
+ "--experimental-strip-types", "--import", `data:text/javascript;base64,${Buffer.from(preload).toString("base64")}`,
+ fileURLToPath(new URL("../../../../scripts/domain-tools/refresh-tld-directory.mjs", import.meta.url)),
+ ], { cwd: directory, encoding: "utf8" });
+ assert.equal(result.status, 1);
+ assert.match(result.stderr, error);
+ assert.equal(await readFile(output, "utf8"), "previous valid dataset\n");
+ }
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});