diff --git a/content/docs/guide/ci-cd-pipeline.md b/content/docs/guide/ci-cd-pipeline.md index 65e9043f4..d79cae441 100644 --- a/content/docs/guide/ci-cd-pipeline.md +++ b/content/docs/guide/ci-cd-pipeline.md @@ -250,9 +250,13 @@ Backend pins live in `e2e/live/ci/backend.env` and must match the `@objectstack/ all**, which is the point of the workflow. It appears in the checks list as **Internal Docs Link Check**. -Runs `scripts/check-doc-links.mjs`, which walks every `.md` / `.mdx` file under `content/docs/` and -asks of each internal markdown link the only question a reader cares about — **does the site serve -this URL?** Four checks, by href shape: +Runs `scripts/check-doc-links.mjs`, which walks every `.md` / `.mdx` file in the surfaces listed in +its `SCAN_ROOTS` — `content/docs/`, `examples/` and the root `README.md` — and asks of each internal +markdown link whether its target is really there. + +**Two rules, because the two groups are read through different machinery** (objectui#3536). For +`content/docs/` the question is the one a site reader cares about, **does the site serve this URL?** +Four checks, by href shape: | Href shape | Resolved against | Rejected when | |---|---|---| @@ -261,8 +265,20 @@ this URL?** Four checks, by href shape: | absolute `/docs/...` | `content/docs/` as a **route** | no `foo.md`, `foo.mdx` or `foo/index.md*` backs it — a `.md`/`.mdx` suffix always fails, since that URL 404s whatever is on disk | | any other absolute (`/spec/...`, `/img/...`) | the **site itself**: route segments enumerated from `apps/site/app`, plus static files under `apps/site/public` | no route pattern or static file matches | -External `http(s)` and `mailto:` links and bare `#anchors` are skipped — those belong to Lychee, -below. No install, no build, no network: a checkout and one `node` call. +`examples/` and the root `README.md` are read on **GitHub**, not served by the site, so a relative +href there names a path on disk and is checked for existence only — a directory (`./packages/core`) +or a non-markdown file (`./vite.config.ts`) is a perfectly good target, and there is no collection +to escape. A leading `/` is rejected outright: GitHub resolves it against `github.com`, not against +this repository. Applying the `content/docs/` rules to these files instead would reject 61 links +that render correctly today. + +One href shape is checked in **every** surface: a +`https://github.com/objectstack-ai/objectui/(blob|tree)/main/` URL points back into this +repository, so `` must exist in the working tree. Other repos' URLs, other refs, and +`#fragments` are not resolvable offline and stay Lychee's job. + +Everything else — external `http(s)` and `mailto:` links, bare `#anchors` — is skipped. No install, +no build, no network: a checkout and one `node` call. The last two rows are objectui#3490. Reading `apps/site` widens the script's responsibility, and that is the deliberate purchase: it is the only way to catch a link to a route that does not exist, @@ -306,7 +322,7 @@ There are **two** link checkers, and they cover different things (objectui#3213) | | Covers | Network | Runs | |---|---|---|---| -| `scripts/check-doc-links.mjs` | **Internal** links in `content/docs/`: relative hrefs, `/docs/...` routes, and every other site-absolute href (against `apps/site`) | No | `docs-links.yml` — every push and PR, no path filter (previous section) | +| `scripts/check-doc-links.mjs` | **Internal** links in `content/docs/` (relative hrefs, `/docs/...` routes, every other site-absolute href against `apps/site`), in `examples/` and the root `README.md` (as paths on disk), plus this repo's own `blob/main/` and `tree/main/` GitHub URLs everywhere | No | `docs-links.yml` — every push and PR, no path filter (previous section) | | Lychee (this workflow) | **External** URLs, plus **relative** in-repo file links, in `content/docs/`, `docs/` and `README.md` | Yes | Weekly cron and manual dispatch | Lychee sweeps **both** documentation trees: `content/docs/` (the 183 pages the site publishes) and diff --git a/scripts/__tests__/check-doc-links.test.ts b/scripts/__tests__/check-doc-links.test.ts index a5e5eaa1d..9a73e323a 100644 --- a/scripts/__tests__/check-doc-links.test.ts +++ b/scripts/__tests__/check-doc-links.test.ts @@ -4,7 +4,17 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { collectBrokenLinks, collectSiteRoutes, routeExists, siteUrlExists, stripCode } from '../check-doc-links.mjs'; +import { + SCAN_ROOTS, + collectBrokenLinks, + collectFiles, + collectSiteRoutes, + diskPathExists, + routeExists, + selfRepoPath, + siteUrlExists, + stripCode, +} from '../check-doc-links.mjs'; /** * objectui#3479 — the behaviour test for `scripts/check-doc-links.mjs`. @@ -35,6 +45,19 @@ import { collectBrokenLinks, collectSiteRoutes, routeExists, siteUrlExists, stri * behind them. The gate now enumerates `apps/site/app` + `apps/site/public` as * its truth source for absolute hrefs, and rejects relative hrefs that resolve * out of the docs collection. The describes at the bottom pin both. + * + * objectui#3536 extended the gate in two directions, and the last three + * describes pin them: + * + * - the SCAN SURFACE now includes `examples/**` and the root `README.md`, + * under a *different* rule — those files are read on GitHub, so a relative + * href there names a path on DISK, not a fumadocs route. The tests that + * matter most are the contrast pairs: the same href accepted under one rule + * and rejected under the other, in both directions; + * - the HREF SHAPE `https://github.com/objectstack-ai/objectui/(blob|tree)/ + * main/` is now resolved against the working tree in every surface — + * an in-repo reference that had been skipped by scheme, and sat dead for + * three months between two gates (#3507). */ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); @@ -72,7 +95,12 @@ function docsRootWith(files: Record): string { } function scan(repo: string): { file: string; href: string; line: number; reason: string }[] { - return collectBrokenLinks(path.join(repo, 'content/docs'), path.join(repo, 'apps/site')); + return collectBrokenLinks(repo); +} + +/** `[href, reason]` for every rejection, in scan order. */ +function rejections(files: Record): [string, string][] { + return scan(repoWith(files)).map((item) => [item.href, item.reason]); } /** The hrefs reported broken, in file order. */ @@ -295,14 +323,63 @@ describe('code is stripped before scanning — markdown syntax quoted in code is }); describe('the repo it guards', () => { - it('has no broken internal docs links', () => { + it('has no broken internal links in any scanned surface', () => { // The other half of objectui#3479: the extended check must land GREEN, not - // arrive with a backlog it merely describes. + // arrive with a backlog it merely describes. objectui#3536 widened what + // "any scanned surface" means — see the next test, which is what stops this + // one from passing because nothing was looked at. const broken = scan(repoRoot); - const report = broken.map((b) => `${path.relative(repoRoot, b.file)}:${b.line} -> ${b.href}`); + const report = broken.map((b) => `[${b.reason}] ${path.relative(repoRoot, b.file)}:${b.line} -> ${b.href}`); expect(report).toEqual([]); }); + it('really scans every surface in SCAN_ROOTS — a green above must mean "checked"', () => { + // objectui#3536. The assertion the widened scan needs most: `collectFiles` + // returns [] for a root that is not there, so a typo in SCAN_ROOTS, or a + // rename of `examples/`, would silently drop a whole surface and leave the + // test above green about a tree it never opened. + const scanned = Object.fromEntries( + SCAN_ROOTS.map((root: { path: string; rule: string }) => [ + root.path, + collectFiles(path.join(repoRoot, root.path)).length, + ]), + ); + + expect(Object.keys(scanned)).toEqual(['content/docs', 'examples', 'README.md']); + expect(scanned['README.md']).toBe(1); + expect(scanned['examples']).toBeGreaterThanOrEqual(4); + expect(scanned['content/docs']).toBeGreaterThanOrEqual(100); + }); + + it('pairs each scan root with the link semantics that root actually has', () => { + // The rule split is the design decision of objectui#3536, so it is pinned + // as data: `examples/**` and the root README are read on GitHub (disk + // paths), `content/docs` is served by fumadocs (site routes). + expect(SCAN_ROOTS).toEqual([ + { path: 'content/docs', rule: 'docs' }, + { path: 'examples', rule: 'disk' }, + { path: 'README.md', rule: 'disk' }, + ]); + }); + + it('has no dead self-repo GitHub URLs — the #3507 sweep, now a gate', () => { + // #3509 cleared this class to zero (25 targets, exactly 2 dead); PR #3506 + // then added 8 more with nothing checking them. This is that sweep, run on + // every push instead of by hand in an issue comment. + const targets = new Set(); + for (const root of SCAN_ROOTS as { path: string }[]) { + for (const file of collectFiles(path.join(repoRoot, root.path)) as string[]) { + for (const match of stripCode(fs.readFileSync(file, 'utf8')).matchAll(/\[[^\]]+\]\(([^)]+)\)/g)) { + const target = selfRepoPath(match[1].trim()); + if (target !== null) targets.add(target); + } + } + } + + expect(targets.size).toBeGreaterThanOrEqual(25); + expect([...targets].filter((target) => !fs.existsSync(path.join(repoRoot, target)))).toEqual([]); + }); + it('exposes routeExists with the context the scan gives it', () => { const root = docsRootWith({ 'guide/a.md': '# A', 'plugins/plugin-charts.mdx': '# Charts' }); const fromFile = path.join(root, 'guide/a.md'); @@ -442,7 +519,249 @@ describe('relative hrefs may not leave the collection — objectui#3490 A-class' ).toEqual([]); }); - it('labels each failure with the check that rejected it', () => { + // The `labels each failure with the check that rejected it` test that stood + // here is superseded by `labels all seven checks distinctly` in the last + // describe — same fixture shape, extended with objectui#3536's three reasons. +}); + +describe('examples/** and the root README are scanned as DISK paths — objectui#3536 extension 1', () => { + const EXAMPLE = 'examples/hello-world/README.md'; + + it('reports the #3486 shape: a link to an example directory that was deleted', () => { + // `examples/hello-world/README.md` really carried `../crm/` and `../todo/` + // after both examples were removed. Two people found them by eye; nothing + // in CI ever would have, because this file was outside the scan surface. + expect( + rejections({ + [EXAMPLE]: '[CRM](../crm/) and [Todo](../todo/)', + 'examples/schema-catalog/README.md': '# Schema Catalog', + }), + ).toEqual([ + ['../crm/', 'example-relative'], + ['../todo/', 'example-relative'], + ]); + }); + + it('reports the PR #3485 shape: a dead example entry in the root README', () => { + expect( + rejections({ + 'README.md': '- [Todo](./examples/todo)\n- [Hello World](./examples/hello-world)', + 'examples/hello-world/README.md': '# Hello World', + }), + ).toEqual([['./examples/todo', 'example-relative']]); + }); + + it('accepts a directory, a non-markdown file, and an extensionless file', () => { + // All three are things GitHub renders happily and `routeCandidates()` knows + // nothing about: the docs rule would reject every one of them. + expect( + rejections({ + 'README.md': '[pkg](./packages/core) [vite](./vite.config.ts) [licence](./LICENSE)', + 'packages/core/index.ts': 'export {}', + 'vite.config.ts': 'export default {}', + LICENSE: 'MIT', + }), + ).toEqual([]); + }); + + it('rejects an extensionless spelling of a markdown file — the docs rule accepts it', () => { + // The sharpest contrast pair. On the site `../plugins/plugin-charts` may + // resolve in the browser, so the docs rule lets it through (see the + // "accepts the extensionless-route spelling" test above). GitHub serves + // FILES: there is nothing at that name, so it is simply a 404. + expect( + rejections({ + 'examples/README.md': '[catalog](./schema-catalog/README)', + 'examples/schema-catalog/README.md': '# Schema Catalog', + }), + ).toEqual([['./schema-catalog/README', 'example-relative']]); + }); + + it('accepts a relative link that leaves the example — there is no collection to escape', () => { + // The other half of the contrast. The identical shape out of content/docs + // is `escapes-collection` (pinned below); out of an example README it is + // exactly how console-starter's README links the app-shell source and a + // docs page, and both render. + expect( + rejections({ + 'examples/console-starter/README.md': [ + '[shell](../../packages/app-shell/src/console/ConsoleShell.tsx)', + '[lookup](../../content/docs/fields/lookup.mdx)', + ].join('\n\n'), + 'packages/app-shell/src/console/ConsoleShell.tsx': 'export {}', + 'content/docs/fields/lookup.mdx': '# Lookup', + }), + ).toEqual([]); + }); + + it('rejects a relative link that climbs out of the repository', () => { + // The one containment rule the disk surface does have: GitHub cannot render + // a path above the repo root, whatever exists on the machine running this. + expect(rejections({ 'examples/README.md': '[up](../../../etc/hosts)' })).toEqual([ + ['../../../etc/hosts', 'example-relative'], + ]); + }); + + it('rejects an absolute /... href — GitHub resolves it against github.com', () => { + // No file in these surfaces carries this shape today; the rejection is + // preventive, and it is what the renderer forces. `/packages/core` in the + // root README links to https://github.com/packages/core, not to this repo. + expect( + rejections({ + 'README.md': '[core](/packages/core)', + 'packages/core/index.ts': 'export {}', + }), + ).toEqual([['/packages/core', 'example-absolute']]); + }); + + it('leaves external URLs and in-page anchors alone here too', () => { + expect( + rejections({ + 'examples/README.md': '[web](https://example.com/nope) [mail](mailto:a@b.c) [top](#quick-start)', + }), + ).toEqual([]); + }); + + it('scans every markdown file under examples/, not only README.md', () => { + expect(rejections({ 'examples/hello-world/NOTES.md': '[gone](./nowhere.md)' })).toEqual([ + ['./nowhere.md', 'example-relative'], + ]); + }); + + it('does not walk into an example dependency tree', () => { + // `pnpm install` puts a real `node_modules` in every example. Its READMEs + // are not ours, and there are tens of thousands of them. + expect( + rejections({ + 'examples/hello-world/node_modules/some-dep/README.md': '[gone](./nowhere.md)', + 'examples/hello-world/dist/README.md': '[gone](./nowhere.md)', + }), + ).toEqual([]); + }); + + it('exposes diskPathExists with the context the scan gives it', () => { + const repo = repoWith({ 'examples/README.md': '# Examples', 'examples/hello-world/vite.config.ts': 'x' }); + const fromFile = path.join(repo, 'examples/README.md'); + + expect(diskPathExists('./hello-world/vite.config.ts', { fromFile, repoRoot: repo })).toBe(true); + expect(diskPathExists('./hello-world/vite.config', { fromFile, repoRoot: repo })).toBe(false); + expect(diskPathExists('/hello-world/vite.config.ts', { fromFile, repoRoot: repo })).toBe(false); + }); + + it('keeps the docs rules off the disk surface and the disk rules off docs', () => { + // One fixture, one target, four links: the same two hrefs judged by both + // rules. If either rule ever leaks into the other's surface, exactly one of + // these four verdicts flips. + const repo = repoWith({ + ...SITE_FIXTURE, + 'content/docs/guide/a.md': '[out](../../../packages/core/README.md) and [bare](../plugins/plugin-charts)', + 'content/docs/plugins/plugin-charts.mdx': '# Charts', + 'examples/README.md': '[out](../packages/core/README.md) and [bare](../content/docs/plugins/plugin-charts)', + 'packages/core/README.md': '# Core', + }); + + expect(scan(repo).map((item) => [path.relative(repo, item.file), item.href, item.reason])).toEqual([ + ['content/docs/guide/a.md', '../../../packages/core/README.md', 'escapes-collection'], + ['examples/README.md', '../content/docs/plugins/plugin-charts', 'example-relative'], + ]); + }); +}); + +describe("this repo's own GitHub blob/tree URLs are resolved offline — objectui#3536 extension 2", () => { + it('reports the #3507 shape: a tree/main URL to a deleted example', () => { + // `examples/crm` and `examples/todo` were deleted in 12b287d8b and the two + // links to them stayed dead about three months: this script skipped them by + // scheme, and lychee (weekly cron, continue-on-error) gates nothing. + expect( + rejections({ + ...SITE_FIXTURE, + 'content/docs/guide/building-crud-app.md': [ + '[CRM](https://github.com/objectstack-ai/objectui/tree/main/examples/crm)', + '[Catalog](https://github.com/objectstack-ai/objectui/tree/main/examples/schema-catalog)', + ].join('\n\n'), + 'examples/schema-catalog/README.md': '# Schema Catalog', + }), + ).toEqual([['https://github.com/objectstack-ai/objectui/tree/main/examples/crm', 'self-repo-url']]); + }); + + it('checks the shape in the disk surfaces too, not only content/docs', () => { + expect( + rejections({ + 'examples/README.md': '[gone](https://github.com/objectstack-ai/objectui/blob/main/packages/gone/README.md)', + }), + ).toEqual([['https://github.com/objectstack-ai/objectui/blob/main/packages/gone/README.md', 'self-repo-url']]); + }); + + it('accepts a blob URL to a real file and a tree URL to a real directory', () => { + expect( + rejections({ + ...SITE_FIXTURE, + 'content/docs/plugins/plugin-charts.mdx': [ + '[README](https://github.com/objectstack-ai/objectui/blob/main/packages/plugin-charts/README.md)', + '[Packages](https://github.com/objectstack-ai/objectui/tree/main/packages)', + ].join('\n\n'), + 'packages/plugin-charts/README.md': '# Charts', + }), + ).toEqual([]); + }); + + it('ignores the fragment and the query — anchors are out of scope', () => { + // `#L42` / `?plain=1` would need the target parsed, which is a different + // gate. The path in front of them is still checked. + expect( + rejections({ + ...SITE_FIXTURE, + 'content/docs/guide/a.md': [ + '[live](https://github.com/objectstack-ai/objectui/blob/main/packages/core/README.md#install)', + '[raw](https://github.com/objectstack-ai/objectui/blob/main/packages/core/README.md?plain=1)', + '[dead](https://github.com/objectstack-ai/objectui/blob/main/packages/gone/README.md#install)', + ].join('\n\n'), + 'packages/core/README.md': '# Core', + }), + ).toEqual([['https://github.com/objectstack-ai/objectui/blob/main/packages/gone/README.md#install', 'self-repo-url']]); + }); + + it('leaves every other GitHub URL alone — nothing else is decidable offline', () => { + // A different repo (lychee's job), a different ref (not in this tree), and + // github.com web routes: the two badge URLs and the issues link that the + // root README really carries. + expect( + rejections({ + 'examples/README.md': [ + '[other repo](https://github.com/objectstack-ai/objectstack/blob/main/packages/gone/README.md)', + '[a tag](https://github.com/objectstack-ai/objectui/blob/v1.2.0/packages/gone/README.md)', + '[a sha](https://github.com/objectstack-ai/objectui/blob/12b287d8b/packages/gone/README.md)', + '[issues](https://github.com/objectstack-ai/objectui/issues)', + '[ci](https://github.com/objectstack-ai/objectui/workflows/CI/badge.svg)', + '[repo root](https://github.com/objectstack-ai/objectui/tree/main)', + ].join('\n\n'), + }), + ).toEqual([]); + }); + + it('exposes selfRepoPath: the path it extracts, and the shapes it declines', () => { + expect(selfRepoPath('https://github.com/objectstack-ai/objectui/blob/main/packages/core/README.md')).toBe( + 'packages/core/README.md', + ); + expect(selfRepoPath('https://github.com/objectstack-ai/objectui/tree/main/packages/#readme')).toBe('packages'); + expect(selfRepoPath('https://github.com/objectstack-ai/objectui/issues')).toBeNull(); + expect(selfRepoPath('https://github.com/objectstack-ai/objectstack/blob/main/a.md')).toBeNull(); + expect(selfRepoPath('./packages/core/README.md')).toBeNull(); + }); + + it('does not let the URL escape the repository', () => { + expect( + rejections({ + 'examples/README.md': '[up](https://github.com/objectstack-ai/objectui/blob/main/../../../etc/hosts)', + }), + ).toEqual([['https://github.com/objectstack-ai/objectui/blob/main/../../../etc/hosts', 'self-repo-url']]); + }); +}); + +describe('every failure carries the reason that rejected it', () => { + it('labels all seven checks distinctly', () => { + // The #3490 test of the same name, extended with objectui#3536's three. + // Every reason here must have a HINTS entry — the next test pins that. const repo = repoWith({ ...SITE_FIXTURE, 'content/docs/guide/a.md': [ @@ -450,7 +769,9 @@ describe('relative hrefs may not leave the collection — objectui#3490 A-class' '[site](/spec/component.md)', '[docs](/docs/guide/a.md)', '[rel](./nowhere.md)', + '[self](https://github.com/objectstack-ai/objectui/blob/main/packages/gone/README.md)', ].join('\n\n'), + 'examples/README.md': '[disk](./nowhere.md) and [abs](/packages/core)', }); expect(scan(repo).map((item) => item.reason)).toEqual([ @@ -458,6 +779,29 @@ describe('relative hrefs may not leave the collection — objectui#3490 A-class' 'site-route', 'docs-route', 'relative', + 'self-repo-url', + 'example-relative', + 'example-absolute', + ]); + }); + + it('prints a hint for every reason the script can produce', () => { + // A reason with no HINTS entry fails silently: the failure line is printed + // and the "how do I fix this" paragraph simply never appears. + const source = fs.readFileSync(path.join(repoRoot, 'scripts/check-doc-links.mjs'), 'utf8'); + const hinted = source + .slice(source.indexOf('const HINTS = {')) + .split('\n};')[0] + .matchAll(/^\s{2}'?([a-z-]+)'?:/gm); + + expect([...hinted].map((match) => match[1]).sort()).toEqual([ + 'docs-route', + 'escapes-collection', + 'example-absolute', + 'example-relative', + 'relative', + 'self-repo-url', + 'site-route', ]); }); }); diff --git a/scripts/check-doc-links.mjs b/scripts/check-doc-links.mjs index ff7e8ec68..60bdd9f08 100644 --- a/scripts/check-doc-links.mjs +++ b/scripts/check-doc-links.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node /** - * Rejects internal documentation links under `content/docs/**` that point nowhere. + * Rejects internal documentation links that point nowhere, across the scan + * surfaces listed in `SCAN_ROOTS` below. * * Run: node scripts/check-doc-links.mjs (also `pnpm docs:check-links`) * Exit: 0 = every internal link resolves, 1 = at least one does not @@ -86,11 +87,12 @@ * `apps/site` moved under it. That failure is *correct* (the link really * would 404), but it does mean the two trees are no longer independent. * - * What is deliberately NOT bought here: `next.config.mjs` rewrites/redirects are - * not modelled (the one rewrite, `/docs/:path*.mdx`, sits under the stricter - * `/docs` branch above and never reaches this one), and the scan surface is - * still only `content/docs` — `examples/**` and the root `README.md` remain - * unscanned, tracked separately. + * (The paragraph that stood here listed two things #3490 deliberately did not + * buy: the scan surface staying at `content/docs`, and `examples/**` + the root + * `README.md` remaining unscanned. objectui#3536 bought both — see the next + * section. What is still not modelled is `next.config.mjs` + * rewrites/redirects; the one rewrite, `/docs/:path*.mdx`, sits under the + * stricter `/docs` branch above and never reaches the site-route branch.) * * **Relative hrefs may not leave the collection.** `../../../packages/foo/ * README.md` resolves on disk, so the pre-#3490 check passed it, but @@ -102,6 +104,88 @@ * else: an absolute `https://github.com/objectstack-ai/objectui/blob/main/...` * URL (the "Package README" form in `content/docs/plugins/*.mdx`). * + * ## Why this file changed again (objectui#3536) + * + * Two extensions, in the two directions the paragraph above had left open. + * + * ### 1. The scan surface: `examples/**` and the root `README.md` + * + * Both had been caught carrying dead links twice, by eye and never by a gate: + * `examples/hello-world/README.md` pointed at `../crm/` and `../todo/` after + * those examples were deleted (#3486 / PR #3495), and the root `README.md` + * listed four dead example entries (PR #3485). `examples/console-starter`'s 14 + * relative links (PR #3524) were checked by a throwaway script its author wrote + * by hand, because nothing in CI would. + * + * **These files do not have docs semantics, and forcing them onto it would be + * wrong.** They are read on GitHub, whose renderer resolves a relative href + * against the file's own directory *in the repository* — so the href names a + * path on DISK, not a fumadocs route. Three consequences, all encoded in the + * `disk` rule below: + * + * - **Existence is the whole question.** A directory is a fine target + * (`./packages/core` renders as a directory listing), and so is a file with + * no extension at all (`./LICENSE`) or one that is not markdown + * (`./src/App.tsx`, `./vite.config.ts`). The docs rule's + * `routeCandidates()` spellings are meaningless here and are not tried. + * - **There is no collection to escape.** `examples/console-starter/README.md` + * links `../../packages/app-shell/src/console/ConsoleShell.tsx` and + * `../../content/docs/fields/lookup.mdx`; both render correctly on GitHub. + * The `escapes-collection` rule that is right for `content/docs` would + * reject them, so it is not applied. The only containment rule is the repo + * boundary: a path resolving above `` cannot be rendered at all. + * - **An extensionless spelling of a markdown file is a dead link, not a + * fragile one.** `../plugins/plugin-charts` is accepted under the docs rule + * (a browser may resolve it); on GitHub it is a 404, because there is no + * file by that name. The disk rule therefore does not accept it. + * + * **Absolute `/...` hrefs in these files: rejected** (`example-absolute`). + * There are none today — the decision is preventive, and it is the one the + * renderer forces. GitHub does not rewrite a leading `/`; it resolves it + * against `github.com` itself, so `[core](/packages/core)` in the root README + * links to `https://github.com/packages/core`, not to this repo's directory. + * The hint names both repairs, because the rare deliberate case (a real + * github.com path) is real: write the repo path relative, or write the + * github.com URL in full. Waving the shape through instead would be the same + * silent waiver #3490 spent 18 live 404s removing. + * + * ### 2. A decidable href shape: this repo's own GitHub blob/tree URLs + * + * `https://github.com/objectstack-ai/objectui/(blob|tree)/main/` is an + * in-repo reference wearing an external URL's clothes, and it fell between the + * two gates (#3507): this script skipped it by scheme, and lychee — the only + * thing that would resolve it — is a weekly cron with `continue-on-error`, so + * it gates nothing. Two such links stayed dead for about three months. The + * backlog was cleared to zero by PR #3509 (25 distinct targets swept, exactly + * the 2 dead), and PR #3506 then introduced 8 more of the shape with nothing + * checking them. This closes that: `` must exist in the working tree. + * + * It applies to **every** surface, `content/docs` included — the shape is + * decidable wherever it is written, and the `escapes-collection` hint above + * actively recommends it, so leaving it unchecked would recommend an unchecked + * form as the fix for a checked one. + * + * Deliberately narrow, because the rest is not decidable offline: + * + * - **Only `main`.** `blob/v1.2.0/...` or `blob//...` names a different + * ref; the working tree cannot answer for it. + * - **Only this repo.** Other repos' GitHub URLs stay external (lychee's job). + * - **Only the path.** `#fragment` (`#L42`, `#readme`) is out of scope, and + * stripped before the check — resolving an anchor means parsing the target, + * which is a different gate. + * - **Only `blob|tree`.** `https://github.com/objectstack-ai/objectui/issues` + * and the `workflows//badge.svg` badge URLs in the root README are + * github.com web routes, not paths in this tree, and stay skipped. + * + * ### Still not bought + * + * The other root-level markdown (`CONTRIBUTING.md`, `ROADMAP.md`, + * `QUICK_REFERENCE.md`, `AGENTS.md`) and the internal `docs/` tree remain + * unscanned. That is a surface, not an oversight: the scan found real dead + * links in two of those files while this was being written, filed separately + * rather than folded in. Adding them is a matter of one `SCAN_ROOTS` row each + * — plus fixing what that turns red. + * * ## Code spans are stripped before scanning * * Required, not tidiness. Extending the scan to relative hrefs turns markdown's @@ -132,14 +216,44 @@ const INLINE_CODE_RE = /(`+)[^`\n]*\1/g; const EXTERNAL_HREF_RE = /^(?:#|[a-zA-Z][a-zA-Z0-9+.-]*:)/; /** Files that turn an App-Router directory into a servable URL. */ const ROUTE_ENTRY_RE = /^(?:page|route)\.(?:js|jsx|ts|tsx|md|mdx)$/; +/** + * This repo's own GitHub URLs, in the one shape that is decidable offline: a + * `main` blob/tree path (objectui#3536). Case-insensitive because github.com + * treats owner and repo that way; the captured path is used as written. + */ +const SELF_REPO_BLOB_RE = /^https:\/\/github\.com\/objectstack-ai\/objectui\/(?:blob|tree)\/main\/(.+)$/i; +/** Never markdown sources of ours, and huge — walking them wastes the scan. */ +const UNSCANNED_DIRS = new Set(['node_modules', 'dist', 'build', '.next', '.turbo', '.git']); + +/** + * The scan surfaces, and the link semantics each one actually has. + * + * `docs` — the fumadocs collection: an href names a site ROUTE, resolved + * through the page index and the `apps/site` router. + * `disk` — files read on GitHub: an href names a PATH in this repository. + * + * The split is the point (objectui#3536). See the header for why applying the + * docs rules to the second group would reject links that render perfectly well. + */ +export const SCAN_ROOTS = [ + { path: 'content/docs', rule: 'docs' }, + { path: 'examples', rule: 'disk' }, + { path: 'README.md', rule: 'disk' }, +]; const blank = (text) => text.replace(/[^\n]/g, ' '); export function walk(dir, files = []) { - for (const entry of readdirSync(dir, { withFileTypes: true })) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return files; // a scan root that does not exist here contributes nothing + } + for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { - walk(fullPath, files); + if (!UNSCANNED_DIRS.has(entry.name)) walk(fullPath, files); continue; } if (/\.(md|mdx)$/.test(entry.name)) { @@ -149,6 +263,16 @@ export function walk(dir, files = []) { return files; } +/** One scan root, which may be a directory to walk or a single markdown file. */ +export function collectFiles(root) { + try { + if (statSync(root).isFile()) return /\.(md|mdx)$/.test(root) ? [root] : []; + } catch { + return []; + } + return walk(root); +} + /** * Replaces every fenced block and inline code span with spaces, leaving the * byte length and every newline untouched so offsets — and therefore reported @@ -189,6 +313,61 @@ function isFile(candidate) { } } +/** + * Existence, the `disk` rule's whole question — a directory counts, and so does + * a file with no extension. GitHub renders links to both. + */ +function pathExists(candidate) { + try { + statSync(candidate); + return true; + } catch { + return false; + } +} + +/** Is `candidate` `root` itself, or inside it? */ +function isInside(root, candidate) { + return candidate === root || candidate.startsWith(root + path.sep); +} + +/** Strips the fragment and query, and decodes escapes — shared by every rule. */ +function hrefPath(href) { + const cleanHref = href.split('#')[0].split('?')[0].trim(); + try { + return decodeURI(cleanHref); + } catch { + return cleanHref; // a malformed escape is checked as written + } +} + +/** + * The in-repo path a self-referential GitHub URL names, or `null` if this href + * is not one (objectui#3536). Fragments are stripped: anchors are out of scope. + */ +export function selfRepoPath(href) { + const match = SELF_REPO_BLOB_RE.exec(hrefPath(href)); + return match ? match[1].replace(/\/+$/, '') : null; +} + +/** + * Does one `disk`-rule href resolve to something in this repository? + * + * @param {string} href + * @param {{ fromFile: string, repoRoot: string }} context + */ +export function diskPathExists(href, { fromFile, repoRoot }) { + const cleanHref = hrefPath(href); + if (!cleanHref) return true; // pure in-page anchor or query + + // GitHub resolves a leading `/` against github.com, never against the repo, + // so this shape cannot name a path here at all. See the header. + if (cleanHref.startsWith('/')) return false; + + const target = path.resolve(path.dirname(fromFile), cleanHref); + return isInside(repoRoot, target) && pathExists(target); +} + /** The extensionless-route spellings of `base`, as fumadocs would serve them. */ function routeCandidates(base) { return [`${base}.md`, `${base}.mdx`, path.join(base, 'index.md'), path.join(base, 'index.mdx')]; @@ -351,31 +530,61 @@ function classifyBroken(href, { fromFile, docsRoot }) { return 'relative'; } +/** Which `disk`-rule check rejected this href — drives the hint printed below. */ +function classifyBrokenDisk(href) { + return hrefPath(href).startsWith('/') ? 'example-absolute' : 'example-relative'; +} + /** - * @param {string} docsRoot the `content/docs` tree to scan - * @param {string} siteRoot `apps/site` — truth source for absolute hrefs + * Judges one href under one scan root's rule. + * + * @returns {string | null} the failing check's name, or `null` when it resolves + */ +function judgeHref(href, context) { + // The self-repo URL check runs FIRST and in every rule: it is the one + // external-looking shape this script can decide, so it must be reached + // before the by-scheme skip below waves it through (objectui#3536). + const selfPath = selfRepoPath(href); + if (selfPath !== null) { + const target = path.resolve(context.repoRoot, selfPath); + return isInside(context.repoRoot, target) && pathExists(target) ? null : 'self-repo-url'; + } + if (EXTERNAL_HREF_RE.test(href)) return null; + + if (context.rule === 'disk') { + return diskPathExists(href, context) ? null : classifyBrokenDisk(href); + } + return routeExists(href, context) ? null : classifyBroken(href, context); +} + +/** + * Scans every surface in `SCAN_ROOTS`, each under its own rule. + * + * Takes the repo root — not a docs root — so a caller cannot configure away a + * scan surface or the `apps/site` truth source and get a green that only means + * "nothing was looked at". `SCAN_ROOTS` is the single list of what is checked. + * + * @param {string} repoRoot the repository to scan * @returns {{ file: string, href: string, line: number, reason: string }[]} */ -export function collectBrokenLinks(docsRoot, siteRoot) { +export function collectBrokenLinks(repoRoot) { const broken = []; - const site = collectSiteRoutes(siteRoot); - - for (const file of walk(docsRoot)) { - const source = stripCode(readFileSync(file, 'utf8')); - MARKDOWN_LINK_RE.lastIndex = 0; - let match; - - while ((match = MARKDOWN_LINK_RE.exec(source)) !== null) { - const href = match[1].trim(); - if (EXTERNAL_HREF_RE.test(href)) continue; - if (routeExists(href, { fromFile: file, docsRoot, site })) continue; - - broken.push({ - file, - href, - line: source.slice(0, match.index).split('\n').length, - reason: classifyBroken(href, { fromFile: file, docsRoot }), - }); + const docsRoot = path.join(repoRoot, 'content', 'docs'); + const site = collectSiteRoutes(path.join(repoRoot, 'apps', 'site')); + + for (const scanRoot of SCAN_ROOTS) { + for (const file of collectFiles(path.join(repoRoot, scanRoot.path))) { + const source = stripCode(readFileSync(file, 'utf8')); + MARKDOWN_LINK_RE.lastIndex = 0; + let match; + + while ((match = MARKDOWN_LINK_RE.exec(source)) !== null) { + const href = match[1].trim(); + const reason = judgeHref(href, { fromFile: file, docsRoot, repoRoot, site, rule: scanRoot.rule }); + if (reason === null) continue; + + broken.push({ file, href, line: source.slice(0, match.index).split('\n').length, reason }); + } } } @@ -405,17 +614,37 @@ const HINTS = { ' 404s even though the file exists. Use an absolute' + ' `https://github.com/objectstack-ai/objectui/blob/main/...` URL instead' + ' (the "Package README" form used throughout content/docs/plugins/).', + 'example-relative': + 'Outside content/docs (examples/**, the root README) a relative link is a' + + ' PATH IN THIS REPO, resolved by GitHub against the linking file — so it' + + ' must name something that exists and lives inside the repository. A' + + ' directory or a non-markdown file is fine; an extensionless spelling of a' + + ' `.md` file is not, since GitHub serves files, not routes.', + 'example-absolute': + 'Outside content/docs a leading `/` is NOT the root of this repository —' + + ' GitHub resolves it against github.com, so `/packages/core` links to' + + ' https://github.com/packages/core. Write the repo path relative to the' + + ' file (`./packages/core`), or, if a github.com URL really was meant,' + + ' write it in full with the scheme.', + 'self-repo-url': + 'A `https://github.com/objectstack-ai/objectui/(blob|tree)/main/...` URL' + + ' points into this repository, so its path is checked against the working' + + ' tree — this one is not there. Fix the path, or link the equivalent' + + ' `/docs/...` page. (Only `main` and only this repo are checked; other' + + ' refs and other repos cannot be resolved offline.)', }; if (invokedDirectly) { - const docsRoot = path.resolve('content/docs'); - const siteRoot = path.resolve('apps/site'); - const broken = collectBrokenLinks(docsRoot, siteRoot); + // Derived from this file's own location, not the cwd: a cwd-relative root + // would make running the script from a subdirectory scan nothing and report + // success — a false green is the one failure mode this gate must not have. + const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + const broken = collectBrokenLinks(repoRoot); if (broken.length > 0) { const targets = new Set(broken.map((item) => `${path.dirname(item.file)}|${item.href.split('#')[0]}`)); console.error( - `Found ${broken.length} broken docs link${broken.length === 1 ? '' : 's'} (${targets.size} distinct target${targets.size === 1 ? '' : 's'}):`, + `Found ${broken.length} broken link${broken.length === 1 ? '' : 's'} (${targets.size} distinct target${targets.size === 1 ? '' : 's'}):`, ); for (const item of broken) { console.error(`- [${item.reason}] ${path.relative(process.cwd(), item.file)}:${item.line} -> ${item.href}`); @@ -427,5 +656,5 @@ if (invokedDirectly) { process.exit(1); } - console.log('Docs links are valid.'); + console.log(`Links are valid across ${SCAN_ROOTS.length} scan roots.`); }