diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7be9f178..3185eaac9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,8 +11,8 @@ repos: pass_filenames: false - id: nav-drift - name: check:nav - entry: task check:nav + name: check:navigation + entry: task check:navigation language: python pass_filenames: false - files: '(^docs/.*\.pages$|^nav\.yml$|^tools/build_nav\.py$)' + files: '(^docs/.*\.pages$|^nav\.yml$|^tools/build_navigation\.py$)' diff --git a/README.md b/README.md index 99cc9fb4f..6c963d19e 100644 --- a/README.md +++ b/README.md @@ -48,24 +48,19 @@ check** - the build fails if any of it regresses: | `tablesort`, `glightbox` | vendored under `docs/assets/`; `tools/localize_bundle_assets.py` rewrites the CDN URLs Zensical bakes into its JS bundle | | Redirects | static stubs under `docs/` | | Comment opt-out | `overrides/partials/comments.html` | -| Tag listings ([#38](https://github.com/zensical/backlog/issues/38)) | `tools/render_tag_listings.py` - **temporary**, see `tasks/spec.md` | -| Tag chip links ([#38](https://github.com/zensical/backlog/issues/38)) | `overrides/partials/tags.html` - **temporary**, same removal trigger | - -The tag-listing renderer expands the `` markers on `/tags/` and -`/tutorials/` after the build, and the `tags.html` override links each page's tag chips to -its section there. Both are deliberately throwaway: the Markdown sources still use -Material's own marker syntax and Zensical's stock template already knows how to render a -linked chip - it just has no listing to point at yet. When Zensical ships listings the -feature works natively, the renderer prints a banner telling you to delete it, and the -override can go with it. - -The two build the anchor slug independently - MiniJinja in the template, Python in the -renderer - so `check_zensical_output.py` asserts that every chip anchor resolves on -`/tags/`. That check is what turns a slug mismatch into a failed build instead of 703 dead -links. - -`task check:nav` additionally fails if `nav.yml` no longer matches the `docs/**/.pages` -files, which remain the source of truth for navigation (`task nav` regenerates it). + +Tag listings and the links from each page's tag chips to them are **native** as of +Zensical 0.0.58. The local stand-ins for both - a post-build renderer and a `tags.html` +partial override - are gone; the Markdown sources still carry Material's own +`` markers, which Zensical now expands itself. + +`check-zensical-output` keeps guarding the result: it asserts that tag chips link +somewhere at all and that every anchor they point at exists on `/tags/`. A slug mismatch +between a chip and its listing would otherwise ship as hundreds of dead links rather than +fail the build. + +`task check:navigation` additionally fails if `nav.yml` no longer matches the `docs/**/.pages` +files, which remain the source of truth for navigation (`task update:navigation` regenerates it). See `tasks/handoff.md` for the full migration notes. diff --git a/Taskfile.yml b/Taskfile.yml index 22114be70..c20d14363 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -63,39 +63,22 @@ tasks: cmds: - task: check:links - task: check:rumdl - - task: check:nav + - task: check:navigation - task: check:output - check:nav: + check:navigation: desc: Fail if nav.yml is out of sync with the docs/**/.pages files deps: - install cmds: - - | - expected=$(mktemp) - trap 'rm -f "$expected"' EXIT - poetry run python tools/build_nav.py > "$expected" - if ! diff -u nav.yml "$expected"; then - echo - echo "nav.yml is out of date with respect to the docs/**/.pages files." - echo "Run 'task nav' and commit the result." - exit 1 - fi - echo "nav.yml matches the .pages files." + - poetry run dec-tool build-navigation --check check:output: desc: Fail if a feature we reimplemented for Zensical regressed in the build deps: - build cmds: - - poetry run python tools/check_zensical_output.py site - - nav: - desc: Regenerate nav.yml from the docs/**/.pages files - deps: - - install - cmds: - - poetry run python tools/build_nav.py > nav.yml + - poetry run dec-tool check-zensical-output check:links: desc: Check outgoing links @@ -136,7 +119,7 @@ tasks: # Zensical bakes unpkg.com URLs into its JS bundle; rewrite them to the # vendored copies. Part of building, not of checking - without it the # published site issues third-party requests. - - poetry run python tools/localize_bundle_assets.py site + - poetry run dec-tool localize-bundle-assets serve: desc: Serve the page on localhost with live reload (no post-build steps) @@ -265,6 +248,13 @@ tasks: PATHS: ./docs/build/integrations/index.md ignore_error: true + update:navigation: + desc: Regenerate nav.yml from the docs/**/.pages files + deps: + - install + cmds: + - poetry run dec-tool build-navigation + public:versions: desc: List public documentation versions deps: diff --git a/tests/test_build_navigation.py b/tests/test_build_navigation.py new file mode 100644 index 000000000..741cc9bb6 --- /dev/null +++ b/tests/test_build_navigation.py @@ -0,0 +1,283 @@ +"""Test the navigation builder""" +from pathlib import Path + +import click +import pytest +from click.testing import CliRunner + +from tools.build_navigation import ( + build_nav_list, + build_navigation, + dir_title, + discover_dir, + expand_dir, + expand_item, + has_markdown, + read_pages, + render_nav, +) + + +def make_docs(root: Path, files: dict[str, str]) -> Path: + """Create a docs tree below root from a {relative path: content} mapping.""" + docs = root / "docs" + docs.mkdir(exist_ok=True) + for rel, content in files.items(): + path = docs / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return docs + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("getting-started", "Getting started"), + ("build", "Build"), + ("with_your_sandbox", "With your sandbox"), + # Capitalisation only happens for all-lowercase names, which is what + # keeps acronyms intact instead of mangling them to "Link Ids Event". + ("link-IDS-event-to-KG", "link IDS event to KG"), + ("cmemc", "Cmemc"), + ], +) +def test_dir_title(name, expected): + """Directory names become titles the way MkDocs derives them""" + assert dir_title(name) == expected + + +def test_read_pages_without_file(tmp_path): + """A directory without a .pages file has no nav configuration""" + assert read_pages(tmp_path) is None + + +def test_read_pages_empty_file(tmp_path): + """An empty .pages file is treated as no configuration, not as an error""" + (tmp_path / ".pages").write_text("") + assert read_pages(tmp_path) is None + + +def test_read_pages_returns_the_parsed_yaml(tmp_path): + """A .pages file is parsed into its YAML mapping""" + (tmp_path / ".pages").write_text("title: Overview\nnav:\n - index.md\n") + assert read_pages(tmp_path) == {"title": "Overview", "nav": ["index.md"]} + + +def test_has_markdown_finds_nested_pages(tmp_path): + """Markdown anywhere below a directory counts, however deeply nested""" + (tmp_path / "deep" / "deeper").mkdir(parents=True) + (tmp_path / "deep" / "deeper" / "page.md").write_text("# Page") + assert has_markdown(tmp_path) is True + + +def test_has_markdown_ignores_directories_without_markdown(tmp_path): + """A directory holding only assets is not navigable""" + (tmp_path / "assets").mkdir() + (tmp_path / "assets" / "logo.svg").write_text("") + assert has_markdown(tmp_path) is False + + +def test_discover_dir_orders_like_mkdocs(tmp_path): + """Index first, then the remaining markdown files, then subdirectories""" + docs = make_docs(tmp_path, { + "guide/zebra.md": "# Zebra", + "guide/index.md": "# Guide", + "guide/alpha.md": "# Alpha", + "guide/nested/index.md": "# Nested", + }) + + assert discover_dir(docs / "guide", Path("guide")) == [ + "guide/index.md", + "guide/alpha.md", + "guide/zebra.md", + {"Nested": "guide/nested/index.md"}, + ] + + +def test_discover_dir_skips_directories_without_markdown(tmp_path): + """Asset directories never reach the navigation""" + docs = make_docs(tmp_path, { + "guide/index.md": "# Guide", + "guide/img/diagram.svg": "", + }) + + assert discover_dir(docs / "guide", Path("guide")) == ["guide/index.md"] + + +def test_expand_dir_prefers_a_nested_pages_file(tmp_path): + """A directory's own .pages wins over discovery, including its order""" + docs = make_docs(tmp_path, { + "guide/.pages": "nav:\n - second.md\n - first.md\n", + "guide/first.md": "# First", + "guide/second.md": "# Second", + }) + + assert expand_dir("Guide", docs / "guide", Path("guide")) == { + "Guide": ["guide/second.md", "guide/first.md"] + } + + +def test_expand_dir_expands_a_directory_without_pages(tmp_path): + """A directory without .pages is walked, never emitted as a bare reference + + Zensical does not resolve a bare directory to a page object: doing so costs + the entry its icon front matter, empties the sidebar under navigation.tabs + and drops every other page in the directory from the navigation. + """ + docs = make_docs(tmp_path, { + "guide/index.md": "# Guide", + "guide/details.md": "# Details", + }) + + resolved = expand_dir("Guide", docs / "guide", Path("guide")) + + assert resolved == {"Guide": ["guide/index.md", "guide/details.md"]} + assert resolved != {"Guide": "guide"} + + +def test_expand_dir_collapses_a_lone_index_page(tmp_path): + """A directory holding only an index page stays a plain link""" + docs = make_docs(tmp_path, {"guide/index.md": "# Guide"}) + + assert expand_dir("Guide", docs / "guide", Path("guide")) == { + "Guide": "guide/index.md" + } + + +def test_expand_dir_without_a_title_returns_bare_children(tmp_path): + """Titleless expansion splices the children into the parent list""" + docs = make_docs(tmp_path, { + "guide/index.md": "# Guide", + "guide/details.md": "# Details", + }) + + assert expand_dir(None, docs / "guide", Path("guide")) == [ + "guide/index.md", + "guide/details.md", + ] + + +def test_expand_item_resolves_a_plain_filename(tmp_path): + """A bare filename becomes a docs-relative path""" + docs = make_docs(tmp_path, {"guide/index.md": "# Guide"}) + + assert expand_item("index.md", docs / "guide", Path("guide")) == "guide/index.md" + + +def test_expand_item_inherits_the_title_from_a_subdirectory(tmp_path): + """An untitled directory entry picks up the title: key of its own .pages""" + docs = make_docs(tmp_path, { + "guide/.pages": "title: Inherited\nnav:\n - index.md\n", + "guide/index.md": "# Guide", + }) + + assert expand_item("guide", docs, Path("")) == {"Inherited": ["guide/index.md"]} + + +def test_expand_item_titles_a_single_file(tmp_path): + """A one-key mapping to a file becomes a titled link""" + docs = make_docs(tmp_path, {"intro.md": "# Intro"}) + + assert expand_item({"Introduction": "intro.md"}, docs, Path("")) == { + "Introduction": "intro.md" + } + + +def test_expand_item_builds_an_inline_section(tmp_path): + """A one-key mapping to a list becomes a section, flattening expansions""" + docs = make_docs(tmp_path, { + "intro.md": "# Intro", + "guide/index.md": "# Guide", + "guide/details.md": "# Details", + }) + + assert expand_item({"Section": ["intro.md", "guide"]}, docs, Path("")) == { + "Section": ["intro.md", "guide/index.md", "guide/details.md"] + } + + +def test_build_nav_list_skips_unresolvable_entries(tmp_path): + """Entries that resolve to nothing drop out instead of failing the build""" + docs = make_docs(tmp_path, {"intro.md": "# Intro"}) + + assert build_nav_list(["intro.md", None], docs, Path("")) == ["intro.md"] + + +def test_render_nav_without_a_pages_file(tmp_path): + """A docs tree with no .pages cannot describe a navigation""" + docs = make_docs(tmp_path, {"index.md": "# Home"}) + + with pytest.raises(click.ClickException): + render_nav(docs) + + +def test_render_nav_without_a_nav_block(tmp_path): + """A .pages file carrying only a title cannot describe a navigation""" + docs = make_docs(tmp_path, {".pages": "title: Docs\n", "index.md": "# Home"}) + + with pytest.raises(click.ClickException): + render_nav(docs) + + +def test_render_nav_keeps_unicode_unescaped(tmp_path): + """Titles survive the YAML dump as characters, not escape sequences""" + docs = make_docs(tmp_path, { + ".pages": "nav:\n - \u00dcberblick: index.md\n", + "index.md": "# Home", + }) + + assert "\u00dcberblick" in render_nav(docs) + + +def test_check_passes_when_in_sync(tmp_path): + """A nav file matching the .pages files exits 0""" + docs = make_docs(tmp_path, {".pages": "nav:\n - index.md\n", "index.md": "# Home"}) + nav = tmp_path / "nav.yml" + + runner = CliRunner() + written = runner.invoke(build_navigation, ["--docs-dir", str(docs), "-o", str(nav)]) + assert written.exit_code == 0 + assert nav.read_text() == "nav:\n- index.md\n" + + checked = runner.invoke( + build_navigation, ["--docs-dir", str(docs), "-o", str(nav), "--check"] + ) + assert checked.exit_code == 0 + assert "matches the .pages files" in checked.output + + +def test_check_fails_and_diffs_on_drift(tmp_path): + """A stale nav file exits 1 and reports what drifted""" + docs = make_docs(tmp_path, {".pages": "nav:\n - index.md\n", "index.md": "# Home"}) + nav = tmp_path / "nav.yml" + nav.write_text("nav:\n- outdated.md\n") + + result = CliRunner().invoke( + build_navigation, ["--docs-dir", str(docs), "-o", str(nav), "--check"] + ) + assert result.exit_code == 1 + assert "-- outdated.md" in result.output + assert "+- index.md" in result.output + assert "Run 'task update:navigation' and commit the result." in result.output + # the stale file is left untouched, so the diff stays reproducible + assert nav.read_text() == "nav:\n- outdated.md\n" + + +def test_check_treats_a_missing_nav_file_as_drift(tmp_path): + """Checking before the file exists reports drift rather than crashing""" + docs = make_docs(tmp_path, {".pages": "nav:\n - index.md\n", "index.md": "# Home"}) + nav = tmp_path / "nav.yml" + + result = CliRunner().invoke( + build_navigation, ["--docs-dir", str(docs), "-o", str(nav), "--check"] + ) + assert result.exit_code == 1 + assert not nav.exists() + + +def test_missing_docs_dir_is_a_usage_error(tmp_path): + """Pointing at a docs tree that is not there fails as a usage error""" + result = CliRunner().invoke( + build_navigation, ["--docs-dir", str(tmp_path / "absent")] + ) + assert result.exit_code == 2 diff --git a/tools/README.md b/tools/README.md index d4340a76c..55a449fc6 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,4 +1,17 @@ # documentation.eccenca.com -> tools -This directory is not used at the moment. +This directory is the `dec-tool` package: every script that builds, generates or +checks part of this site is a subcommand of it. Run `poetry run dec-tool --help` +for the current list, or a subcommand with `--help` for its options. +Nothing here is meant to be called directly - the `Taskfile.yml` targets are the +supported entry points and pass the right options. + +| Command | Used by | Purpose | +| :------ | :------ | :------ | +| `build-navigation` | `task update:navigation`, `task check:navigation` | Build `nav.yml` from the `docs/**/.pages` files; `--check` diffs instead of writing and fails on drift | +| `check-zensical-output` | `task check:output` | Inspect the built `site/` and fail if a feature we reimplemented for Zensical regressed | +| `localize-bundle-assets` | `task build` | Rewrite the third-party asset URLs Zensical bakes into its JavaScript bundle to the vendored copies | +| `update-icons` | `task update:icons` | Fetch the eccenca icon set from the gui-elements repository | +| `update-di-reference` | `task update:di-reference` | Generate the task and operator reference pages from a running Corporate Memory | +| `update-integrations` | `task update:integrations` | Render the integrations page from `data/integrations.yml` | diff --git a/tools/__init__.py b/tools/__init__.py index 56827c4fb..09c8c0786 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -2,6 +2,9 @@ from typing import List import click +from tools.build_navigation import build_navigation +from tools.check_zensical_output import check_zensical_output +from tools.localize_bundle_assets import localize_bundle_assets from tools.update_di_reference import update_di_reference from tools.update_icons import update_icons from tools.update_integrations import update_integrations @@ -10,6 +13,9 @@ def cli(): """documentation.eccenca.com build tool""" +cli.add_command(build_navigation) +cli.add_command(check_zensical_output) +cli.add_command(localize_bundle_assets) cli.add_command(update_icons) cli.add_command(update_di_reference) cli.add_command(update_integrations) diff --git a/tools/build_nav.py b/tools/build_navigation.py similarity index 76% rename from tools/build_nav.py rename to tools/build_navigation.py index 0815cd3e1..727595cd5 100644 --- a/tools/build_nav.py +++ b/tools/build_navigation.py @@ -2,13 +2,13 @@ from __future__ import annotations +import difflib import sys from pathlib import Path +import click import yaml -DOCS_DIR = Path("docs") - def read_pages(directory: Path) -> dict | None: pf = directory / ".pages" @@ -141,25 +141,68 @@ def build_nav_list(nav: list, abs_dir: Path, docs_rel: Path) -> list: return result -def main() -> None: - pages = read_pages(DOCS_DIR) +def render_nav(docs_dir: Path) -> str: + """Render the nav.yml body for a docs tree.""" + pages = read_pages(docs_dir) if not pages or "nav" not in pages: - print("ERROR: docs/.pages missing or has no nav: block", file=sys.stderr) - sys.exit(1) + raise click.ClickException(f"{docs_dir}/.pages missing or has no nav: block") - nav = build_nav_list(pages["nav"], DOCS_DIR, Path("")) + nav = build_nav_list(pages["nav"], docs_dir, Path("")) # Dump with a custom representer that keeps strings unquoted where safe # and preserves unicode (e.g.   in titles) - output = yaml.dump( + return yaml.dump( {"nav": nav}, default_flow_style=False, allow_unicode=True, width=120, indent=2, ) - print(output, end="") -if __name__ == "__main__": - main() +@click.command() +@click.option( + "--docs-dir", + type=click.Path(exists=True, dir_okay=True, file_okay=False), + default="docs", + help="Where to read the .pages files from?", + show_default=True, +) +@click.option( + "--output-file", "-o", + type=click.Path(exists=False, dir_okay=False, file_okay=True), + default="nav.yml", + help="Where to save the navigation to?", + show_default=True, +) +@click.option( + "--check", + is_flag=True, + help="Compare against the output file instead of writing it, exit 1 on drift.", +) +def build_navigation(docs_dir: str, output_file: str, check: bool) -> None: + """Build the navigation from the .pages files.""" + output = render_nav(Path(docs_dir)) + target = Path(output_file) + + if not check: + click.echo(f"Write the navigation of {docs_dir} to {target}") + target.write_text(output) + return + + current = target.read_text() if target.exists() else "" + if current == output: + click.echo(f"{target} matches the .pages files.") + return + + diff = difflib.unified_diff( + current.splitlines(keepends=True), + output.splitlines(keepends=True), + fromfile=str(target), + tofile=f"{docs_dir}/**/.pages", + ) + click.echo("".join(diff), nl=False) + click.echo() + click.echo(f"{target} is out of date with respect to the {docs_dir}/**/.pages files.") + click.echo("Run 'task update:navigation' and commit the result.") + sys.exit(1) diff --git a/tools/check_zensical_output.py b/tools/check_zensical_output.py index b46de4778..18e043985 100644 --- a/tools/check_zensical_output.py +++ b/tools/check_zensical_output.py @@ -13,7 +13,7 @@ but they shout loudly once they start passing, which is the signal to revisit the migration. -Usage: python tools/check_zensical_output.py [site_dir] +Usage: dec-tool check-zensical-output [--site-dir SITE_DIR] """ from __future__ import annotations @@ -22,6 +22,8 @@ import sys from pathlib import Path +import click + # Hosts that may legitimately appear in the output. Everything else must be # vendored locally - see the privacy-plugin replacement in handoff.md. ALLOWED_EXTERNAL_HOSTS = { @@ -156,7 +158,7 @@ def check_tag_chip_links(site: Path, pages: list[Path]) -> None: chips > 0, f"{chips} chip(s) on {linked_pages} page(s) link to /tags/" if chips - else "no tag chips link anywhere - is overrides/partials/tags.html in place?", + else "no tag chips link anywhere - did Zensical stop linking tag chips?", required=True, ) report( @@ -197,11 +199,20 @@ def check_pending(site: Path, pages: list[Path]) -> None: report("revision-dates", revision > 0, f"last-update on {revision} pages (backlog #18)", required=False) -def main() -> int: - site = Path(sys.argv[1] if len(sys.argv) > 1 else "site") +@click.command() +@click.option( + "--site-dir", + type=click.Path(exists=False, dir_okay=True, file_okay=False), + default="site", + help="Which build output should be checked?", + show_default=True, +) +def check_zensical_output(site_dir: str) -> None: + """Check the build output for regressed Zensical workarounds.""" + site = Path(site_dir) if not site.is_dir(): print(f"error: {site}/ not found - run `task build` first", file=sys.stderr) - return 2 + sys.exit(2) pages = html_files(site) print(f"Checking {len(pages)} HTML files in {site}/\n") @@ -224,11 +235,6 @@ def main() -> int: print(f"\n{len(failures)} required check(s) failed:") for item in failures: print(f" - {item}") - return 1 + sys.exit(1) print("All required checks passed.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/localize_bundle_assets.py b/tools/localize_bundle_assets.py index f0d2cc561..5c1e2bb9c 100644 --- a/tools/localize_bundle_assets.py +++ b/tools/localize_bundle_assets.py @@ -18,7 +18,7 @@ asserted to stay unchanged, so that a Zensical upgrade which adds or moves a third-party URL fails the build instead of silently shipping it. -Run after ``zensical build``. Usage: python tools/localize_bundle_assets.py [site_dir] +Run after ``zensical build``. Usage: dec-tool localize-bundle-assets [--site-dir SITE_DIR] """ from __future__ import annotations @@ -27,6 +27,8 @@ import sys from pathlib import Path +import click + # Rewritten to the vendored copies under docs/assets/glightbox/. Zensical writes # a per-page `{"base": "."|".."|...}` into the `__config` element, which is what # the bundle itself uses to build relative URLs; reusing it keeps the rewrite @@ -79,16 +81,25 @@ def check_inert(site: Path) -> list[str]: return problems -def main() -> int: - site = Path(sys.argv[1] if len(sys.argv) > 1 else "site") +@click.command() +@click.option( + "--site-dir", + type=click.Path(exists=False, dir_okay=True, file_okay=False), + default="site", + help="Which build output should be rewritten?", + show_default=True, +) +def localize_bundle_assets(site_dir: str) -> None: + """Rewrite third-party asset URLs in the built JavaScript bundle.""" + site = Path(site_dir) if not site.is_dir(): print(f"error: {site}/ not found - run `task build` first", file=sys.stderr) - return 2 + sys.exit(2) found = bundles(site) if not found: print("error: no assets/javascripts/bundle*.min.js in the build", file=sys.stderr) - return 1 + sys.exit(1) problems: list[str] = [] for bundle in found: @@ -124,11 +135,6 @@ def main() -> int: print(f"\n{len(problems)} problem(s):", file=sys.stderr) for item in problems: print(f" - {item}", file=sys.stderr) - return 1 + sys.exit(1) print(f"[OK] {len(INERT_URLS)} remaining third-party URL(s) are unreachable for this corpus") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())