diff --git a/docs/build_docs.py b/docs/build_docs.py index 786f4ca..903f5a4 100644 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -12,11 +12,11 @@ import tomllib from datetime import datetime from fnmatch import fnmatchcase -from html import escape +from html import escape, unescape from html.parser import HTMLParser from importlib import import_module from pathlib import Path -from urllib.parse import urljoin +from urllib.parse import urljoin, urlsplit from xml.etree import ElementTree import jupytext @@ -327,6 +327,237 @@ def sitemap_last_modified(html: Path, doctrees: Path) -> dict[str, str]: } +def navigation_head_html() -> str: + return """ + +""" + + +def html_fragment(contents: str, class_name: str) -> str: + class Fragment(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.lines = [0, *(match.end() for match in re.finditer("\n", contents))] + self.depth = 0 + self.start = 0 + self.matches: list[str] = [] + + def position(self) -> int: + line, column = self.getpos() + return self.lines[line - 1] + column + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + if self.depth: + self.depth += tag not in ArticleText.void_tags + elif class_name in (dict(attrs).get("class") or "").split(): + self.start = self.position() + self.depth = 1 + + def handle_startendtag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + pass + + def handle_endtag(self, tag: str) -> None: + if not self.depth: + return + self.depth -= 1 + if not self.depth: + end = contents.index(">", self.position()) + 1 + self.matches.append(contents[self.start : end]) + + parser = Fragment() + parser.feed(contents) + if parser.depth or len(parser.matches) != 1: + raise ValueError(f"Expected one complete .{class_name} element") + return parser.matches[0] + + +def include_api_navigation(site: Path) -> None: + base_url = os.environ.get("BASE_URL", "").rstrip("/") + source = DOCS / "build" / "api" / "index.html" + navigation = html_fragment(source.read_text(), "docs-api-tree") + links = [] + for href, title in re.findall( + r']*\bhref="([^"]+)"[^>]*>(.*?)', navigation, re.DOTALL + ): + href = unescape(href) + if urlsplit(href).scheme or href.startswith("//") or not href.endswith(".html"): + raise ValueError(f"Expected an API page link in {source}: {href}") + links.append( + { + "title": " ".join(unescape(re.sub(r"<[^>]+>", "", title)).split()), + "url": urljoin(f"{base_url}/api/", href), + "open_in_same_tab": True, + } + ) + if not links: + raise ValueError(f"No API navigation links in {source}") + config = yaml.safe_load((site / "myst.yml").read_text()) + entries = list(config["project"]["toc"]) + matches = [] + while entries: + entry = entries.pop() + if entry.get("url") == f"{base_url}/api/index.html": + matches.append(entry) + entries.extend(entry.get("children", [])) + if len(matches) != 1: + raise ValueError(f"Expected one API reference entry in {site / 'myst.yml'}") + matches[0]["children"] = links + (site / "myst.yml").write_text(yaml.safe_dump(config, sort_keys=False)) + + +def integrate_api_navigation(html: Path) -> None: + def static_classes(match: re.Match[str]) -> str: + classes = [ + token + for token in match[1].split() + if ( + token.startswith("myst-") + or token in {"collapsible-content", "sr-only", "hide-mac", "show-mac"} + ) + and token not in {"myst-toc-item-exact", "myst-search-bar-disabled"} + ] + if "myst-search-bar" in classes: + classes.append("search-button__button") + if "myst-top-nav-menu-button" in classes: + classes.extend(("sidebar-toggle", "primary-toggle")) + return f'class="{" ".join(classes)}"' + + home = (html / "index.html").read_text() + header = html_fragment(home, "myst-top-nav-bar") + header = re.sub(r"]*>.*?", "", header, flags=re.DOTALL) + header = re.sub(r'\bclass="([^"]*)"', static_classes, header) + header, count = re.subn( + r']*\bclass="[^"]*\bmyst-search-bar\b[^"]*"[^>]*>', + lambda match: re.sub( + r' (?:aria-expanded|data-state|aria-label|aria-controls)="[^"]*"', + "", + match[0][:-1], + ) + + ' aria-label="Search documentation" aria-controls="pst-search-dialog">', + header, + ) + if count != 1: + raise ValueError("Expected one MyST search button in the guide header") + navigation = html_fragment(home, "myst-primary-sidebar-toc") + navigation = re.sub(r' aria-current="[^"]*"', "", navigation) + navigation = re.sub(r'\bclass="([^"]*)"', static_classes, navigation) + base_url = os.environ.get("BASE_URL", "").rstrip("/") + for page in (html / "api").glob("*.html"): + contents = page.read_text() + route = f"{base_url}/{page.relative_to(html).as_posix()}" + + def mark_current(match: re.Match[str]) -> str: + tag = match[0] + href = re.search(r'\bhref="([^"]+)"', tag) + return ( + tag[:-1] + ' aria-current="page">' + if href and unescape(href[1]) == route + else tag + ) + + current_navigation = re.sub(r"]*>", mark_current, navigation) + contents = contents.replace( + html_fragment(contents, "docs-header-placeholder"), header + ) + contents = contents.replace( + html_fragment(contents, "docs-api-tree"), current_navigation + ) + if contents.count("") != 1: + raise ValueError(f"Expected one HTML head in {page}") + page.write_text( + contents.replace("", navigation_head_html() + "", 1) + ) + + +def update_page_metadata(contents: str, canonical: str) -> str: + class Description(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.values: list[str] = [] + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + attributes = dict(attrs) + if tag == "meta" and attributes.get("name") == "description": + self.values.append(attributes.get("content") or "") + + if contents.count("") != 1: + raise ValueError(f"Expected one HTML head for {canonical}") + head, body = contents.split("", 1) + titles = re.findall(r"(.*?)", head, re.DOTALL) + description = Description() + description.feed(head) + if len(titles) != 1 or len(description.values) != 1 or not description.values[0]: + raise ValueError(f"Expected a title and description for {canonical}") + title = unescape(titles[0]) + tool = PUBLIC_URL.rsplit("/", 1)[-1] + parts = re.split(r" (?:-|—) ", title) + if len(parts) == 2 and ( + parts[0] == parts[1] + or canonical == f"{PUBLIC_URL}/api/index.html" + and parts[1] == f"{tool} API reference" + ): + title = parts[0] + if canonical == f"{PUBLIC_URL}/" and title == tool: + title = f"{tool} documentation" + if ( + canonical == f"{PUBLIC_URL}/api/{tool.lower()}.html" + and title == f"{tool} API reference" + ): + title = f"{tool} Python API" + image = f"https://docs.nygen.io/assets/brand/logo/{tool.lower()}-dark.png" + metadata = { + "og:title": title, + "og:description": description.values[0], + "og:type": "website", + "og:url": canonical, + "og:image": image, + "og:image:type": "image/png", + "og:image:width": "512", + "og:image:height": "512", + "og:image:alt": f"{tool} logo", + "twitter:card": "summary", + "twitter:image": image, + "twitter:image:alt": f"{tool} logo", + } + + def remove_managed(match: re.Match[str]) -> str: + key = re.search(r"""\b(?:name|property)=["']([^"']+)["']""", match[0]) + return "" if key and key[1] in metadata else match[0] + + head = re.sub(r"]*>", remove_managed, head) + head = re.sub( + r".*?", + lambda _: f"{escape(title)}", + head, + count=1, + flags=re.DOTALL, + ) + for name, value in metadata.items(): + attribute = "property" if name.startswith("og:") else "name" + head += f'' + return head + "" + body + + def write_discovery_files( html: Path, api_text: Path, @@ -390,7 +621,7 @@ def write_discovery_files( page = html / route.lstrip("/") if route.endswith("/"): page /= "index.html" - contents = page.read_text() + contents = update_page_metadata(page.read_text(), canonical) canonical_tag = ( f'' ) @@ -401,28 +632,7 @@ def write_discovery_files( f'' ) if route.endswith("/"): - script_url = "https://docs.nygen.io/assets/brand/docs.js" - links += ( - '' - ) - links += """""" - links += f'' + links += navigation_head_html() if not count: links = canonical_tag + links if contents.count("") != 1: @@ -767,29 +977,47 @@ def prepare_site_template(site: Path) -> None: check=True, ) replacements = { - "build/index.js": ( - 'Pje.default.useState(i);(0,Pje.useEffect)(()=>{o.state==="idle"' - "&&s(tzt([t],e,r).includes(t.id))}", - 'Pje.default.useState(!0);(0,Pje.useEffect)(()=>{o.state==="idle"&&s(!0)}', - ), - "public/build/_shared/chunk-LS4ALTVG.js": ( - '_0.default.useState(o);(0,_0.useEffect)(()=>{i.state==="idle"' - "&&a(DS([t],e,r).includes(t.id))}", - '_0.default.useState(!0);(0,_0.useEffect)(()=>{i.state==="idle"&&a(!0)}', - ), + "build/index.js": [ + ( + 'Pje.default.useState(i);(0,Pje.useEffect)(()=>{o.state==="idle"' + "&&s(tzt([t],e,r).includes(t.id))}", + 'Pje.default.useState(!0);(0,Pje.useEffect)(()=>{o.state==="idle"&&s(!0)}', + ), + ( + 'title:o!=null&&o.title?`${o.title}${a?` - ${a}`:""}`:a', + 'title:o!=null&&o.title?o.title===a?`${a} documentation`:`${o.title}${a?` - ${a}`:""}`:a', + ), + ], + "public/build/_shared/chunk-LS4ALTVG.js": [ + ( + '_0.default.useState(o);(0,_0.useEffect)(()=>{i.state==="idle"' + "&&a(DS([t],e,r).includes(t.id))}", + '_0.default.useState(!0);(0,_0.useEffect)(()=>{i.state==="idle"&&a(!0)}', + ), + ], + "public/build/_shared/chunk-WGW5MSUV.js": [ + ( + 'title:m!=null&&m.title?`${m.title}${g?` - ${g}`:""}`:g', + 'title:m!=null&&m.title?m.title===g?`${g} documentation`:`${m.title}${g?` - ${g}`:""}`:g', + ), + ], } fingerprint = hashlib.sha256( json.dumps(replacements, sort_keys=True).encode() ).hexdigest()[:12] - for filename, (original, expanded) in replacements.items(): + for filename, rules in replacements.items(): path = template / filename if not path.is_file() and filename.startswith("public/"): - path = path.with_stem(f"{path.stem}-{fingerprint}") - content = path.read_text() - if content.count(original) == 1 and expanded not in content: - path.write_text(content.replace(original, expanded)) - elif original in content or content.count(expanded) != 1: - raise ValueError(f"Unexpected navigation component in {path}") + matches = list(path.parent.glob(f"{path.stem}-*.js")) + if len(matches) != 1: + raise ValueError(f"Expected one cached theme module for {path}") + path = matches[0] + for original, replacement in rules: + content = path.read_text() + if content.count(original) == 1 and replacement not in content: + path.write_text(content.replace(original, replacement)) + elif original in content or content.count(replacement) != 1: + raise ValueError(f"Unexpected theme component in {path}") modules = list((template / "public" / "build").rglob("*.js")) filenames = { path.name: f"{path.stem}-{fingerprint}.js" @@ -821,12 +1049,18 @@ def main() -> None: ) args = parser.parse_args() site = prepare_site(preview=args.preview) + include_api_navigation(site) prepare_site_template(site) subprocess.run( ["myst", "build", "--html", "--strict"], cwd=site, check=True, ) + shutil.copytree( + DOCS / "build" / "api", + SITE / "_build" / "html" / "api", + ignore=shutil.ignore_patterns(".doctrees"), + ) revision = subprocess.run( ["git", "describe", "--always", "--dirty", "--abbrev=40"], cwd=DOCS.parent, @@ -843,6 +1077,7 @@ def main() -> None: ), ) link_shared_brand_assets(SITE / "_build" / "html") + integrate_api_navigation(SITE / "_build" / "html") if __name__ == "__main__": diff --git a/docs/source/myst.yml b/docs/source/myst.yml index 74c7732..396d973 100644 --- a/docs/source/myst.yml +++ b/docs/source/myst.yml @@ -12,7 +12,6 @@ project: - reference/api/** static_files: - llms.txt - - ../build/api toc: - file: index.md - title: Getting started diff --git a/docs/source/reference/api/_templates/docs-navigation.html b/docs/source/reference/api/_templates/docs-navigation.html new file mode 100644 index 0000000..4c7d26a --- /dev/null +++ b/docs/source/reference/api/_templates/docs-navigation.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/source/reference/api/_templates/layout.html b/docs/source/reference/api/_templates/layout.html new file mode 100644 index 0000000..baa2db0 --- /dev/null +++ b/docs/source/reference/api/_templates/layout.html @@ -0,0 +1,31 @@ +{% extends "sphinx_book_theme/layout.html" %} + +{% block css %} + +{{ super() }} +{% endblock %} + +{% block extrahead %} +{{ super() }} +{% if pagename == "search" %} + +{% endif %} +{% endblock %} + +{% block body_tag %} + +{% endblock %} + +{% block docs_navbar %} +
+ +
+{% endblock %} diff --git a/docs/source/reference/api/conf.py b/docs/source/reference/api/conf.py index cc41538..2b07c2e 100644 --- a/docs/source/reference/api/conf.py +++ b/docs/source/reference/api/conf.py @@ -43,6 +43,8 @@ always_use_bars_union = True html_theme = "sphinx_book_theme" +templates_path = ["_templates"] +html_sidebars = {"**": ["docs-navigation.html"]} html_css_files = ["https://docs.nygen.io/assets/brand/docs.css"] html_logo = "https://docs.nygen.io/assets/brand/logo/cytetype.svg" html_favicon = "https://docs.nygen.io/assets/brand/logo/cytetype.ico" @@ -52,7 +54,11 @@ "logo": {"text": "CyteType", "alt_text": "CyteType"}, "repository_url": "https://github.com/NygenAnalytics/CyteType", "path_to_docs": "docs/source/reference/api", - "use_repository_button": True, + "article_header_start": [], + "article_header_end": ["toggle-secondary-sidebar.html"], + "show_prev_next": False, + "footer_content_items": ["extra-footer.html"], + "extra_footer": 'Nygen documentation home', "show_navbar_depth": 2, "navigation_with_keys": False, } diff --git a/tests/test_docs_render.py b/tests/test_docs_render.py index 9e68d9f..1b29abe 100644 --- a/tests/test_docs_render.py +++ b/tests/test_docs_render.py @@ -5,6 +5,7 @@ from copy import deepcopy from datetime import datetime +from html import unescape from html.parser import HTMLParser import json import os @@ -724,7 +725,8 @@ def rendered_discovery_files( page = html / route.strip("/") / "index.html" page.parent.mkdir(parents=True, exist_ok=True) page.write_text( - f"{name}" + f'{name}' f"

{name}

" "

Saved result: 42 cells.

" "
def count():\n    return 42\n\nprint(count())\n42
" @@ -757,7 +759,9 @@ def rendered_discovery_files( "# CyteType class\n\n```{autoclass} cytetype.CyteType\n:members:\n```\n" ) (html / "api" / "cytetype.html").write_text( - 'CyteType class" + '' + '' "CyteType(adata, group_key, rank_key='rank_genes_groups')" ) @@ -1437,3 +1441,256 @@ def test_staged_config_paths_are_relative_to_the_site(executable_docs: Any) -> N for path, original in zip(paths, originals, strict=True): assert not Path(path).is_absolute(), path assert (site / path).resolve() == (build_docs.SOURCE / original).resolve() + + +@pytest.mark.parametrize("base_url", ["", "/CyteType"]) +def test_api_pages_join_the_staged_guide_navigation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, base_url: str +) -> None: + pytest.importorskip("jupytext") + pytest.importorskip("nbformat") + yaml = pytest.importorskip("yaml") + from docs import build_docs + + api = tmp_path / "docs" / "build" / "api" + api.mkdir(parents=True) + (api / "index.html").write_text( + '" + ) + site = tmp_path / "site" + site.mkdir() + config = { + "project": { + "toc": [ + {"file": "index.md"}, + { + "title": "Reference", + "children": [ + {"url": f"{base_url}/api/index.html", "title": "API reference"} + ], + }, + ] + } + } + (site / "myst.yml").write_text(yaml.safe_dump(config)) + monkeypatch.setattr(build_docs, "DOCS", api.parents[1]) + monkeypatch.setenv("BASE_URL", base_url) + + build_docs.include_api_navigation(site) + + toc = yaml.safe_load((site / "myst.yml").read_text())["project"]["toc"] + assert toc[0] == {"file": "index.md"} + assert toc[1]["children"][0]["children"] == [ + { + "title": "CyteType class", + "url": f"{base_url}/api/cytetype.html", + "open_in_same_tab": True, + }, + { + "title": "Helpers & utilities", + "url": f"{base_url}/api/helpers.html", + "open_in_same_tab": True, + }, + ] + + +@pytest.mark.parametrize("base_url", ["", "/CyteType"]) +def test_api_pages_keep_the_guide_header_navigation_and_existing_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, base_url: str +) -> None: + pytest.importorskip("jupytext") + pytest.importorskip("nbformat") + pytest.importorskip("yaml") + from docs import build_docs + + monkeypatch.setenv("BASE_URL", base_url) + html = tmp_path / "html" + api = html / "api" + api.mkdir(parents=True) + header = ( + '' + ) + navigation = ( + '" + ) + (html / "index.html").write_text( + f"{header}{navigation}" + ) + article = '' + for name in ("index.html", "cytetype.html", "search.html", "genindex.html"): + (api / name).write_text( + '
' + '
' + '
' + '' + f"
{article}" + ) + (api / "objects.inv").write_bytes(b"existing Sphinx inventory") + static = api / "_static" + static.mkdir() + (static / "webpack-macros.html").write_text("{% macro assets() %}{% endmacro %}") + + build_docs.integrate_api_navigation(html) + + for page in api.glob("*.html"): + contents = page.read_text() + copied_header = build_docs.html_fragment(contents, "myst-top-nav-bar") + copied_navigation = build_docs.html_fragment( + contents, "myst-primary-sidebar-toc" + ) + assert 'class="myst-search-bar search-button__button"' in copied_header + assert ( + 'class="myst-top-nav-menu-button sidebar-toggle primary-toggle"' + in copied_header + ) + assert "myst-search-bar-disabled" not in copied_header + assert 'aria-label="Search documentation"' in copied_header + assert 'aria-controls="pst-search-dialog"' in copied_header + assert "aria-expanded" not in copied_header + assert "data-state" not in copied_header + assert "]*aria-current="page"[^>]*>', copied_navigation) + if page.name in {"index.html", "cytetype.html"}: + assert len(selected) == 1 + assert f'href="{base_url}/api/{page.name}"' in selected[0] + else: + assert not selected + assert article in contents + assert contents.count('src="https://docs.nygen.io/assets/brand/docs.js"') == 1 + assert "nygen-docs-sidebar:" in contents + assert "fonts/inter/latin.woff2" in contents + assert (api / "objects.inv").read_bytes() == b"existing Sphinx inventory" + assert ( + static / "webpack-macros.html" + ).read_text() == "{% macro assets() %}{% endmacro %}" + + +def test_html_fragment_rejects_missing_or_duplicate_navigation() -> None: + pytest.importorskip("jupytext") + pytest.importorskip("nbformat") + pytest.importorskip("yaml") + from docs import build_docs + + for contents in ( + "", + '', + ): + with pytest.raises(ValueError, match="Expected one complete .menu element"): + build_docs.html_fragment(contents, "menu") + + +@pytest.mark.parametrize( + ("route", "title", "expected"), + [ + ("/", "CyteType - CyteType", "CyteType documentation"), + ( + "/api/index.html", + "CyteType Python API reference — CyteType API reference", + "CyteType Python API reference", + ), + ( + "/guides/example/", + "Data & labels - CyteType", + "Data & labels - CyteType", + ), + ], +) +def test_page_metadata_preserves_content_and_completes_share_tags( + route: str, title: str, expected: str +) -> None: + pytest.importorskip("jupytext") + pytest.importorskip("nbformat") + pytest.importorskip("yaml") + from docs import build_docs + + class Metadata(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.values: dict[str, list[str]] = {} + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + attributes = dict(attrs) + if tag == "meta": + name = attributes.get("name") or attributes.get("property") + content = attributes.get("content") + assert name is not None and content is not None + self.values.setdefault(name, []).append(content) + + canonical = build_docs.PUBLIC_URL + route + body = "

Keep this title

Keep this content.

" + source = ( + f"{title}" + '' + '' + '' + '' + "" + body + ) + + result = build_docs.update_page_metadata(source, canonical) + + assert result.endswith(body) + assert f"{expected}" in result + assert build_docs.update_page_metadata(result, canonical) == result + metadata = Metadata() + metadata.feed(result) + values = metadata.values + assert values["og:title"] == [unescape(expected)] + assert all(len(value) == 1 for value in values.values()) + assert ( + values["description"] + == values["og:description"] + == ['Compare "labels" & cell types.'] + ) + assert values["og:url"] == [canonical] + assert values["og:type"] == ["website"] + image = "https://docs.nygen.io/assets/brand/logo/cytetype-dark.png" + assert values["og:image"] == values["twitter:image"] == [image] + assert values["og:image:width"] == values["og:image:height"] == ["512"] + assert values["og:image:type"] == ["image/png"] + assert values["og:image:alt"] == values["twitter:image:alt"] == ["CyteType logo"] + assert values["twitter:card"] == ["summary"] + + +@pytest.mark.parametrize( + "head", + [ + "Missing description", + '', + 'Empty description', + ], +) +def test_page_metadata_rejects_missing_authored_metadata(head: str) -> None: + pytest.importorskip("jupytext") + pytest.importorskip("nbformat") + pytest.importorskip("yaml") + from docs import build_docs + + with pytest.raises(ValueError, match="Expected a title and description"): + build_docs.update_page_metadata( + f"{head}Content", + build_docs.PUBLIC_URL + "/", + ) diff --git a/tests/test_docs_structure.py b/tests/test_docs_structure.py index c186829..211f6a4 100644 --- a/tests/test_docs_structure.py +++ b/tests/test_docs_structure.py @@ -188,7 +188,8 @@ def test_navigation_publishes_every_page() -> None: "https://github.com/NygenAnalytics/CyteType/issues", "https://github.com/NygenAnalytics/CyteTypeR/issues", } - assert {"llms.txt", "../build/api"} <= set(project["static_files"]) + assert "llms.txt" in project["static_files"] + assert "../build/api" not in project["static_files"] assert "reference/api/**" in project["exclude"] published = _toc_files()