From 2fd868743125db978b6dd3bd249a5b5a66ed6532 Mon Sep 17 00:00:00 2001 From: Rens Date: Sun, 4 Oct 2026 19:58:04 +0200 Subject: [PATCH 01/16] Add optional Octavo backend to nbdev docs commands Select Octavo when octavo.yml exists in the notebooks directory, retaining Quarto otherwise. - Route docs, README, and CONTRIBUTING through the selected backend - Skip Quarto config generation during prepare for Octavo projects - Support local symbol links through an injected lookup - Reject non-default Quarto file filters for Octavo builds - Fix _nbglob_docs delegation to expose nbglob parameters - Add the nbdev[octavo] extra and build examples Export and Octavo notebook tests pass. Preview and source migration remain pending. --- nbdev/_modidx.py | 13 + nbdev/doclinks.py | 18 +- nbdev/octavo.py | 115 +++++++++ nbdev/processors.py | 4 +- nbdev/quarto.py | 20 +- nbs/api/05_doclinks.ipynb | 18 +- nbs/api/10_processors.ipynb | 4 +- nbs/api/14_quarto.ipynb | 30 ++- nbs/api/21_octavo.ipynb | 500 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 3 +- 10 files changed, 695 insertions(+), 30 deletions(-) create mode 100644 nbdev/octavo.py create mode 100644 nbs/api/21_octavo.ipynb diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index 1456e861a..96592d46a 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -213,6 +213,18 @@ 'nbdev.migrate.migrate_nb': ('api/migrate.html#migrate_nb', 'nbdev/migrate.py'), 'nbdev.migrate.nbdev_migrate': ('api/migrate.html#nbdev_migrate', 'nbdev/migrate.py'), 'nbdev.migrate.nbdev_migrate_config': ('api/migrate.html#nbdev_migrate_config', 'nbdev/migrate.py')}, + 'nbdev.octavo': { 'nbdev.octavo.OctavoProc': ('api/octavo.html#octavoproc', 'nbdev/octavo.py'), + 'nbdev.octavo.OctavoProc.__call__': ('api/octavo.html#octavoproc.__call__', 'nbdev/octavo.py'), + 'nbdev.octavo.OctavoProc.__init__': ('api/octavo.html#octavoproc.__init__', 'nbdev/octavo.py'), + 'nbdev.octavo.OctavoProc.add_links': ('api/octavo.html#octavoproc.add_links', 'nbdev/octavo.py'), + 'nbdev.octavo.OctavoProc.nb_proc': ('api/octavo.html#octavoproc.nb_proc', 'nbdev/octavo.py'), + 'nbdev.octavo.OctavoProc.procs': ('api/octavo.html#octavoproc.procs', 'nbdev/octavo.py'), + 'nbdev.octavo.local_libraries': ('api/octavo.html#local_libraries', 'nbdev/octavo.py'), + 'nbdev.octavo.octavo_config': ('api/octavo.html#octavo_config', 'nbdev/octavo.py'), + 'nbdev.octavo.octavo_docs': ('api/octavo.html#octavo_docs', 'nbdev/octavo.py'), + 'nbdev.octavo.octavo_readme': ('api/octavo.html#octavo_readme', 'nbdev/octavo.py'), + 'nbdev.octavo.render_script': ('api/octavo.html#render_script', 'nbdev/octavo.py'), + 'nbdev.octavo.script_pages': ('api/octavo.html#script_pages', 'nbdev/octavo.py')}, 'nbdev.process': { 'nbdev.process.NBProcessor': ('api/process.html#nbprocessor', 'nbdev/process.py'), 'nbdev.process.NBProcessor.__init__': ('api/process.html#nbprocessor.__init__', 'nbdev/process.py'), 'nbdev.process.NBProcessor._proc': ('api/process.html#nbprocessor._proc', 'nbdev/process.py'), @@ -304,6 +316,7 @@ 'nbdev.quarto._install_linux': ('api/quarto.html#_install_linux', 'nbdev/quarto.py'), 'nbdev.quarto._install_mac': ('api/quarto.html#_install_mac', 'nbdev/quarto.py'), 'nbdev.quarto._nbglob_docs': ('api/quarto.html#_nbglob_docs', 'nbdev/quarto.py'), + 'nbdev.quarto._octavo': ('api/quarto.html#_octavo', 'nbdev/quarto.py'), 'nbdev.quarto._pre': ('api/quarto.html#_pre', 'nbdev/quarto.py'), 'nbdev.quarto._pre_docs': ('api/quarto.html#_pre_docs', 'nbdev/quarto.py'), 'nbdev.quarto._readme_cands': ('api/quarto.html#_readme_cands', 'nbdev/quarto.py'), diff --git a/nbdev/doclinks.py b/nbdev/doclinks.py index 9f49f55fa..9d52e5fcd 100644 --- a/nbdev/doclinks.py +++ b/nbdev/doclinks.py @@ -213,27 +213,27 @@ def _get_exps(mod): def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None # %% ../nbs/api/05_doclinks.ipynb #c1b8c9fa -def _qual_sym(s, settings): +def _qual_sym(s, settings, local=False): "Get qualified nb, py, and github paths for a symbol s" if not isinstance(s,tuple): return s nb,py = s nbbase = urljoin(settings["doc_host"]+'/',settings["doc_baseurl"]) - nb = urljoin(nbbase+'/', nb) + nb = '/' + nb.lstrip('/') if local else urljoin(nbbase+'/', nb) gh = urljoin(settings["git_url"]+'/', f'blob/{settings["branch"]}/{py}') return nb,py,gh -def _qual_mod(mod_d, settings): return {sym:_qual_sym(s, settings) for sym,s in mod_d.items()} -def _qual_syms(entries): +def _qual_mod(mod_d, settings, local=False): return {sym:_qual_sym(s, settings, local) for sym,s in mod_d.items()} +def _qual_syms(entries, local=False): settings = entries['settings'] if 'doc_host' not in settings: return entries - return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings} + return {'syms': {mod:_qual_mod(d, settings, local) for mod,d in entries['syms'].items()}, 'settings':settings} # %% ../nbs/api/05_doclinks.ipynb #53505fa4 _re_backticks = re.compile(r'`([^`\s]+?)(?:\(\))?`') # %% ../nbs/api/05_doclinks.ipynb #3a24b883 @lru_cache(None) -def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None): +def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None, local_lib=None): cfg = get_config(also_settings=True) if strip_libs is None: try: strip_libs = cfg.get('strip_libs') or cfg.lib_name @@ -248,7 +248,7 @@ def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None): for o in eps: if incl_libs is not None and o.dist.name not in incl_libs: continue - try: entries[o.name] = _qual_syms(o.load()) + try: entries[o.name] = _qual_syms(o.load(), local=o.name in tuplify(local_lib)) except Exception: pass py_syms = merge(*L(o['syms'].values() for o in entries.values()).concat()) for m in strip_libs: @@ -266,8 +266,8 @@ def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None): # %% ../nbs/api/05_doclinks.ipynb #3257b6bf class NbdevLookup: "Mapping from symbol names to docs and source URLs" - def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None): - self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods) + def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None, local_lib=None): + self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods, local_lib) self.aliases = {n:o.__name__ for n,o in (ns or {}).items() if isinstance(o, ModuleType)} def __getitem__(self, s): diff --git a/nbdev/octavo.py b/nbdev/octavo.py new file mode 100644 index 000000000..3248e3549 --- /dev/null +++ b/nbdev/octavo.py @@ -0,0 +1,115 @@ +"""Build nbdev documentation with Octavo + +Docs: https://nbdev.fast.ai/api/octavo.html.md""" + +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. + +# %% auto #0 +__all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_readme'] + +# %% ../nbs/api/21_octavo.ipynb #fd0fd32f +from contextlib import redirect_stdout +from io import StringIO +from fastcore.utils import * +import yaml +from .config import get_config, import_obj, _load_toml +from .doclinks import NbdevLookup +from .process import NBProcessor +from .processors import FilterDefaults, add_links +from .serve import _is_qpy +from octavo.site import Site +from octavo.pages import Page, GeneratedFile, scan, page_nm, hidden, ipynb2md +from octavo.core import md2doc +from mdhtml import md2gfm +from .quarto import _doc_mtime_not_older + +# %% ../nbs/api/21_octavo.ipynb #1cc66460 +class OctavoProc(FilterDefaults): + "Process nbdev notebooks for Octavo without changing Quarto defaults" + def __init__(self, local_lib=None, strip_libs=None): self.nl = NbdevLookup(strip_libs=strip_libs or local_lib, local_lib=local_lib) + def add_links(self, cell): return add_links(cell, nl=self.nl) + def procs(self): return L(self.add_links if o is add_links else o for o in super().procs()) + def nb_proc(self, nb): return NBProcessor(nb=nb, procs=self.procs(), rm_directives=False) + + def __call__(self, nb): + super().__call__(nb) + if getattr(nb, 'frontmatter_', None) and nb.cells[0].cell_type=='raw': nb.cells[0].cell_type = 'markdown' + +# %% ../nbs/api/21_octavo.ipynb #9a0258ee +def local_libraries(cfg): + "Names of the project's declared nbdev symbol indexes" + proj = _load_toml(cfg.config_file).get('project', {}) + return tuple(proj.get('entry-points', {}).get('nbdev', {})) + +# %% ../nbs/api/21_octavo.ipynb #f9f85ebf +def render_script(path): + "Captured stdout from a double-suffix render script; prepend optional `.md.py` frontmatter" + path = Path(path) + if path.suffix!='.py' or len(path.suffixes)<2: return + fm = _is_qpy(path) if path.name.endswith('.md.py') else None + f = StringIO() + with redirect_stdout(f): exec(compile(path.read_text(encoding='utf-8'), str(path), 'exec'), {}) + return (f'---\n{fm}\n---\n\n' if fm else '') + f.getvalue() + +# %% ../nbs/api/21_octavo.ipynb #e85c51f3 +def script_pages(root): + "Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s" + root,pages,files = Path(root),L(),L() + for o in sorted(root.rglob('*.py')): + rel = o.relative_to(root) + if hidden(rel) or (txt:=render_script(o)) is None: continue + dest = rel.with_suffix('') + if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest))) + else: files.append(GeneratedFile(str(rel), str(dest), txt)) + return pages,files + +# %% ../nbs/api/21_octavo.ipynb #d6904375 +def octavo_config(root): + "Read an existing octavo.yml mapping before running project code" + root = Path(root) + if not (root/'octavo.yml').is_file(): raise FileNotFoundError(root/'octavo.yml') + settings = yaml.safe_load((root/'octavo.yml').read_text()) + if settings is None: return {} + if not isinstance(settings, dict): raise ValueError(f'{root / "octavo.yml"} must contain a YAML mapping') + return settings + +# %% ../nbs/api/21_octavo.ipynb #f6830357 +def octavo_docs( + path:str=None, # Path to the project or its notebooks +): + "Create Octavo docs and run the optional docs_post_build callback" + cfg = get_config(path) + settings = octavo_config(cfg.nbs_path) + with working_directory(cfg.config_path): + pages,files = script_pages(cfg.nbs_path) + docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages + site = Site(docs, root=cfg.nbs_path, files=files, **settings) + if (miss := site.build()): print(miss) + if (post := cfg.get('docs_post_build')): + out = Path(site.settings['output']).expanduser() + if not out.is_absolute(): out = cfg.nbs_path/out + import_obj(post)(out.resolve(), cfg) + +# %% ../nbs/api/21_octavo.ipynb #0d38402b +def octavo_readme( + path:str=None, # Path to the project or its notebooks + chk_time:bool=False, # Only build if the source is newer + contributing:bool=False, # Render CONTRIBUTING instead of README +): + "Render a repository Markdown document from a notebook's stored outputs" + cfg = get_config(path) + nbname = cfg.get('contributing_nb', 'contributing.ipynb') if contributing else cfg.readme_nb + root = Path(path).absolute() if path else cfg.nbs_path + src = root/nbname + if not src.exists(): src = cfg.nbs_path/nbname + if not src.exists(): return + name = 'CONTRIBUTING' if contributing else 'README' + dest = cfg.config_path/f'{name}.md' + if chk_time and _doc_mtime_not_older(dest, src): return + octavo_config(cfg.nbs_path) + with working_directory(cfg.config_path): + proc = OctavoProc(strip_libs=local_libraries(cfg)) + doc = md2doc(ipynb2md(src, nbproc=proc)) + head = f'# {doc.meta["title"]}\n\n' if doc.meta.get('title') else '' + if desc:=doc.meta.get('description'): head += f'{desc}\n\n' + md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{name}_files', raw=('md', 'html')) diff --git a/nbdev/processors.py b/nbdev/processors.py index 222e8564f..6354e0024 100644 --- a/nbdev/processors.py +++ b/nbdev/processors.py @@ -121,9 +121,9 @@ def _default_exp(nb): return default_exp.group(1) if default_exp else None # %% ../nbs/api/10_processors.ipynb #03e8fece-bf92-4a1d-9f8b-ef209107ff95 -def add_links(cell): +def add_links(cell, nl=None): "Add links to markdown cells" - nl = NbdevLookup() + if nl is None: nl = NbdevLookup() if cell.cell_type == 'markdown': cell.source = nl.linkify(cell.source) for o in cell.get('outputs', []): if hasattr(o, 'data') and hasattr(o['data'], 'text/markdown'): diff --git a/nbdev/quarto.py b/nbdev/quarto.py index 9668e3d3f..a34f5a980 100644 --- a/nbdev/quarto.py +++ b/nbdev/quarto.py @@ -13,6 +13,7 @@ import subprocess,sys,shutil,ast,warnings,traceback from os import system from contextlib import contextmanager +from inspect import signature from .config import * from .doclinks import * @@ -75,7 +76,7 @@ def _sort(a): _def_file_re = r'\.(?:ipynb|qmd|html)$' -@delegates +@delegates(nbglob) def _nbglob_docs( path:str=None, # Path to notebooks file_glob:str=None, # Only include files matching glob @@ -272,12 +273,19 @@ def _save_cached_readme(cache, cfg): if _rdmi.exists(): _copytree(_rdmi, cfg.config_path/_rdmi.name) +# %% ../nbs/api/14_quarto.ipynb #803e45af +def _octavo(path=None): + if not (get_config(path).nbs_path/'octavo.yml').is_file(): return None + from nbdev import octavo + return octavo + # %% ../nbs/api/14_quarto.ipynb #45d6bb5d @call_parse def nbdev_readme( path:str=None, # Path to notebooks (or project root) chk_time:bool=False): # Only build if out of date "Create README.md from readme_nb (index.ipynb by default). Skips if the file doesn't exist." + if backend:=_octavo(path): return backend.octavo_readme(path, chk_time=chk_time) cfg = get_config(path) path = Path(path).absolute() if path else cfg.nbs_path if not (path/cfg.readme_nb).exists(): path = cfg.nbs_path # given a project root, fall back to its configured nbs @@ -313,6 +321,7 @@ def nbdev_contributing( chk_time:bool=False # Only build if out-of-date ): """Create CONTRIBUTING.md from contributing_nb (defaults to 'contributing.ipynb' if present). Skips if the file doesn't exist.""" + if backend:=_octavo(path): return backend.octavo_readme(path, chk_time=chk_time, contributing=True) cfg = get_config() path = Path(path) if path else cfg.nbs_path contrib_nb_name = cfg.get('contributing_nb', 'contributing.ipynb') @@ -339,7 +348,12 @@ def nbdev_docs( path:str=None, # Path to notebooks n_workers:int=defaults.cpus, # Preprocessing and Quarto workers (0 or 1: serial) **kwargs): - "Create Quarto docs" + "Build docs with the backend selected by the project's configuration" + if (backend:=_octavo(path)): + params = signature(_nbglob_docs).parameters + unsupported = [k for k,v in kwargs.items() if k not in params or v != params[k].default] + if unsupported: raise ValueError(f'Octavo does not support these Quarto file filters: {", ".join(unsupported)}') + return backend.octavo_docs(path) cache,cfg,_ = _pre_docs(path, n_workers=n_workers, **kwargs) render_quarto(cache, cache/cfg.doc_path.name, n_workers=n_workers) shutil.rmtree(cfg.doc_path, ignore_errors=True) @@ -354,7 +368,7 @@ def prepare(): nbdev_export.__wrapped__() nbdev.test.nbdev_test.__wrapped__() nbdev.clean.nbdev_clean.__wrapped__() - refresh_quarto_yml() + if not _octavo(): refresh_quarto_yml() nbdev_readme.__wrapped__(chk_time=True) nbdev_contributing.__wrapped__(chk_time=True) diff --git a/nbs/api/05_doclinks.ipynb b/nbs/api/05_doclinks.ipynb index fa950e2e2..82fb9088c 100644 --- a/nbs/api/05_doclinks.ipynb +++ b/nbs/api/05_doclinks.ipynb @@ -602,20 +602,20 @@ "outputs": [], "source": [ "#| export\n", - "def _qual_sym(s, settings):\n", + "def _qual_sym(s, settings, local=False):\n", " \"Get qualified nb, py, and github paths for a symbol s\"\n", " if not isinstance(s,tuple): return s\n", " nb,py = s\n", " nbbase = urljoin(settings[\"doc_host\"]+'/',settings[\"doc_baseurl\"])\n", - " nb = urljoin(nbbase+'/', nb)\n", + " nb = '/' + nb.lstrip('/') if local else urljoin(nbbase+'/', nb)\n", " gh = urljoin(settings[\"git_url\"]+'/', f'blob/{settings[\"branch\"]}/{py}')\n", " return nb,py,gh\n", "\n", - "def _qual_mod(mod_d, settings): return {sym:_qual_sym(s, settings) for sym,s in mod_d.items()}\n", - "def _qual_syms(entries):\n", + "def _qual_mod(mod_d, settings, local=False): return {sym:_qual_sym(s, settings, local) for sym,s in mod_d.items()}\n", + "def _qual_syms(entries, local=False):\n", " settings = entries['settings']\n", " if 'doc_host' not in settings: return entries\n", - " return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}" + " return {'syms': {mod:_qual_mod(d, settings, local) for mod,d in entries['syms'].items()}, 'settings':settings}" ] }, { @@ -660,7 +660,7 @@ "source": [ "#| export\n", "@lru_cache(None)\n", - "def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None):\n", + "def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None, local_lib=None):\n", " cfg = get_config(also_settings=True)\n", " if strip_libs is None:\n", " try: strip_libs = cfg.get('strip_libs') or cfg.lib_name\n", @@ -675,7 +675,7 @@ " \n", " for o in eps:\n", " if incl_libs is not None and o.dist.name not in incl_libs: continue\n", - " try: entries[o.name] = _qual_syms(o.load())\n", + " try: entries[o.name] = _qual_syms(o.load(), local=o.name in tuplify(local_lib))\n", " except Exception: pass\n", " py_syms = merge(*L(o['syms'].values() for o in entries.values()).concat())\n", " for m in strip_libs:\n", @@ -724,8 +724,8 @@ "#| export\n", "class NbdevLookup:\n", " \"Mapping from symbol names to docs and source URLs\"\n", - " def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None):\n", - " self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods)\n", + " def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None, local_lib=None):\n", + " self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods, local_lib)\n", " self.aliases = {n:o.__name__ for n,o in (ns or {}).items() if isinstance(o, ModuleType)}\n", " \n", " def __getitem__(self, s): \n", diff --git a/nbs/api/10_processors.ipynb b/nbs/api/10_processors.ipynb index d79d3dd38..4aefe0fd6 100644 --- a/nbs/api/10_processors.ipynb +++ b/nbs/api/10_processors.ipynb @@ -347,9 +347,9 @@ "outputs": [], "source": [ "#| export\n", - "def add_links(cell):\n", + "def add_links(cell, nl=None):\n", " \"Add links to markdown cells\"\n", - " nl = NbdevLookup()\n", + " if nl is None: nl = NbdevLookup()\n", " if cell.cell_type == 'markdown': cell.source = nl.linkify(cell.source)\n", " for o in cell.get('outputs', []):\n", " if hasattr(o, 'data') and hasattr(o['data'], 'text/markdown'):\n", diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb index 23111070c..96e71aa12 100644 --- a/nbs/api/14_quarto.ipynb +++ b/nbs/api/14_quarto.ipynb @@ -40,6 +40,7 @@ "import subprocess,sys,shutil,ast,warnings,traceback\n", "from os import system\n", "from contextlib import contextmanager\n", + "from inspect import signature\n", "\n", "from nbdev.config import *\n", "from nbdev.doclinks import *\n", @@ -162,7 +163,7 @@ "\n", "_def_file_re = r'\\.(?:ipynb|qmd|html)$'\n", "\n", - "@delegates\n", + "@delegates(nbglob)\n", "def _nbglob_docs(\n", " path:str=None, # Path to notebooks\n", " file_glob:str=None, # Only include files matching glob \n", @@ -521,6 +522,20 @@ " if _rdmi.exists(): _copytree(_rdmi, cfg.config_path/_rdmi.name)\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "803e45af", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _octavo(path=None):\n", + " if not (get_config(path).nbs_path/'octavo.yml').is_file(): return None\n", + " from nbdev import octavo\n", + " return octavo" + ] + }, { "cell_type": "code", "execution_count": null, @@ -534,6 +549,7 @@ " path:str=None, # Path to notebooks (or project root)\n", " chk_time:bool=False): # Only build if out of date\n", " \"Create README.md from readme_nb (index.ipynb by default). Skips if the file doesn't exist.\"\n", + " if backend:=_octavo(path): return backend.octavo_readme(path, chk_time=chk_time)\n", " cfg = get_config(path)\n", " path = Path(path).absolute() if path else cfg.nbs_path\n", " if not (path/cfg.readme_nb).exists(): path = cfg.nbs_path # given a project root, fall back to its configured nbs\n", @@ -556,7 +572,7 @@ "id": "a6f1a4ce", "metadata": {}, "source": [ - "`nbdev-readme` calls \"quarto render,\" which is explained in the Quarto guide [here](https://quarto.org/docs/projects/code-execution.html).\n", + "`nbdev-readme` selects Octavo when the configured notebooks directory contains `octavo.yml`; otherwise it calls `quarto render`. `nbdev-contributing` uses the same backend selection. Octavo uses stored outputs, extracts embedded images, and leaves authored relative destinations unchanged. See `octavo_readme`.\n", "\n", "Pandoc's gfm writer hard-wraps prose at 72 columns by default; `-M wrap:preserve` keeps each paragraph's line breaks exactly as authored in the notebook instead.\n", "\n", @@ -608,6 +624,7 @@ " chk_time:bool=False # Only build if out-of-date\n", "):\n", " \"\"\"Create CONTRIBUTING.md from contributing_nb (defaults to 'contributing.ipynb' if present). Skips if the file doesn't exist.\"\"\"\n", + " if backend:=_octavo(path): return backend.octavo_readme(path, chk_time=chk_time, contributing=True)\n", " cfg = get_config()\n", " path = Path(path) if path else cfg.nbs_path\n", " contrib_nb_name = cfg.get('contributing_nb', 'contributing.ipynb')\n", @@ -675,7 +692,12 @@ " path:str=None, # Path to notebooks\n", " n_workers:int=defaults.cpus, # Preprocessing and Quarto workers (0 or 1: serial)\n", " **kwargs):\n", - " \"Create Quarto docs\"\n", + " \"Build docs with the backend selected by the project's configuration\"\n", + " if (backend:=_octavo(path)):\n", + " params = signature(_nbglob_docs).parameters\n", + " unsupported = [k for k,v in kwargs.items() if k not in params or v != params[k].default]\n", + " if unsupported: raise ValueError(f'Octavo does not support these Quarto file filters: {\", \".join(unsupported)}')\n", + " return backend.octavo_docs(path)\n", " cache,cfg,_ = _pre_docs(path, n_workers=n_workers, **kwargs)\n", " render_quarto(cache, cache/cfg.doc_path.name, n_workers=n_workers)\n", " shutil.rmtree(cfg.doc_path, ignore_errors=True)\n", @@ -721,7 +743,7 @@ " nbdev_export.__wrapped__()\n", " nbdev.test.nbdev_test.__wrapped__()\n", " nbdev.clean.nbdev_clean.__wrapped__()\n", - " refresh_quarto_yml()\n", + " if not _octavo(): refresh_quarto_yml()\n", " nbdev_readme.__wrapped__(chk_time=True)\n", " nbdev_contributing.__wrapped__(chk_time=True)" ] diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb new file mode 100644 index 000000000..97806ffe5 --- /dev/null +++ b/nbs/api/21_octavo.ipynb @@ -0,0 +1,500 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c0313881", + "metadata": {}, + "source": [ + "# Octavo\n", + "\n", + "> Build nbdev documentation with Octavo\n", + "- order: 21" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a6d574e", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp octavo" + ] + }, + { + "cell_type": "markdown", + "id": "56ff9208", + "metadata": {}, + "source": [ + "Install `nbdev[octavo]` to enable this integration. `nbdev-docs` selects Octavo when `octavo.yml` exists in the configured notebooks directory; otherwise it uses Quarto. The Quarto path does not import Octavo. Octavo builds the whole site; non-default Quarto file filters are rejected, and `n_workers` only controls Quarto builds.\n", + "\n", + "Run `nbdev-export` before building so generated modules and symbol indexes are current. Migration is a source conversion, not a way to render unchanged notebooks with either backend. README and CONTRIBUTING use Octavo for opted-in projects. `nbdev-prepare` retains export, test, and clean, skips Quarto configuration generation for Octavo projects, and dispatches README and CONTRIBUTING to the selected backend. Preview and the migration command are not yet connected to this integration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd0fd32f", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "from contextlib import redirect_stdout\n", + "from io import StringIO\n", + "from fastcore.utils import *\n", + "import yaml\n", + "from nbdev.config import get_config, import_obj, _load_toml\n", + "from nbdev.doclinks import NbdevLookup\n", + "from nbdev.process import NBProcessor\n", + "from nbdev.processors import FilterDefaults, add_links\n", + "from nbdev.serve import _is_qpy\n", + "from octavo.site import Site\n", + "from octavo.pages import Page, GeneratedFile, scan, page_nm, hidden, ipynb2md\n", + "from octavo.core import md2doc\n", + "from mdhtml import md2gfm\n", + "from nbdev.quarto import _doc_mtime_not_older" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f93ea36d", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "import tempfile\n", + "from copy import deepcopy\n", + "from fastcore.test import *\n", + "from fastcore.nbio import new_nb, mk_cell, write_nb\n", + "from nbdev.quarto import nbdev_docs" + ] + }, + { + "cell_type": "markdown", + "id": "66b2ef09", + "metadata": {}, + "source": [ + "## Notebook processing\n", + "\n", + "`OctavoProc` retains custom `doc_procs` and `skip_procs` from nbdev. It preserves rendering directives and exposes generated YAML frontmatter as Markdown. Generated project-symbol links are local; authored production URLs are unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cc66460", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "class OctavoProc(FilterDefaults):\n", + " \"Process nbdev notebooks for Octavo without changing Quarto defaults\"\n", + " def __init__(self, local_lib=None, strip_libs=None): self.nl = NbdevLookup(strip_libs=strip_libs or local_lib, local_lib=local_lib)\n", + " def add_links(self, cell): return add_links(cell, nl=self.nl)\n", + " def procs(self): return L(self.add_links if o is add_links else o for o in super().procs())\n", + " def nb_proc(self, nb): return NBProcessor(nb=nb, procs=self.procs(), rm_directives=False)\n", + "\n", + " def __call__(self, nb):\n", + " super().__call__(nb)\n", + " if getattr(nb, 'frontmatter_', None) and nb.cells[0].cell_type=='raw': nb.cells[0].cell_type = 'markdown'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e180af", + "metadata": {}, + "outputs": [], + "source": [ + "source = new_nb([mk_cell('# Example\\n\\n> A notebook with rendering directives', 'markdown'),\n", + " mk_cell('Use `ModuleMaker`. See [production docs](https://nbdev.fast.ai/api/maker.html#modulemaker).', 'markdown'),\n", + " mk_cell('#| echo: false\\n1 + 1')])\n", + "source.path_ = Path('example.ipynb')\n", + "nb = deepcopy(source)\n", + "OctavoProc(local_lib='nbdev')(nb)\n", + "test_eq(nb.cells[0].cell_type, 'markdown')\n", + "text = '\\n'.join(c.source for c in nb.cells)\n", + "assert '[`ModuleMaker`](/api/maker.html#modulemaker)' in text\n", + "assert '[production docs](https://nbdev.fast.ai/api/maker.html#modulemaker)' in text\n", + "assert '#| echo: false' in text\n", + "quarto_nb = deepcopy(source)\n", + "FilterDefaults()(quarto_nb)\n", + "test_eq(quarto_nb.cells[0].cell_type, 'raw')\n", + "assert '#| echo: false' in '\\n'.join(c.source for c in quarto_nb.cells)\n", + "nb.cells[0].source" + ] + }, + { + "cell_type": "markdown", + "id": "ce1056e3", + "metadata": {}, + "source": [ + "A distribution name need not match its import package or nbdev entry-point name. Site builds use the project's declared nbdev entry points to select local symbol indexes; README generation keeps published symbol URLs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a0258ee", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def local_libraries(cfg):\n", + " \"Names of the project's declared nbdev symbol indexes\"\n", + " proj = _load_toml(cfg.config_file).get('project', {})\n", + " return tuple(proj.get('entry-points', {}).get('nbdev', {}))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86824851", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " project = Path(d)\n", + " (project/'pyproject.toml').write_text('[project]\\nname = \"different-distribution\"\\n[project.entry-points.nbdev]\\nnbdev = \"nbdev._modidx:d\"\\n[tool.nbdev]\\n')\n", + " libs = local_libraries(get_config(project))\n", + " test_eq(libs, ('nbdev',))\n", + " lookup = NbdevLookup(local_lib=libs)\n", + " assert lookup.doc('nbdev.maker.ModuleMaker').startswith('/api/maker.html#')\n", + " assert NbdevLookup().doc('nbdev.maker.ModuleMaker').startswith('https://')\n", + " nb = deepcopy(source)\n", + " with working_directory(project): OctavoProc(local_lib=libs)(nb)\n", + " assert '[`ModuleMaker`](/api/maker.html#modulemaker)' in '\\n'.join(c.source for c in nb.cells)" + ] + }, + { + "cell_type": "markdown", + "id": "5ad885db", + "metadata": {}, + "source": [ + "## Render scripts\n", + "\n", + "A *render script* is a Python file with a double suffix, such as `page.md.py` or `drawing.svg.py`. nbdev executes it and saves stdout to the rendered Octavo site without the final `.py` suffix. A `.md.py` file may declare frontmatter in its module docstring; every other generated file contains exactly what its script prints.\n", + "\n", + "Octavo never executes anything: nbdev's `script_pages` hands it finished `Page`s and `GeneratedFile`s. A script that fails to run fails the build, since a half-built site is worse than a clear error. Renderscript sources such as `fname.ext.py` are not sent to Octavo, while ordinary single-suffix `.py`, like any non-page sourcefile, files are published as assets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9f85ebf", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def render_script(path):\n", + " \"Captured stdout from a double-suffix render script; prepend optional `.md.py` frontmatter\"\n", + " path = Path(path)\n", + " if path.suffix!='.py' or len(path.suffixes)<2: return\n", + " fm = _is_qpy(path) if path.name.endswith('.md.py') else None\n", + " f = StringIO()\n", + " with redirect_stdout(f): exec(compile(path.read_text(encoding='utf-8'), str(path), 'exec'), {})\n", + " return (f'---\\n{fm}\\n---\\n\\n' if fm else '') + f.getvalue()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e85c51f3", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def script_pages(root):\n", + " \"Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s\"\n", + " root,pages,files = Path(root),L(),L()\n", + " for o in sorted(root.rglob('*.py')):\n", + " rel = o.relative_to(root)\n", + " if hidden(rel) or (txt:=render_script(o)) is None: continue\n", + " dest = rel.with_suffix('')\n", + " if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest)))\n", + " else: files.append(GeneratedFile(str(rel), str(dest), txt))\n", + " return pages,files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1864ec1d", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " d = Path(d)\n", + " (d/'01_intro.md.py').write_text('\"\"\"---\\ntitle: Hi\\n---\"\"\"\\nprint(\"# Generated\")\\nprint(\"body\")')\n", + " (d/'images').mkdir(); (d/'images/dot.svg.py').write_text('print(\"\")')\n", + " (d/'plain.md.py').write_text('print(\"# No frontmatter\")')\n", + " (d/'ordinary.py').write_text('x = 1')\n", + " (d/'_skip').mkdir(); (d/'_skip/no.md.py').write_text('\"\"\"---\\n---\"\"\"\\nprint(\"no\")')\n", + " pages,files = script_pages(d)\n", + " test_eq([p.dest for p in pages], ['intro.html','plain.html'])\n", + " test_eq(pages[0].text, '---\\ntitle: Hi\\n---\\n\\n# Generated\\nbody\\n')\n", + " test_eq(pages[0].meta['title'], 'Hi')\n", + " test_eq(pages[1].text, '# No frontmatter\\n')\n", + " test_eq([(f.src,f.dest,f.content) for f in files], [('images/dot.svg.py','images/dot.svg','\\n')])\n", + " test_is(render_script(d/'ordinary.py'), None)\n", + " (d/'bad.md.py').write_text('1/0')\n", + " test_fail(lambda: script_pages(d), contains='division')" + ] + }, + { + "cell_type": "markdown", + "id": "a38f3e46", + "metadata": {}, + "source": [ + "## Post-build callback" + ] + }, + { + "cell_type": "markdown", + "id": "24badb1f", + "metadata": {}, + "source": [ + "Set `docs_post_build` to an importable `module:function` in `pyproject.toml`:\n", + "\n", + "```toml\n", + "[tool.nbdev]\n", + "docs_post_build = \"my_project.docbuild:build_extras\"\n", + "```\n", + "\n", + "After a successful site build, nbdev calls the function once with `(out, cfg)`. `out` is the absolute directory containing the completed site. `cfg` is the nbdev project configuration. The callback writes additional files into `out`; its return value is ignored. Callback errors fail the build. Without this setting, no callback runs.\n", + "\n", + "Author the callback in a notebook and export it before building the docs, as with custom processors. Its dependencies belong to the project's documentation setup." + ] + }, + { + "cell_type": "markdown", + "id": "6aea6659", + "metadata": {}, + "source": [ + "## Build configuration\n", + "\n", + "`octavo.yml` must exist and contain a YAML mapping. An empty file uses defaults. Validate it before executing render scripts or processing notebooks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6904375", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def octavo_config(root):\n", + " \"Read an existing octavo.yml mapping before running project code\"\n", + " root = Path(root)\n", + " if not (root/'octavo.yml').is_file(): raise FileNotFoundError(root/'octavo.yml')\n", + " settings = yaml.safe_load((root/'octavo.yml').read_text())\n", + " if settings is None: return {}\n", + " if not isinstance(settings, dict): raise ValueError(f'{root / \"octavo.yml\"} must contain a YAML mapping')\n", + " return settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87c206ca", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " root = Path(d)\n", + " test_fail(lambda: octavo_config(root), contains='octavo.yml')\n", + " for text in ('- not a mapping', '[]', 'false', '0'):\n", + " (root/'octavo.yml').write_text(text)\n", + " test_fail(lambda: octavo_config(root), contains='mapping')\n", + " (root/'octavo.yml').write_text('')\n", + " test_eq(octavo_config(root), {})" + ] + }, + { + "cell_type": "markdown", + "id": "5407994a", + "metadata": {}, + "source": [ + "The site build gives nbdev’s symbol linker the current project identity. Generated project references become site-root-relative document addresses; dependency references and explicitly authored absolute links remain absolute. Octavo resolves local document addresses for each output format." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6830357", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def octavo_docs(\n", + " path:str=None, # Path to the project or its notebooks\n", + "):\n", + " \"Create Octavo docs and run the optional docs_post_build callback\"\n", + " cfg = get_config(path)\n", + " settings = octavo_config(cfg.nbs_path)\n", + " with working_directory(cfg.config_path):\n", + " pages,files = script_pages(cfg.nbs_path)\n", + " docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages\n", + " site = Site(docs, root=cfg.nbs_path, files=files, **settings)\n", + " if (miss := site.build()): print(miss)\n", + " if (post := cfg.get('docs_post_build')):\n", + " out = Path(site.settings['output']).expanduser()\n", + " if not out.is_absolute(): out = cfg.nbs_path/out\n", + " import_obj(post)(out.resolve(), cfg)" + ] + }, + { + "cell_type": "markdown", + "id": "0d5735ac", + "metadata": {}, + "source": [ + "## Build an opted-in project\n", + "\n", + "`nbdev-docs` reads `octavo.yml` and processes notebooks and render scripts together. It does not migrate sources or change `_quarto.yml`. This temporary project builds linked pages and generated assets, then demonstrates that invalid configuration is rejected before executing a render script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c8fa2af", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " project = Path(d)\n", + " nbs = project/'nbs'\n", + " nbs.mkdir()\n", + " (project/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\nskip_procs = \"insert_warning\"\\n')\n", + " (nbs/'octavo.yml').write_text('title: Demo\\noutput: ../_octavo\\nsite_url: https://example.com\\n')\n", + " (nbs/'_quarto.yml').write_text('project:\\n type: website\\n')\n", + " (nbs/'index.md').write_text('# Home\\n\\nRead the [guide](guide.ipynb).')\n", + " guide = new_nb([mk_cell('# Guide\\n\\n> A notebook page', 'markdown'), mk_cell('Notebook content.', 'markdown')])\n", + " write_nb(guide, nbs/'guide.ipynb')\n", + " (nbs/'generated.md.py').write_text('print(\"# Generated page\")')\n", + " (nbs/'image.svg.py').write_text('print(\"\")')\n", + " cwd = Path.cwd()\n", + " with working_directory(project): run(r'nbdev-docs --file-re \"\\.(?:ipynb|qmd|html)$\"')\n", + " nbdev_docs.__wrapped__(project, file_glob=None)\n", + " test_eq(Path.cwd(), cwd)\n", + " out = project/'_octavo'\n", + " assert 'guide.html' in (out/'index.html').read_text()\n", + " assert 'Notebook content.' in (out/'guide.html').read_text()\n", + " assert 'AUTOGENERATED' not in (out/'guide.md').read_text()\n", + " assert (out/'generated.html').exists()\n", + " test_eq((out/'image.svg').read_text(), '\\n')\n", + " test_eq((nbs/'_quarto.yml').read_text(), 'project:\\n type: website\\n')\n", + " test_fail(lambda: nbdev_docs.__wrapped__(project, file_glob='*.ipynb'), contains='file_glob')\n", + " (nbs/'octavo.yml').write_text('- invalid')\n", + " (nbs/'generated.md.py').write_text('raise RuntimeError(\"script ran before validation\")')\n", + " test_fail(lambda: nbdev_docs.__wrapped__(project), contains='mapping')\n", + " test_eq(Path.cwd(), cwd)" + ] + }, + { + "cell_type": "markdown", + "id": "b53bd453", + "metadata": {}, + "source": [ + "## Repository Markdown" + ] + }, + { + "cell_type": "markdown", + "id": "09b41c23", + "metadata": {}, + "source": [ + "`nbdev-readme` and `nbdev-contributing` render the configured notebooks to the project root without Quarto. They use stored outputs and do not execute ordinary code cells. nbdev processors still run, including custom processors and generated API documentation. Titles and descriptions become Markdown. Symbol links use published documentation URLs.\n", + "\n", + "Embedded images are extracted into `README_files` or `CONTRIBUTING_files`. Authored relative destinations are unchanged, including reference-style links. Authors must choose paths suitable for the repository-root document. A missing source notebook is skipped; `chk_time=True` skips a document newer than its source." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d38402b", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def octavo_readme(\n", + " path:str=None, # Path to the project or its notebooks\n", + " chk_time:bool=False, # Only build if the source is newer\n", + " contributing:bool=False, # Render CONTRIBUTING instead of README\n", + "):\n", + " \"Render a repository Markdown document from a notebook's stored outputs\"\n", + " cfg = get_config(path)\n", + " nbname = cfg.get('contributing_nb', 'contributing.ipynb') if contributing else cfg.readme_nb\n", + " root = Path(path).absolute() if path else cfg.nbs_path\n", + " src = root/nbname\n", + " if not src.exists(): src = cfg.nbs_path/nbname\n", + " if not src.exists(): return\n", + " name = 'CONTRIBUTING' if contributing else 'README'\n", + " dest = cfg.config_path/f'{name}.md'\n", + " if chk_time and _doc_mtime_not_older(dest, src): return\n", + " octavo_config(cfg.nbs_path)\n", + " with working_directory(cfg.config_path):\n", + " proc = OctavoProc(strip_libs=local_libraries(cfg))\n", + " doc = md2doc(ipynb2md(src, nbproc=proc))\n", + " head = f'# {doc.meta[\"title\"]}\\n\\n' if doc.meta.get('title') else ''\n", + " if desc:=doc.meta.get('description'): head += f'{desc}\\n\\n'\n", + " md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{name}_files', raw=('md', 'html'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5e7d85b", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " project = Path(d)\n", + " nbs = project/'nbs'\n", + " nbs.mkdir()\n", + " (project/'pyproject.toml').write_text('[project]\\nname = \"different-distribution\"\\n[project.entry-points.nbdev]\\nnbdev = \"nbdev._modidx:d\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\ncontributing_nb = \"help.ipynb\"\\n')\n", + " (nbs/'octavo.yml').write_text('{}')\n", + " png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aX1sAAAAASUVORK5CYII='\n", + " image = mk_cell('#| echo: false\\nraise RuntimeError(\"must not execute\")')\n", + " image.outputs = [dict(output_type='display_data', metadata={}, data={'image/png': png})]\n", + " book = new_nb([mk_cell('# Example\\n\\n> Repository document', 'markdown'),\n", + " mk_cell('Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md', 'markdown'),\n", + " image, mk_cell('#| include: false\\n\"excluded cell\"')])\n", + " write_nb(book, nbs/'index.ipynb')\n", + " write_nb(book, nbs/'help.ipynb')\n", + " cwd = Path.cwd()\n", + " octavo_readme(project)\n", + " with working_directory(project):\n", + " run('nbdev-readme')\n", + " run('nbdev-contributing')\n", + " test_eq(Path.cwd(), cwd)\n", + " for name in ('README', 'CONTRIBUTING'):\n", + " text = (project/f'{name}.md').read_text()\n", + " assert '# Example' in text and 'Repository document' in text\n", + " assert 'https://nbdev.fast.ai/api/maker.html#modulemaker' in text\n", + " assert '(guide.md)' in text and '[ref]: guide.md' in text\n", + " assert 'must not execute' not in text and 'excluded cell' not in text\n", + " assert 'data:image' not in text and f'{name}_files/' in text\n", + " assert list((project/f'{name}_files').glob('*.png'))\n", + " before = (project/'README.md').stat().st_mtime_ns\n", + " octavo_readme(project, chk_time=True)\n", + " test_eq((project/'README.md').stat().st_mtime_ns, before)\n", + " assert not (nbs/'_quarto.yml').exists()\n", + " (nbs/'help.ipynb').unlink()\n", + " octavo_readme(project, contributing=True)" + ] + } + ], + "metadata": { + "solveit": { + "default_code": true, + "mode": "learning", + "use_thinking": false, + "use_tools": true, + "ver": 2 + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 3ffacb753..de8d02872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,8 @@ dependencies = [ "fastcore>=2.2.28", "execnb>=0.3.3", "astunparse", "ghapi>=2.0. "setuptools", "build", "fastgit>=0.0.7", "pyyaml", "tomli; python_version < '3.11'", ] [project.optional-dependencies] -dev = [ "ipywidgets", "nbdev-numpy", "nbdev-stdlib", "pandas", "matplotlib", "black", "svg.py", "pysym2md>=0.0.6", "llms-txt", "sphinx", "plum-dispatch", "nbclient", "ipykernel", ] +octavo = ["octavo>=0.0.1"] +dev = [ "ipywidgets", "nbdev-numpy", "nbdev-stdlib", "pandas", "matplotlib", "black", "svg.py", "pysym2md>=0.0.6", "llms-txt", "sphinx", "plum-dispatch", "nbclient", "ipykernel", "octavo>=0.0.1", ] [project.urls] Repository = "https://github.com/AnswerDotAI/nbdev" From 589035786b41a18e34cd126587656da29f53d271 Mon Sep 17 00:00:00 2001 From: Rens Date: Tue, 6 Oct 2026 16:08:22 +0200 Subject: [PATCH 02/16] add quarto-to-octavo migration cmd --- nbdev/_modidx.py | 21 + nbdev/octavo.py | 230 ++++++- nbs/api/21_octavo.ipynb | 1446 ++++++++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 4 files changed, 1601 insertions(+), 97 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index 96592d46a..a7dc6ae20 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -219,11 +219,32 @@ 'nbdev.octavo.OctavoProc.add_links': ('api/octavo.html#octavoproc.add_links', 'nbdev/octavo.py'), 'nbdev.octavo.OctavoProc.nb_proc': ('api/octavo.html#octavoproc.nb_proc', 'nbdev/octavo.py'), 'nbdev.octavo.OctavoProc.procs': ('api/octavo.html#octavoproc.procs', 'nbdev/octavo.py'), + 'nbdev.octavo._apply_procs': ('api/octavo.html#_apply_procs', 'nbdev/octavo.py'), + 'nbdev.octavo._create_octavo_cfg': ('api/octavo.html#_create_octavo_cfg', 'nbdev/octavo.py'), + 'nbdev.octavo._dump_fm': ('api/octavo.html#_dump_fm', 'nbdev/octavo.py'), + 'nbdev.octavo._migrate_octavo': ('api/octavo.html#_migrate_octavo', 'nbdev/octavo.py'), + 'nbdev.octavo._nav_href': ('api/octavo.html#_nav_href', 'nbdev/octavo.py'), + 'nbdev.octavo._nav_item': ('api/octavo.html#_nav_item', 'nbdev/octavo.py'), + 'nbdev.octavo._octavo_nav': ('api/octavo.html#_octavo_nav', 'nbdev/octavo.py'), + 'nbdev.octavo._octavo_output': ('api/octavo.html#_octavo_output', 'nbdev/octavo.py'), + 'nbdev.octavo._qmd_moves': ('api/octavo.html#_qmd_moves', 'nbdev/octavo.py'), + 'nbdev.octavo._qmd_repls': ('api/octavo.html#_qmd_repls', 'nbdev/octavo.py'), + 'nbdev.octavo._quarto_config': ('api/octavo.html#_quarto_config', 'nbdev/octavo.py'), + 'nbdev.octavo._repl_cv_match': ('api/octavo.html#_repl_cv_match', 'nbdev/octavo.py'), + 'nbdev.octavo._tool_item': ('api/octavo.html#_tool_item', 'nbdev/octavo.py'), 'nbdev.octavo.local_libraries': ('api/octavo.html#local_libraries', 'nbdev/octavo.py'), + 'nbdev.octavo.migrate_md_file': ('api/octavo.html#migrate_md_file', 'nbdev/octavo.py'), + 'nbdev.octavo.migrate_nb_cells': ('api/octavo.html#migrate_nb_cells', 'nbdev/octavo.py'), + 'nbdev.octavo.nbdev_migrate_octavo': ('api/octavo.html#nbdev_migrate_octavo', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_config': ('api/octavo.html#octavo_config', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_docs': ('api/octavo.html#octavo_docs', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_readme': ('api/octavo.html#octavo_readme', 'nbdev/octavo.py'), 'nbdev.octavo.render_script': ('api/octavo.html#render_script', 'nbdev/octavo.py'), + 'nbdev.octavo.repl_content_visible': ('api/octavo.html#repl_content_visible', 'nbdev/octavo.py'), + 'nbdev.octavo.repl_frontmatter_keys': ('api/octavo.html#repl_frontmatter_keys', 'nbdev/octavo.py'), + 'nbdev.octavo.repl_qmd_links': ('api/octavo.html#repl_qmd_links', 'nbdev/octavo.py'), + 'nbdev.octavo.repl_script_frontmatter_keys': ( 'api/octavo.html#repl_script_frontmatter_keys', + 'nbdev/octavo.py'), 'nbdev.octavo.script_pages': ('api/octavo.html#script_pages', 'nbdev/octavo.py')}, 'nbdev.process': { 'nbdev.process.NBProcessor': ('api/process.html#nbprocessor', 'nbdev/process.py'), 'nbdev.process.NBProcessor.__init__': ('api/process.html#nbprocessor.__init__', 'nbdev/process.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 3248e3549..f824faaee 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -5,27 +5,29 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. # %% auto #0 -__all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_readme'] +__all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_readme', + 'repl_qmd_links', 'repl_content_visible', 'repl_frontmatter_keys', 'repl_script_frontmatter_keys', + 'migrate_nb_cells', 'migrate_md_file', 'nbdev_migrate_octavo'] -# %% ../nbs/api/21_octavo.ipynb #fd0fd32f +# %% ../nbs/api/21_octavo.ipynb #7b098bb5 from contextlib import redirect_stdout from io import StringIO from fastcore.utils import * +from fastcore.script import call_parse import yaml -from .config import get_config, import_obj, _load_toml +from octavo import * +from mdhtml import md2gfm +from .config import get_config, import_obj, _load_toml, read_nb from .doclinks import NbdevLookup from .process import NBProcessor from .processors import FilterDefaults, add_links from .serve import _is_qpy -from octavo.site import Site -from octavo.pages import Page, GeneratedFile, scan, page_nm, hidden, ipynb2md -from octavo.core import md2doc -from mdhtml import md2gfm +from .sync import write_nb from .quarto import _doc_mtime_not_older # %% ../nbs/api/21_octavo.ipynb #1cc66460 class OctavoProc(FilterDefaults): - "Process nbdev notebooks for Octavo without changing Quarto defaults" + "Prepare notebook content, metadata, and symbol links for Octavo" def __init__(self, local_lib=None, strip_libs=None): self.nl = NbdevLookup(strip_libs=strip_libs or local_lib, local_lib=local_lib) def add_links(self, cell): return add_links(cell, nl=self.nl) def procs(self): return L(self.add_links if o is add_links else o for o in super().procs()) @@ -43,7 +45,7 @@ def local_libraries(cfg): # %% ../nbs/api/21_octavo.ipynb #f9f85ebf def render_script(path): - "Captured stdout from a double-suffix render script; prepend optional `.md.py` frontmatter" + "Execute a double-suffix render script and return its printed output with optional page metadata" path = Path(path) if path.suffix!='.py' or len(path.suffixes)<2: return fm = _is_qpy(path) if path.name.endswith('.md.py') else None @@ -65,7 +67,7 @@ def script_pages(root): # %% ../nbs/api/21_octavo.ipynb #d6904375 def octavo_config(root): - "Read an existing octavo.yml mapping before running project code" + "Read `octavo.yml` as a settings dictionary; return an empty dictionary for an empty file" root = Path(root) if not (root/'octavo.yml').is_file(): raise FileNotFoundError(root/'octavo.yml') settings = yaml.safe_load((root/'octavo.yml').read_text()) @@ -79,7 +81,7 @@ def octavo_docs( ): "Create Octavo docs and run the optional docs_post_build callback" cfg = get_config(path) - settings = octavo_config(cfg.nbs_path) + settings = {'output': cfg.doc_path, **octavo_config(cfg.nbs_path)} with working_directory(cfg.config_path): pages,files = script_pages(cfg.nbs_path) docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages @@ -112,4 +114,208 @@ def octavo_readme( doc = md2doc(ipynb2md(src, nbproc=proc)) head = f'# {doc.meta["title"]}\n\n' if doc.meta.get('title') else '' if desc:=doc.meta.get('description'): head += f'{desc}\n\n' - md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{name}_files', raw=('md', 'html')) + md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{src.stem}_files', raw=('md', 'html')) + +# %% ../nbs/api/21_octavo.ipynb #267d0150 +def _qmd_moves(path): + "`.qmd` documents and `.qmd.py` render scripts under `path`, mapped to Octavo names" + path = Path(path) + ps = L(path.rglob('*.qmd')) + L(path.rglob('*.qmd.py')) + return {p:p.with_name(re.sub(r'\.qmd(?=\.py$|$)', '.md', p.name)) for p in sorted(ps) + if not any(o[0] in '._' for o in p.relative_to(path).parts)} + +# %% ../nbs/api/21_octavo.ipynb #a6688f7e +def _qmd_repls(src, path, moves): + "Old and new spellings of each moved document, as seen from `src`" + def _rel(p, start): return os.path.relpath(p, start).replace(os.sep, '/') + res = {} + for old,new in moves.items(): + res[_rel(old, path)] = _rel(new, path) + res[_rel(old, src.parent)] = _rel(new, src.parent) + return sorted(res.items(), key=lambda o: len(o[0]), reverse=True) + +def repl_qmd_links(txt, moves, src, root): + "Rewrite relative and root-relative `.qmd` link targets to `.md`" + for old,new in _qmd_repls(src, root, moves): txt = txt.replace(old, new) + return txt + +# %% ../nbs/api/21_octavo.ipynb #237aea2c +_re_cv = re.compile(r'^:{3,}[^\S\r\n]*\{([^}]*\.content-visible[^}]*)\}[^\S\r\n]*\n(.*?)\n:{3,}[^\S\r\n]*$', re.MULTILINE | re.DOTALL) + +def _repl_cv_match(m): + attrs,body = m.group(1),m.group(2).strip() + m_fmt = re.search(r'when-format=["\']?(\w+)["\']?', attrs) + fmt = m_fmt.group(1).lower() if m_fmt else 'html' + target = 'md' if fmt in ('markdown', 'commonmark', 'md') else fmt + if target=='html': + if (m_style := re.search(r'style=["\']([^"\']+)["\']', attrs)): + body = f'
\n{body}\n
' + return f'```{{={target}}}\n{body}\n```' + +def repl_content_visible(s): + "Convert Quarto `::: {.content-visible when-format=...}` to ````{=md}```` / ````{=html}````" + return _re_cv.sub(_repl_cv_match, s) + +# %% ../nbs/api/21_octavo.ipynb #04d5a738 +def _dump_fm(fm): return yaml.dump(fm, sort_keys=False, allow_unicode=True, width=10**9) + +def repl_frontmatter_keys(s): + "Normalize legacy and Quarto frontmatter keys for Octavo" + fm,body = frontmatter(s) + if not fm: return s + changed = False + aliases = [('categories','tags'), ('subtitle','description'), ('summary','description')] + for old,new in aliases + [(k,k.replace('-','_')) for k in fm if '-' in k and k!='page-layout']: + if old not in fm: continue + fm.setdefault(new, fm.pop(old)) + changed = True + if fm.pop('toc', None) is False: + fm['toc_depth'] = 0 + changed = True + if changed: return f'---\n{_dump_fm(fm)}---\n\n{body.lstrip(chr(10))}' + return s + +# %% ../nbs/api/21_octavo.ipynb #0ceec29d +_script_fm = re.compile(r'\A(?P"{3}|\'{3})---\n(?P.*?)\n---(?P=q)', re.DOTALL) + +def repl_script_frontmatter_keys(s): + "Normalize a render script's leading YAML docstring for Octavo" + if not (m := _script_fm.match(s)): return s + old = f"---\n{m.group('fm')}\n---\n" + new = repl_frontmatter_keys(old) + if new == old: return s + fm,_ = frontmatter(new) + return f"{m.group('q')}---\n{_dump_fm(fm)}---{m.group('q')}" + s[m.end():] + +# %% ../nbs/api/21_octavo.ipynb #0c93c529 +def _apply_procs(txt, procs): + for p in procs: txt = p(txt) + return txt + +def migrate_nb_cells(path, procs): + "Apply `procs` to all markdown cells in a notebook in-place" + nb,changed = read_nb(path),False + for c in nb.cells: + if c.cell_type != 'markdown' or not c.get('source'): continue + new_src = _apply_procs(c.source, procs) + if new_src != c.source: c.source,changed = new_src,True + if changed: write_nb(nb, path) + +def migrate_md_file(path, procs): + "Apply `procs` to a Markdown/QMD file in-place" + txt = path.read_text() + new_txt = _apply_procs(txt, procs) + if new_txt!=txt: path.write_text(new_txt) + +# %% ../nbs/api/21_octavo.ipynb #01ed763e +def _nav_href(h): + if not h or '://' in h or h.startswith(('#','mailto:')): return h + return re.sub(r'\.(?:ipynb|qmd|md)(?=[?#]|$)', '.html', h) + +def _nav_item(item): + res = {'text': item['text']} + if 'menu' in item: res['children'] = [{'text': m['text'], 'href': _nav_href(m['href'])} for m in item['menu'] if 'href' in m] + elif 'href' in item: res['href'] = _nav_href(item['href']) + return res + +# %% ../nbs/api/21_octavo.ipynb #acd395a1 +_ICON_MAP = {'twitter':'x', 'twitter-x':'x'} +_ICON_LABELS = {'github':'GitHub', 'rss':'RSS'} + +def _tool_item(item): + "An octavo `tools` entry from a Quarto icon link, or None where octavo has no such icon" + from octavo.theme import ICONS + ico = _ICON_MAP.get(item['icon'], item['icon']) + if ico not in ICONS: return None + return {'icon': ico, 'href': _nav_href(item['href']), 'label': item.get('aria-label') or item.get('text') or _ICON_LABELS.get(ico, ico.title())} + +# %% ../nbs/api/21_octavo.ipynb #deb7ee06 +def _quarto_config(nbs_path): + qy_p,ny_p = nbs_path/'_quarto.yml',nbs_path/'nbdev.yml' + qy = yaml.safe_load(qy_p.read_text()) if qy_p.exists() else {} + ny = yaml.safe_load(ny_p.read_text()) if ny_p.exists() else {} + for fname,settings in ((qy_p,qy), (ny_p,ny)): + if settings is not None and not isinstance(settings, dict): raise ValueError(f'{fname} must contain a YAML mapping') + return qy or {},ny or {} + +# %% ../nbs/api/21_octavo.ipynb #127739bd +def _octavo_output(nbs_path, qy, ny): + "Return a Quarto output override relative to notebooks, or None when doc_path suffices" + out_dir = (ny.get('project', {}) or {}).get('output-dir') or (qy.get('project', {}) or {}).get('output-dir') + if not out_dir: return + cfg = get_config(nbs_path) + out_dir = (cfg.config_path/Path(out_dir).expanduser()).resolve() + if out_dir == cfg.doc_path.resolve(): return + return os.path.relpath(out_dir, Path(nbs_path).resolve()).replace(os.sep, '/') + +# %% ../nbs/api/21_octavo.ipynb #201d0976 +def _octavo_nav(nbs_path, qy, ny): + "Convert Quarto navbar settings into Octavo navigation links and icon tools" + cfg = {} + def _nav(k): return nested_idx(qy,'website','navbar',k) or nested_idx(ny,'website','navbar',k) or [] + if nav_left := _nav('left'): cfg['nav_links'] = [_nav_item(o) for o in nav_left if 'text' in o and ('href' in o or 'menu' in o)] + links = L(_nav('right')+_nav('tools')).filter(lambda o: 'icon' in o and 'href' in o) + tools = links.map(_tool_item) + for o,t in zip(links,tools): + if t is None: print(f"{nbs_path/'octavo.yml'}: dropped tool icon {o['icon']!r} ({o['href']}): not bundled by octavo. Add it back with `icon: ` to your own SVG.") + if (tools := tools.filter()): cfg['tools'] = list(tools) + return cfg + +# %% ../nbs/api/21_octavo.ipynb #2f2d4b3b +def _create_octavo_cfg(nbs_path): + "Return Octavo settings derived from `_quarto.yml` and `nbdev.yml`" + nbs_path = Path(nbs_path).resolve() + qy,ny = _quarto_config(nbs_path) + ws_q,ws_n = qy.get('website', {}) or {},ny.get('website', {}) or {} + fmt_html = (qy.get('format', {}) or {}).get('html', {}) or {} + cfg = {} + if (output := _octavo_output(nbs_path, qy, ny)) is not None: cfg['output'] = output + title = ws_n.get('title') or ws_q.get('title') + if title: cfg['title'] = title + site_url = ws_n.get('site-url') or ws_q.get('site-url') + if site_url: cfg['site_url'] = site_url.rstrip('/') + if 'toc-depth' in fmt_html: + try: cfg['toc_depth'] = max(1, int(fmt_html['toc-depth']) - 1) + except (ValueError, TypeError): pass + else: cfg['toc_depth'] = 0 if fmt_html.get('toc') is False else 3 + favicon = ws_n.get('favicon') or ws_q.get('favicon') + if favicon: cfg['favicon'] = favicon + cfg.update(_octavo_nav(nbs_path, qy, ny)) + return cfg + +# %% ../nbs/api/21_octavo.ipynb #c4b4478c +def _migrate_octavo(path): + "Migrate documentation sources in place, create missing Octavo configuration, and return the rename mapping" + path = Path(path).expanduser() + if not path.is_dir(): raise NotADirectoryError(path) + moves = _qmd_moves(path) + for new in moves.values(): + if new.exists(): raise FileExistsError(f'Cannot rename to existing file: {new}') + qy = path/'_quarto.yml' + if (path/'octavo.yml').exists(): octavo_config(path) + elif not qy.is_file(): raise FileNotFoundError(qy) + cfg = _create_octavo_cfg(path) if qy.exists() and not (path/'octavo.yml').exists() else None + + srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md') + and not any(o[0] in '._' for o in p.relative_to(path).parts)] + scripts = [p for p in sorted(path.rglob('*.qmd.py')) + if not any(o[0] in '._' for o in p.relative_to(path).parts)] + for src in srcs: + procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_content_visible, repl_frontmatter_keys] + if src.suffix=='.ipynb': migrate_nb_cells(src, procs) + else: migrate_md_file(src, procs) + for src in scripts: migrate_md_file(src, [repl_script_frontmatter_keys]) + + for old,new in moves.items(): old.rename(new) + + if cfg: (path/'octavo.yml').write_text(yaml.dump(cfg, sort_keys=False)) + return moves + +# %% ../nbs/api/21_octavo.ipynb #e332ece4 +@call_parse +def nbdev_migrate_octavo( + path:str=None, # Documentation directory (defaults to configured nbs_path) +): + "Migrate Quarto-era nbdev project to Octavo" + moves = _migrate_octavo(get_config().nbs_path if path is None else path) + for old,new in moves.items(): print(f'{old} -> {new}') diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index 97806ffe5..bd1e18e17 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -26,15 +26,57 @@ "id": "56ff9208", "metadata": {}, "source": [ - "Install `nbdev[octavo]` to enable this integration. `nbdev-docs` selects Octavo when `octavo.yml` exists in the configured notebooks directory; otherwise it uses Quarto. The Quarto path does not import Octavo. Octavo builds the whole site; non-default Quarto file filters are rejected, and `n_workers` only controls Quarto builds.\n", + "Octavo is an alternative to Quarto for building a documentation website from nbdev notebooks. Install `nbdev[octavo]` to use it. A project selects Octavo by placing `octavo.yml` in its configured notebooks directory. Projects without that file continue to use Quarto and do not import Octavo.\n", "\n", - "Run `nbdev-export` before building so generated modules and symbol indexes are current. Migration is a source conversion, not a way to render unchanged notebooks with either backend. README and CONTRIBUTING use Octavo for opted-in projects. `nbdev-prepare` retains export, test, and clean, skips Quarto configuration generation for Octavo projects, and dispatches README and CONTRIBUTING to the selected backend. Preview and the migration command are not yet connected to this integration." + "The usual `nbdev-docs` command builds the site. Octavo currently builds all pages together. File-selection options are rejected unless left at their defaults; `n_workers` affects only Quarto builds.\n", + "\n", + "Run `nbdev-export` before building to update the generated Python modules and symbol indexes. `nbdev-readme` and `nbdev-contributing` also use Octavo for these projects. `nbdev-prepare` still exports, tests, and cleans notebooks, but skips Quarto configuration generation.\n", + "\n", + "For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Octavo support for `nbdev-preview` is not yet implemented." + ] + }, + { + "cell_type": "markdown", + "id": "7b66c605", + "metadata": {}, + "source": [ + "## Migration finishing prompt" + ] + }, + { + "cell_type": "markdown", + "id": "dc8e677d", + "metadata": {}, + "source": [ + "After running `nbdev-migrate-octavo`, use this prompt with an assistant that can inspect your project. `_quarto.yml` remains available for comparison. Migration changes sources in place; it does not keep them compatible with both renderers.\n", + "\n", + "```markdown\n", + "Help finish this project's migration from Quarto to Octavo. Inspect the project before proposing changes. Agree on the changes with me before editing. Preserve unrelated work. Do not reset a checkout, clean untracked files, commit, or deploy.\n", + "\n", + "Read the retained _quarto.yml, nbdev.yml, pyproject.toml, and generated octavo.yml. Inspect page metadata, stylesheets, render scripts, custom HTML, build hooks, and deployment workflows. Compare against the original site or screenshots where available. If no visual reference is available, say so rather than claiming visual parity.\n", + "\n", + "Automatic migration renames sources and updates selected document references, directives, metadata, and site settings. It does not port the site's design. It does not copy format.html.css into octavo.yml. It leaves stylesheet files and page-layout metadata unchanged. Existing page-level CSS references can still load; inspect those too. An existing octavo.yml is preserved rather than regenerated.\n", + "\n", + "Identify which of these checks apply, then propose a small migration plan:\n", + "\n", + "- Styles and custom pages: inspect the rendered Octavo HTML before adapting selectors. Check homepage grids, buttons, spacing, responsive behavior, and dependencies on Bootstrap or Quarto classes. Reuse compatible CSS selectively. Do not enable an old stylesheet wholesale or copy a compatibility layer without checking the current DOM.\n", + "- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as landing pages and blog listings.\n", + "- Embedded content: inspect iframes, videos, raw HTML, fenced divs, callouts, tabs, shortcodes, and format-conditional blocks used by this project. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the rendered result and relevant Markdown outputs rather than rewriting every block by pattern.\n", + "- Render scripts and documentation: inspect renamed .md.py scripts and generated assets. Update tutorials and examples that still describe Quarto-specific commands, file extensions, or intermediate files. Source renaming alone does not make those explanations correct. Review executable scripts and custom processors before running a build.\n", + "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and the project's actual footer where needed. Do not infer license text from a project name.\n", + "- Additional build outputs: identify artifacts previously generated by Quarto hooks or custom scripts. If the project needs API lists or LLM context files, adapt its own generator using nbdev's docs_post_build callback. Do not add those artifacts to projects that do not need them. Check llms.txt destinations against actual output files, including old .html.md links and malformed Markdown links.\n", + "- Deployment: workflows are not migrated. Check documentation dependencies, replace direct Quarto build commands where needed, and verify the published directory. Octavo's output is relative to the notebooks directory. Use the ordinary nbdev-docs command. Do not copy a test harness's import-path workarounds or obsolete builder calls into production.\n", + "\n", + "After approval, make the agreed changes in small steps. Export notebook-authored modules before building. Run nbdev-docs and the relevant repository Markdown commands. Inspect representative pages at desktop and narrow widths. Check navigation, anchors, images, embedded content, generated assets, and any additional build artifacts. A successful build alone does not establish a successful migration.\n", + "\n", + "Report the changes, checks actually run, unresolved differences, and decisions requiring my review. Keep the original configuration until the migration has been reviewed. Do not copy test-only galleries or fixtures into the site unless I request them.\n", + "```" ] }, { "cell_type": "code", "execution_count": null, - "id": "fd0fd32f", + "id": "7b098bb5", "metadata": {}, "outputs": [], "source": [ @@ -42,16 +84,16 @@ "from contextlib import redirect_stdout\n", "from io import StringIO\n", "from fastcore.utils import *\n", + "from fastcore.script import call_parse\n", "import yaml\n", - "from nbdev.config import get_config, import_obj, _load_toml\n", + "from octavo import *\n", + "from mdhtml import md2gfm\n", + "from nbdev.config import get_config, import_obj, _load_toml, read_nb\n", "from nbdev.doclinks import NbdevLookup\n", "from nbdev.process import NBProcessor\n", "from nbdev.processors import FilterDefaults, add_links\n", "from nbdev.serve import _is_qpy\n", - "from octavo.site import Site\n", - "from octavo.pages import Page, GeneratedFile, scan, page_nm, hidden, ipynb2md\n", - "from octavo.core import md2doc\n", - "from mdhtml import md2gfm\n", + "from nbdev.sync import write_nb\n", "from nbdev.quarto import _doc_mtime_not_older" ] }, @@ -77,7 +119,9 @@ "source": [ "## Notebook processing\n", "\n", - "`OctavoProc` retains custom `doc_procs` and `skip_procs` from nbdev. It preserves rendering directives and exposes generated YAML frontmatter as Markdown. Generated project-symbol links are local; authored production URLs are unchanged." + "Notebooks still need nbdev’s documentation processing before Octavo renders them. `OctavoProc` reuses that processing, including the project’s `doc_procs` and `skip_procs` settings.\n", + "\n", + "It keeps directives such as `#| echo: false` for Octavo to interpret. It also places generated page metadata, such as the title, in a Markdown cell that Octavo can read. The caller chooses whether generated symbol links point into the current site or to published documentation. Links written explicitly by the author are unchanged." ] }, { @@ -89,7 +133,7 @@ "source": [ "#| export\n", "class OctavoProc(FilterDefaults):\n", - " \"Process nbdev notebooks for Octavo without changing Quarto defaults\"\n", + " \"Prepare notebook content, metadata, and symbol links for Octavo\"\n", " def __init__(self, local_lib=None, strip_libs=None): self.nl = NbdevLookup(strip_libs=strip_libs or local_lib, local_lib=local_lib)\n", " def add_links(self, cell): return add_links(cell, nl=self.nl)\n", " def procs(self): return L(self.add_links if o is add_links else o for o in super().procs())\n", @@ -105,7 +149,18 @@ "execution_count": null, "id": "78e180af", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'---\\ndescription: A notebook with rendering directives\\noutput-file: example.html\\ntitle: Example\\n\\n---\\n\\n'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "source = new_nb([mk_cell('# Example\\n\\n> A notebook with rendering directives', 'markdown'),\n", " mk_cell('Use `ModuleMaker`. See [production docs](https://nbdev.fast.ai/api/maker.html#modulemaker).', 'markdown'),\n", @@ -130,7 +185,9 @@ "id": "ce1056e3", "metadata": {}, "source": [ - "A distribution name need not match its import package or nbdev entry-point name. Site builds use the project's declared nbdev entry points to select local symbol indexes; README generation keeps published symbol URLs." + "A website build needs to distinguish this project’s API from dependencies’ APIs. References to this project should lead to pages in the site being built; references to dependencies should lead to their published documentation.\n", + "\n", + "nbdev identifies the project’s symbol indexes through `[project.entry-points.nbdev]` in `pyproject.toml`. It cannot infer those names from the package’s installation name. In the example below, the distribution is named `different-distribution`, but its symbol index is registered as `nbdev`." ] }, { @@ -174,9 +231,13 @@ "source": [ "## Render scripts\n", "\n", - "A *render script* is a Python file with a double suffix, such as `page.md.py` or `drawing.svg.py`. nbdev executes it and saves stdout to the rendered Octavo site without the final `.py` suffix. A `.md.py` file may declare frontmatter in its module docstring; every other generated file contains exactly what its script prints.\n", + "Render scripts generate documentation content during a build. Their filenames specify what they produce: `page.md.py` prints Markdown for a page, while `drawing.svg.py` prints an SVG image.\n", + "\n", + "nbdev executes each script and captures its printed output. Generated Markdown goes through Octavo’s page renderer. Other output is written verbatim under the filename without `.py`; for example, `drawing.svg.py` produces `drawing.svg`.\n", + "\n", + "A `.md.py` script can put YAML page metadata in its opening docstring. nbdev prepends that metadata to the printed Markdown.\n", "\n", - "Octavo never executes anything: nbdev's `script_pages` hands it finished `Page`s and `GeneratedFile`s. A script that fails to run fails the build, since a half-built site is worse than a clear error. Renderscript sources such as `fname.ext.py` are not sent to Octavo, while ordinary single-suffix `.py`, like any non-page sourcefile, files are published as assets." + "Render-script source files are not published. Ordinary single-suffix `.py` files are copied as assets rather than executed. An exception in a render script stops the build." ] }, { @@ -188,7 +249,7 @@ "source": [ "#| export\n", "def render_script(path):\n", - " \"Captured stdout from a double-suffix render script; prepend optional `.md.py` frontmatter\"\n", + " \"Execute a double-suffix render script and return its printed output with optional page metadata\"\n", " path = Path(path)\n", " if path.suffix!='.py' or len(path.suffixes)<2: return\n", " fm = _is_qpy(path) if path.name.endswith('.md.py') else None\n", @@ -239,7 +300,7 @@ " test_eq([(f.src,f.dest,f.content) for f in files], [('images/dot.svg.py','images/dot.svg','\\n')])\n", " test_is(render_script(d/'ordinary.py'), None)\n", " (d/'bad.md.py').write_text('1/0')\n", - " test_fail(lambda: script_pages(d), contains='division')" + " with expect_fail(ZeroDivisionError, contains='division'): script_pages(d)" ] }, { @@ -255,6 +316,8 @@ "id": "24badb1f", "metadata": {}, "source": [ + "Some projects publish extra files alongside their documentation, such as an API list or an LLM context file. Use a post-build callback to generate these after the website has been built.\n", + "\n", "Set `docs_post_build` to an importable `module:function` in `pyproject.toml`:\n", "\n", "```toml\n", @@ -274,7 +337,9 @@ "source": [ "## Build configuration\n", "\n", - "`octavo.yml` must exist and contain a YAML mapping. An empty file uses defaults. Validate it before executing render scripts or processing notebooks." + "`octavo.yml` contains named settings, such as `title: Demo`. An empty file selects Octavo’s defaults. A YAML list or a single value such as `false` cannot supply those settings.\n", + "\n", + "The build checks this file before running render scripts or notebook processors. Invalid configuration therefore fails before project code executes." ] }, { @@ -286,7 +351,7 @@ "source": [ "#| export\n", "def octavo_config(root):\n", - " \"Read an existing octavo.yml mapping before running project code\"\n", + " \"Read `octavo.yml` as a settings dictionary; return an empty dictionary for an empty file\"\n", " root = Path(root)\n", " if not (root/'octavo.yml').is_file(): raise FileNotFoundError(root/'octavo.yml')\n", " settings = yaml.safe_load((root/'octavo.yml').read_text())\n", @@ -317,7 +382,9 @@ "id": "5407994a", "metadata": {}, "source": [ - "The site build gives nbdev’s symbol linker the current project identity. Generated project references become site-root-relative document addresses; dependency references and explicitly authored absolute links remain absolute. Octavo resolves local document addresses for each output format." + "Links to this project’s API should stay within the website being built. For example, when building nbdev’s website, a generated link for `ModuleMaker` points to `/api/maker.html#modulemaker` rather than the published nbdev site.\n", + "\n", + "Links to dependencies still point to their published documentation. An absolute URL written by the author is not changed." ] }, { @@ -333,7 +400,7 @@ "):\n", " \"Create Octavo docs and run the optional docs_post_build callback\"\n", " cfg = get_config(path)\n", - " settings = octavo_config(cfg.nbs_path)\n", + " settings = {'output': cfg.doc_path, **octavo_config(cfg.nbs_path)}\n", " with working_directory(cfg.config_path):\n", " pages,files = script_pages(cfg.nbs_path)\n", " docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages\n", @@ -350,7 +417,7 @@ "id": "0d5735ac", "metadata": {}, "source": [ - "## Build an opted-in project\n", + "## Build a documentation site\n", "\n", "`nbdev-docs` reads `octavo.yml` and processes notebooks and render scripts together. It does not migrate sources or change `_quarto.yml`. This temporary project builds linked pages and generated assets, then demonstrates that invalid configuration is rejected before executing a render script." ] @@ -360,36 +427,99 @@ "execution_count": null, "id": "7c8fa2af", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_tmp = tempfile.TemporaryDirectory()\n", + "project = Path(build_tmp.name)\n", + "nbs = project/'nbs'\n", + "nbs.mkdir()\n", + "(project/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\nskip_procs = \"insert_warning\"\\n')\n", + "(nbs/'octavo.yml').write_text('title: Demo\\noutput: ../_octavo\\nsite_url: https://example.com\\n')\n", + "(nbs/'_quarto.yml').write_text('project:\\n type: website\\n')\n", + "(nbs/'index.md').write_text('# Home\\n\\nRead the [guide](guide.ipynb).')\n", + "guide = new_nb([mk_cell('# Guide\\n\\n> A notebook page', 'markdown'), mk_cell('Notebook content.', 'markdown')])\n", + "write_nb(guide, nbs/'guide.ipynb')\n", + "(nbs/'generated.md.py').write_text('print(\"# Generated page\")')\n", + "(nbs/'image.svg.py').write_text('print(\"\")')" + ] + }, + { + "cell_type": "markdown", + "id": "f7e9a3f2", + "metadata": {}, + "source": [ + "The ordinary build command leaves Quarto configuration unchanged. Without `output` in `octavo.yml`, it uses nbdev’s project-relative `doc_path`. An explicit Octavo output takes precedence and is relative to the notebooks directory. Here, `../_octavo` overrides the default project-level `_docs` directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5801459", + "metadata": {}, "outputs": [], "source": [ - "with tempfile.TemporaryDirectory() as d:\n", - " project = Path(d)\n", - " nbs = project/'nbs'\n", - " nbs.mkdir()\n", - " (project/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\nskip_procs = \"insert_warning\"\\n')\n", - " (nbs/'octavo.yml').write_text('title: Demo\\noutput: ../_octavo\\nsite_url: https://example.com\\n')\n", - " (nbs/'_quarto.yml').write_text('project:\\n type: website\\n')\n", - " (nbs/'index.md').write_text('# Home\\n\\nRead the [guide](guide.ipynb).')\n", - " guide = new_nb([mk_cell('# Guide\\n\\n> A notebook page', 'markdown'), mk_cell('Notebook content.', 'markdown')])\n", - " write_nb(guide, nbs/'guide.ipynb')\n", - " (nbs/'generated.md.py').write_text('print(\"# Generated page\")')\n", - " (nbs/'image.svg.py').write_text('print(\"\")')\n", - " cwd = Path.cwd()\n", - " with working_directory(project): run(r'nbdev-docs --file-re \"\\.(?:ipynb|qmd|html)$\"')\n", - " nbdev_docs.__wrapped__(project, file_glob=None)\n", - " test_eq(Path.cwd(), cwd)\n", - " out = project/'_octavo'\n", - " assert 'guide.html' in (out/'index.html').read_text()\n", - " assert 'Notebook content.' in (out/'guide.html').read_text()\n", - " assert 'AUTOGENERATED' not in (out/'guide.md').read_text()\n", - " assert (out/'generated.html').exists()\n", - " test_eq((out/'image.svg').read_text(), '\\n')\n", - " test_eq((nbs/'_quarto.yml').read_text(), 'project:\\n type: website\\n')\n", - " test_fail(lambda: nbdev_docs.__wrapped__(project, file_glob='*.ipynb'), contains='file_glob')\n", - " (nbs/'octavo.yml').write_text('- invalid')\n", - " (nbs/'generated.md.py').write_text('raise RuntimeError(\"script ran before validation\")')\n", - " test_fail(lambda: nbdev_docs.__wrapped__(project), contains='mapping')\n", - " test_eq(Path.cwd(), cwd)" + "with working_directory(project): run('nbdev-docs')\n", + "out = project/'_octavo'\n", + "assert 'guide.html' in (out/'index.html').read_text()\n", + "assert 'Notebook content.' in (out/'guide.html').read_text()\n", + "assert 'AUTOGENERATED' not in (out/'guide.md').read_text()\n", + "assert (out/'generated.html').exists()\n", + "test_eq((out/'image.svg').read_text(), '\\n')\n", + "test_eq((nbs/'_quarto.yml').read_text(), 'project:\\n type: website\\n')" + ] + }, + { + "cell_type": "markdown", + "id": "820a17f9", + "metadata": {}, + "source": [ + "Existing scripts can pass the default file-selection options explicitly. Octavo accepts those calls, but still builds the whole site. For normal use, run `nbdev-docs` without file-selection options." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd46348b", + "metadata": {}, + "outputs": [], + "source": [ + "with working_directory(project): run(r'nbdev-docs --file-re \"\\.(?:ipynb|qmd|html)$\"')\n", + "nbdev_docs(project, file_glob=None)" + ] + }, + { + "cell_type": "markdown", + "id": "424b4dde", + "metadata": {}, + "source": [ + "The Octavo integration currently builds the whole site. Non-default file filters are rejected because selective builds are not implemented.\n", + "\n", + "Invalid configuration must fail before any render script runs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "773d00ae", + "metadata": {}, + "outputs": [], + "source": [ + "test_fail(lambda: nbdev_docs(project, file_glob='*.ipynb'), contains='file_glob')\n", + "(nbs/'octavo.yml').write_text('- invalid')\n", + "(nbs/'generated.md.py').write_text('raise RuntimeError(\"script ran before validation\")')\n", + "test_fail(lambda: nbdev_docs(project), contains='mapping')\n", + "build_tmp.cleanup()" ] }, { @@ -407,7 +537,11 @@ "source": [ "`nbdev-readme` and `nbdev-contributing` render the configured notebooks to the project root without Quarto. They use stored outputs and do not execute ordinary code cells. nbdev processors still run, including custom processors and generated API documentation. Titles and descriptions become Markdown. Symbol links use published documentation URLs.\n", "\n", - "Embedded images are extracted into `README_files` or `CONTRIBUTING_files`. Authored relative destinations are unchanged, including reference-style links. Authors must choose paths suitable for the repository-root document. A missing source notebook is skipped; `chk_time=True` skips a document newer than its source." + "Images stored in notebook outputs are extracted into a directory beside the generated Markdown. The directory uses the source notebook’s name: `index.ipynb` produces `index_files`, even though the generated document is named `README.md`. This matches nbdev’s Quarto convention.\n", + "\n", + "Authored relative links are copied unchanged; their destinations are not adjusted for the document’s new location.\n", + "\n", + "A missing source notebook is skipped. With `chk_time=True`, an existing document is skipped when its modification time is at least as recent as the source notebook’s." ] }, { @@ -439,7 +573,7 @@ " doc = md2doc(ipynb2md(src, nbproc=proc))\n", " head = f'# {doc.meta[\"title\"]}\\n\\n' if doc.meta.get('title') else ''\n", " if desc:=doc.meta.get('description'): head += f'{desc}\\n\\n'\n", - " md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{name}_files', raw=('md', 'html'))" + " md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{src.stem}_files', raw=('md', 'html'))" ] }, { @@ -447,42 +581,1184 @@ "execution_count": null, "id": "f5e7d85b", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "readme_tmp = tempfile.TemporaryDirectory()\n", + "project = Path(readme_tmp.name)\n", + "nbs = project/'nbs'\n", + "nbs.mkdir()\n", + "(project/'pyproject.toml').write_text(r'''[project]\n", + "name = \"different-distribution\"\n", + "[project.entry-points.nbdev]\n", + "nbdev = \"nbdev._modidx:d\"\n", + "[tool.nbdev]\n", + "nbs_path = \"nbs\"\n", + "contributing_nb = \"help.ipynb\"\n", + "''')\n", + "(nbs/'octavo.yml').write_text('{}')" + ] + }, + { + "cell_type": "markdown", + "id": "85cd81cb", + "metadata": {}, + "source": [ + "README generation should use a notebook’s saved results without rerunning its ordinary code cells. The example below constructs a notebook containing a saved image and code that raises an exception if executed.\n", + "\n", + "Successful rendering must include the saved image without running that code. This matches Quarto’s `--no-execute` behaviour." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b588547", + "metadata": {}, "outputs": [], "source": [ - "with tempfile.TemporaryDirectory() as d:\n", - " project = Path(d)\n", - " nbs = project/'nbs'\n", - " nbs.mkdir()\n", - " (project/'pyproject.toml').write_text('[project]\\nname = \"different-distribution\"\\n[project.entry-points.nbdev]\\nnbdev = \"nbdev._modidx:d\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\ncontributing_nb = \"help.ipynb\"\\n')\n", - " (nbs/'octavo.yml').write_text('{}')\n", - " png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aX1sAAAAASUVORK5CYII='\n", - " image = mk_cell('#| echo: false\\nraise RuntimeError(\"must not execute\")')\n", - " image.outputs = [dict(output_type='display_data', metadata={}, data={'image/png': png})]\n", - " book = new_nb([mk_cell('# Example\\n\\n> Repository document', 'markdown'),\n", - " mk_cell('Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md', 'markdown'),\n", - " image, mk_cell('#| include: false\\n\"excluded cell\"')])\n", - " write_nb(book, nbs/'index.ipynb')\n", - " write_nb(book, nbs/'help.ipynb')\n", - " cwd = Path.cwd()\n", - " octavo_readme(project)\n", - " with working_directory(project):\n", - " run('nbdev-readme')\n", - " run('nbdev-contributing')\n", - " test_eq(Path.cwd(), cwd)\n", - " for name in ('README', 'CONTRIBUTING'):\n", - " text = (project/f'{name}.md').read_text()\n", - " assert '# Example' in text and 'Repository document' in text\n", - " assert 'https://nbdev.fast.ai/api/maker.html#modulemaker' in text\n", - " assert '(guide.md)' in text and '[ref]: guide.md' in text\n", - " assert 'must not execute' not in text and 'excluded cell' not in text\n", - " assert 'data:image' not in text and f'{name}_files/' in text\n", - " assert list((project/f'{name}_files').glob('*.png'))\n", - " before = (project/'README.md').stat().st_mtime_ns\n", - " octavo_readme(project, chk_time=True)\n", - " test_eq((project/'README.md').stat().st_mtime_ns, before)\n", - " assert not (nbs/'_quarto.yml').exists()\n", - " (nbs/'help.ipynb').unlink()\n", - " octavo_readme(project, contributing=True)" + "png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aX1sAAAAASUVORK5CYII='\n", + "image = mk_cell('#| echo: false\\nraise RuntimeError(\"must not execute\")')\n", + "image.outputs = [dict(output_type='display_data', metadata={}, data={'image/png': png})]\n", + "book = new_nb([mk_cell('# Example\\n\\n> Repository document', 'markdown'),\n", + " mk_cell('Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md', 'markdown'),\n", + " image, mk_cell('#| include: false\\n\"excluded cell\"')])\n", + "write_nb(book, nbs/'index.ipynb')\n", + "write_nb(book, nbs/'help.ipynb')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "698fba67", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "{ 'cells': [ { 'cell_type': 'markdown',\n", + " 'directives_': {},\n", + " 'id': '03d65b6e',\n", + " 'idx_': 0,\n", + " 'lang_': 'python',\n", + " 'metadata': {},\n", + " 'source': '# Example\\n\\n> Repository document'},\n", + " { 'cell_type': 'markdown',\n", + " 'directives_': {},\n", + " 'id': 'bcf12a9e',\n", + " 'idx_': 1,\n", + " 'lang_': 'python',\n", + " 'metadata': {},\n", + " 'source': 'Use `ModuleMaker`. [Guide](guide.md). '\n", + " '[Reference][ref]\\n'\n", + " '\\n'\n", + " '[ref]: guide.md'},\n", + " { 'cell_type': 'code',\n", + " 'directives_': {},\n", + " 'execution_count': None,\n", + " 'id': '0b2fc6cb',\n", + " 'idx_': 2,\n", + " 'lang_': 'python',\n", + " 'metadata': {},\n", + " 'outputs': [ { 'data': { 'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aX1sAAAAASUVORK5CYII='},\n", + " 'metadata': {},\n", + " 'output_type': 'display_data'}],\n", + " 'source': '#| echo: false\\n'\n", + " 'raise RuntimeError(\"must not execute\")'},\n", + " { 'cell_type': 'code',\n", + " 'directives_': {},\n", + " 'execution_count': None,\n", + " 'id': 'ba6db483',\n", + " 'idx_': 3,\n", + " 'lang_': 'python',\n", + " 'metadata': {},\n", + " 'outputs': [],\n", + " 'source': '#| include: false\\n\"excluded cell\"'}],\n", + " 'metadata': {},\n", + " 'nbformat': 4,\n", + " 'nbformat_minor': 5}\n", + "```" + ], + "text/plain": [ + "{'cells': [{'cell_type': 'markdown',\n", + " 'source': '# Example\\n\\n> Repository document',\n", + " 'directives_': {},\n", + " 'metadata': {},\n", + " 'idx_': 0,\n", + " 'lang_': 'python',\n", + " 'id': '03d65b6e'},\n", + " {'cell_type': 'markdown',\n", + " 'source': 'Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md',\n", + " 'directives_': {},\n", + " 'metadata': {},\n", + " 'idx_': 1,\n", + " 'lang_': 'python',\n", + " 'id': 'bcf12a9e'},\n", + " {'cell_type': 'code',\n", + " 'source': '#| echo: false\\nraise RuntimeError(\"must not execute\")',\n", + " 'directives_': {},\n", + " 'metadata': {},\n", + " 'outputs': [{'output_type': 'display_data',\n", + " 'metadata': {},\n", + " 'data': {'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aX1sAAAAASUVORK5CYII='}}],\n", + " 'execution_count': None,\n", + " 'idx_': 2,\n", + " 'lang_': 'python',\n", + " 'id': '0b2fc6cb'},\n", + " {'cell_type': 'code',\n", + " 'source': '#| include: false\\n\"excluded cell\"',\n", + " 'directives_': {},\n", + " 'metadata': {},\n", + " 'outputs': [],\n", + " 'execution_count': None,\n", + " 'idx_': 3,\n", + " 'lang_': 'python',\n", + " 'id': 'ba6db483'}],\n", + " 'metadata': {},\n", + " 'nbformat': 4,\n", + " 'nbformat_minor': 5}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "book" + ] + }, + { + "cell_type": "markdown", + "id": "36527ea0", + "metadata": {}, + "source": [ + "README and CONTRIBUTING use separate source notebooks and produce separate Markdown files. This test gives both notebooks identical content to check that both commands handle hidden code, saved images, and links consistently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b518cd0", + "metadata": {}, + "outputs": [], + "source": [ + "cwd = Path.cwd()\n", + "octavo_readme(project)\n", + "with working_directory(project):\n", + " run('nbdev-readme')\n", + " run('nbdev-contributing')" + ] + }, + { + "cell_type": "markdown", + "id": "a1f61198", + "metadata": {}, + "source": [ + "A README is read in the repository, not alongside the generated API pages. Its generated symbol links therefore point to published documentation—even when they refer to this project’s own API.\n", + "\n", + "For example, nbdev’s README links `ModuleMaker` to the published nbdev website. The nbdev website itself uses a local API link. This difference depends on where the document will be read, not on which library the symbol belongs to." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94428a10", + "metadata": {}, + "outputs": [], + "source": [ + "for name,stem in (('README', 'index'), ('CONTRIBUTING', 'help')):\n", + " text = (project/f'{name}.md').read_text()\n", + " assert '# Example' in text and 'Repository document' in text\n", + " assert 'https://nbdev.fast.ai/api/maker.html#modulemaker' in text\n", + " assert '(guide.md)' in text and '[ref]: guide.md' in text\n", + " assert 'must not execute' not in text and 'excluded cell' not in text\n", + " assert 'data:image' not in text and f'{stem}_files/' in text\n", + " assert list((project/f'{stem}_files').glob('*.png'))" + ] + }, + { + "cell_type": "markdown", + "id": "5dc6d9c7", + "metadata": {}, + "source": [ + "An up-to-date README should not be rewritten. Projects without a CONTRIBUTING source should still build their other documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7cd1ddd", + "metadata": {}, + "outputs": [], + "source": [ + "before = (project/'README.md').stat().st_mtime_ns\n", + "octavo_readme(project, chk_time=True)\n", + "test_eq((project/'README.md').stat().st_mtime_ns, before)\n", + "assert not (nbs/'_quarto.yml').exists()\n", + "(nbs/'help.ipynb').unlink()\n", + "octavo_readme(project, contributing=True)\n", + "readme_tmp.cleanup()" + ] + }, + { + "cell_type": "markdown", + "id": "c4466973", + "metadata": {}, + "source": [ + "## Source migration" + ] + }, + { + "cell_type": "markdown", + "id": "b6ef5093", + "metadata": {}, + "source": [ + "`nbdev-migrate-octavo` converts Quarto documentation sources in place. Commit or back up the project before running it. The command:\n", + "\n", + "1. Renames `.qmd` documents to `.md` and `.qmd.py` render scripts to `.md.py`.\n", + "2. Rewrites references to renamed documents.\n", + "3. Replaces Quarto content-visible block markers with raw-format fences.\n", + "4. Converts selected page-metadata names, such as `subtitle` to `description`.\n", + "5. Creates `octavo.yml` from selected Quarto settings when no Octavo configuration exists.\n", + "6. Leaves `_quarto.yml` unchanged for reference.\n", + "\n", + "The presence of `octavo.yml` selects Octavo.\n", + "\n", + "Migration converts sources and configuration, not the site’s design. CSS, layouts, and themes need project-specific review. Stylesheet files and `page-layout` metadata remain unchanged. Quarto stylesheet references are not copied into `octavo.yml`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "267d0150", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _qmd_moves(path):\n", + " \"`.qmd` documents and `.qmd.py` render scripts under `path`, mapped to Octavo names\"\n", + " path = Path(path)\n", + " ps = L(path.rglob('*.qmd')) + L(path.rglob('*.qmd.py'))\n", + " return {p:p.with_name(re.sub(r'\\.qmd(?=\\.py$|$)', '.md', p.name)) for p in sorted(ps)\n", + " if not any(o[0] in '._' for o in p.relative_to(path).parts)}" + ] + }, + { + "cell_type": "markdown", + "id": "c876d09b", + "metadata": {}, + "source": [ + "Renaming a document also requires updating references to it. A page beside `guide.qmd` refers to it differently from a page in a subdirectory: `guide.qmd` versus `../guide.qmd`. Migration calculates replacement paths from the documentation root and from each source file’s directory.\n", + "\n", + "The current implementation performs text replacement rather than parsing links. Matching text outside link destinations can also change." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6688f7e", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _qmd_repls(src, path, moves):\n", + " \"Old and new spellings of each moved document, as seen from `src`\"\n", + " def _rel(p, start): return os.path.relpath(p, start).replace(os.sep, '/')\n", + " res = {}\n", + " for old,new in moves.items():\n", + " res[_rel(old, path)] = _rel(new, path)\n", + " res[_rel(old, src.parent)] = _rel(new, src.parent)\n", + " return sorted(res.items(), key=lambda o: len(o[0]), reverse=True)\n", + "\n", + "def repl_qmd_links(txt, moves, src, root):\n", + " \"Rewrite relative and root-relative `.qmd` link targets to `.md`\"\n", + " for old,new in _qmd_repls(src, root, moves): txt = txt.replace(old, new)\n", + " return txt" + ] + }, + { + "cell_type": "markdown", + "id": "19f57493", + "metadata": {}, + "source": [ + "Quarto lets authors mark a block of content for a particular output format—for example, HTML for the website or Markdown for a README. Migration replaces Quarto’s block markers with mdhtml's raw-format fences.\n", + "\n", + "For example, this Quarto block:\n", + "\n", + "````markdown\n", + "::: {.content-visible when-format=\"html\"}\n", + "

Only include this in HTML output.

\n", + ":::\n", + "````\n", + "\n", + "becomes:\n", + "\n", + "````markdown\n", + "```{=html}\n", + "

Only include this in HTML output.

\n", + "```\n", + "````\n", + "\n", + "The function changes the surrounding markers, not the content inside them. Quarto’s `markdown`, `commonmark`, and `md` format names all become `md`. If an HTML block has a `style` attribute, the function preserves it by wrapping the content in a `
` with that style.\n", + "\n", + "This conversion changes block markers, not the language of the body. Markdown inside an HTML-targeted block is not converted to HTML. The current helper also does not preserve `unless-format` conditions. Review these cases before relying on the converted output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "237aea2c", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_re_cv = re.compile(r'^:{3,}[^\\S\\r\\n]*\\{([^}]*\\.content-visible[^}]*)\\}[^\\S\\r\\n]*\\n(.*?)\\n:{3,}[^\\S\\r\\n]*$', re.MULTILINE | re.DOTALL)\n", + "\n", + "def _repl_cv_match(m):\n", + " attrs,body = m.group(1),m.group(2).strip()\n", + " m_fmt = re.search(r'when-format=[\"\\']?(\\w+)[\"\\']?', attrs)\n", + " fmt = m_fmt.group(1).lower() if m_fmt else 'html'\n", + " target = 'md' if fmt in ('markdown', 'commonmark', 'md') else fmt\n", + " if target=='html':\n", + " if (m_style := re.search(r'style=[\"\\']([^\"\\']+)[\"\\']', attrs)):\n", + " body = f'
\\n{body}\\n
'\n", + " return f'```{{={target}}}\\n{body}\\n```'\n", + "\n", + "def repl_content_visible(s):\n", + " \"Convert Quarto `::: {.content-visible when-format=...}` to ````{=md}```` / ````{=html}````\"\n", + " return _re_cv.sub(_repl_cv_match, s)" + ] + }, + { + "cell_type": "markdown", + "id": "067d6a52", + "metadata": {}, + "source": [ + "Explicit Octavo keys take precedence over legacy aliases. Only top-level keys are normalized; nested metadata retains its original meaning. `page-layout` remains unchanged for manual review because Quarto and Octavo layouts are not interchangeable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04d5a738", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _dump_fm(fm): return yaml.dump(fm, sort_keys=False, allow_unicode=True, width=10**9)\n", + "\n", + "def repl_frontmatter_keys(s):\n", + " \"Normalize legacy and Quarto frontmatter keys for Octavo\"\n", + " fm,body = frontmatter(s)\n", + " if not fm: return s\n", + " changed = False\n", + " aliases = [('categories','tags'), ('subtitle','description'), ('summary','description')]\n", + " for old,new in aliases + [(k,k.replace('-','_')) for k in fm if '-' in k and k!='page-layout']:\n", + " if old not in fm: continue\n", + " fm.setdefault(new, fm.pop(old))\n", + " changed = True\n", + " if fm.pop('toc', None) is False:\n", + " fm['toc_depth'] = 0\n", + " changed = True\n", + " if changed: return f'---\\n{_dump_fm(fm)}---\\n\\n{body.lstrip(chr(10))}'\n", + " return s" + ] + }, + { + "cell_type": "markdown", + "id": "35627bb1", + "metadata": {}, + "source": [ + "Render scripts keep their executable body. Only the leading YAML docstring needs the same metadata conversion as a document." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ceec29d", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_script_fm = re.compile(r'\\A(?P\"{3}|\\'{3})---\\n(?P.*?)\\n---(?P=q)', re.DOTALL)\n", + "\n", + "def repl_script_frontmatter_keys(s):\n", + " \"Normalize a render script's leading YAML docstring for Octavo\"\n", + " if not (m := _script_fm.match(s)): return s\n", + " old = f\"---\\n{m.group('fm')}\\n---\\n\"\n", + " new = repl_frontmatter_keys(old)\n", + " if new == old: return s\n", + " fm,_ = frontmatter(new)\n", + " return f\"{m.group('q')}---\\n{_dump_fm(fm)}---{m.group('q')}\" + s[m.end():]" + ] + }, + { + "cell_type": "markdown", + "id": "f8aab4d6", + "metadata": {}, + "source": [ + "Documentation conversion must not rewrite executable notebook code or discard saved results. `migrate_nb_cells` applies the text conversions only to Markdown cells. Code cells and their saved outputs are retained.\n", + "\n", + "`migrate_nb_cells` and `migrate_md_file` write a file only when a conversion changes its content." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c93c529", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _apply_procs(txt, procs):\n", + " for p in procs: txt = p(txt)\n", + " return txt\n", + "\n", + "def migrate_nb_cells(path, procs):\n", + " \"Apply `procs` to all markdown cells in a notebook in-place\"\n", + " nb,changed = read_nb(path),False\n", + " for c in nb.cells:\n", + " if c.cell_type != 'markdown' or not c.get('source'): continue\n", + " new_src = _apply_procs(c.source, procs)\n", + " if new_src != c.source: c.source,changed = new_src,True\n", + " if changed: write_nb(nb, path)\n", + "\n", + "def migrate_md_file(path, procs):\n", + " \"Apply `procs` to a Markdown/QMD file in-place\"\n", + " txt = path.read_text()\n", + " new_txt = _apply_procs(txt, procs)\n", + " if new_txt!=txt: path.write_text(new_txt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01ed763e", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _nav_href(h):\n", + " if not h or '://' in h or h.startswith(('#','mailto:')): return h\n", + " return re.sub(r'\\.(?:ipynb|qmd|md)(?=[?#]|$)', '.html', h)\n", + "\n", + "def _nav_item(item):\n", + " res = {'text': item['text']}\n", + " if 'menu' in item: res['children'] = [{'text': m['text'], 'href': _nav_href(m['href'])} for m in item['menu'] if 'href' in m]\n", + " elif 'href' in item: res['href'] = _nav_href(item['href'])\n", + " return res" + ] + }, + { + "cell_type": "markdown", + "id": "dac4196d", + "metadata": {}, + "source": [ + "Icon names belong to the renderer. Unsupported icons are reported rather than emitted as broken tool links." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acd395a1", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_ICON_MAP = {'twitter':'x', 'twitter-x':'x'}\n", + "_ICON_LABELS = {'github':'GitHub', 'rss':'RSS'}\n", + "\n", + "def _tool_item(item):\n", + " \"An octavo `tools` entry from a Quarto icon link, or None where octavo has no such icon\"\n", + " from octavo.theme import ICONS\n", + " ico = _ICON_MAP.get(item['icon'], item['icon'])\n", + " if ico not in ICONS: return None\n", + " return {'icon': ico, 'href': _nav_href(item['href']), 'label': item.get('aria-label') or item.get('text') or _ICON_LABELS.get(ico, ico.title())}" + ] + }, + { + "cell_type": "markdown", + "id": "47b0e5d9", + "metadata": {}, + "source": [ + "Configuration is parsed before any source files are changed. Empty YAML uses defaults; a scalar or list cannot represent site settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "deb7ee06", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _quarto_config(nbs_path):\n", + " qy_p,ny_p = nbs_path/'_quarto.yml',nbs_path/'nbdev.yml'\n", + " qy = yaml.safe_load(qy_p.read_text()) if qy_p.exists() else {}\n", + " ny = yaml.safe_load(ny_p.read_text()) if ny_p.exists() else {}\n", + " for fname,settings in ((qy_p,qy), (ny_p,ny)):\n", + " if settings is not None and not isinstance(settings, dict): raise ValueError(f'{fname} must contain a YAML mapping')\n", + " return qy or {},ny or {}" + ] + }, + { + "cell_type": "markdown", + "id": "083e9cb7", + "metadata": {}, + "source": [ + "nbdev already knows where to put the website through `doc_path`, relative to the project root. Migration leaves `output` out of `octavo.yml` when that destination is sufficient.\n", + "\n", + "An explicit Quarto `output-dir` can name a different destination. Migration preserves that override, expressing it relative to the notebooks directory for Octavo. For example, a project-level `website` directory becomes `../website` when notebooks live in `nbs`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "127739bd", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _octavo_output(nbs_path, qy, ny):\n", + " \"Return a Quarto output override relative to notebooks, or None when doc_path suffices\"\n", + " out_dir = (ny.get('project', {}) or {}).get('output-dir') or (qy.get('project', {}) or {}).get('output-dir')\n", + " if not out_dir: return\n", + " cfg = get_config(nbs_path)\n", + " out_dir = (cfg.config_path/Path(out_dir).expanduser()).resolve()\n", + " if out_dir == cfg.doc_path.resolve(): return\n", + " return os.path.relpath(out_dir, Path(nbs_path).resolve()).replace(os.sep, '/')" + ] + }, + { + "cell_type": "markdown", + "id": "4463c3c6", + "metadata": {}, + "source": [ + "Quarto and Octavo use different settings for navigation bars. Migration converts text links and dropdowns into Octavo navigation links, and icon links into tools. Links to documentation sources are changed to HTML destinations so visitors reach rendered pages. Explicit accessibility labels are preserved. Icons not bundled with Octavo are reported and omitted for manual replacement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "201d0976", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _octavo_nav(nbs_path, qy, ny):\n", + " \"Convert Quarto navbar settings into Octavo navigation links and icon tools\"\n", + " cfg = {}\n", + " def _nav(k): return nested_idx(qy,'website','navbar',k) or nested_idx(ny,'website','navbar',k) or []\n", + " if nav_left := _nav('left'): cfg['nav_links'] = [_nav_item(o) for o in nav_left if 'text' in o and ('href' in o or 'menu' in o)]\n", + " links = L(_nav('right')+_nav('tools')).filter(lambda o: 'icon' in o and 'href' in o)\n", + " tools = links.map(_tool_item)\n", + " for o,t in zip(links,tools):\n", + " if t is None: print(f\"{nbs_path/'octavo.yml'}: dropped tool icon {o['icon']!r} ({o['href']}): not bundled by octavo. Add it back with `icon: ` to your own SVG.\")\n", + " if (tools := tools.filter()): cfg['tools'] = list(tools)\n", + " return cfg" + ] + }, + { + "cell_type": "markdown", + "id": "e1cd81ec", + "metadata": {}, + "source": [ + "A migrated site needs its title, navigation, and other supported settings in Octavo’s format. `_create_octavo_cfg` collects those settings into a dictionary. An output override is included only when the Quarto destination differs from nbdev’s `doc_path`. The migration command writes the dictionary to `octavo.yml` only when that file does not already exist.\n", + "\n", + "This is not a complete translation of Quarto configuration. Settings without a conversion still need manual review." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f2d4b3b", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _create_octavo_cfg(nbs_path):\n", + " \"Return Octavo settings derived from `_quarto.yml` and `nbdev.yml`\"\n", + " nbs_path = Path(nbs_path).resolve()\n", + " qy,ny = _quarto_config(nbs_path)\n", + " ws_q,ws_n = qy.get('website', {}) or {},ny.get('website', {}) or {}\n", + " fmt_html = (qy.get('format', {}) or {}).get('html', {}) or {}\n", + " cfg = {}\n", + " if (output := _octavo_output(nbs_path, qy, ny)) is not None: cfg['output'] = output\n", + " title = ws_n.get('title') or ws_q.get('title')\n", + " if title: cfg['title'] = title\n", + " site_url = ws_n.get('site-url') or ws_q.get('site-url')\n", + " if site_url: cfg['site_url'] = site_url.rstrip('/')\n", + " if 'toc-depth' in fmt_html:\n", + " try: cfg['toc_depth'] = max(1, int(fmt_html['toc-depth']) - 1)\n", + " except (ValueError, TypeError): pass\n", + " else: cfg['toc_depth'] = 0 if fmt_html.get('toc') is False else 3\n", + " favicon = ws_n.get('favicon') or ws_q.get('favicon')\n", + " if favicon: cfg['favicon'] = favicon\n", + " cfg.update(_octavo_nav(nbs_path, qy, ny))\n", + " return cfg" + ] + }, + { + "cell_type": "markdown", + "id": "af51d9b8", + "metadata": {}, + "source": [ + "Switching builders requires source changes as well as new configuration: renamed documents need updated links, and Quarto-specific markup needs conversion. Migration checks rename conflicts and configuration before editing, preserves an existing `octavo.yml`, and retains `_quarto.yml` for comparison. Changes are made in place without automatic rollback, so commit or back up the project first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4b4478c", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _migrate_octavo(path):\n", + " \"Migrate documentation sources in place, create missing Octavo configuration, and return the rename mapping\"\n", + " path = Path(path).expanduser()\n", + " if not path.is_dir(): raise NotADirectoryError(path)\n", + " moves = _qmd_moves(path)\n", + " for new in moves.values():\n", + " if new.exists(): raise FileExistsError(f'Cannot rename to existing file: {new}')\n", + " qy = path/'_quarto.yml'\n", + " if (path/'octavo.yml').exists(): octavo_config(path)\n", + " elif not qy.is_file(): raise FileNotFoundError(qy)\n", + " cfg = _create_octavo_cfg(path) if qy.exists() and not (path/'octavo.yml').exists() else None\n", + "\n", + " srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md')\n", + " and not any(o[0] in '._' for o in p.relative_to(path).parts)]\n", + " scripts = [p for p in sorted(path.rglob('*.qmd.py'))\n", + " if not any(o[0] in '._' for o in p.relative_to(path).parts)]\n", + " for src in srcs:\n", + " procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_content_visible, repl_frontmatter_keys]\n", + " if src.suffix=='.ipynb': migrate_nb_cells(src, procs)\n", + " else: migrate_md_file(src, procs)\n", + " for src in scripts: migrate_md_file(src, [repl_script_frontmatter_keys])\n", + "\n", + " for old,new in moves.items(): old.rename(new)\n", + "\n", + " if cfg: (path/'octavo.yml').write_text(yaml.dump(cfg, sort_keys=False))\n", + " return moves" + ] + }, + { + "cell_type": "markdown", + "id": "ed43ff9d", + "metadata": {}, + "source": [ + "The default comes from the project configuration, so the command also works when notebooks live outside `nbs`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e332ece4", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@call_parse\n", + "def nbdev_migrate_octavo(\n", + " path:str=None, # Documentation directory (defaults to configured nbs_path)\n", + "):\n", + " \"Migrate Quarto-era nbdev project to Octavo\"\n", + " moves = _migrate_octavo(get_config().nbs_path if path is None else path)\n", + " for old,new in moves.items(): print(f'{old} -> {new}')" + ] + }, + { + "cell_type": "markdown", + "id": "65ed915c", + "metadata": {}, + "source": [ + "A documentation project can mix written pages, notebooks, and pages generated by Python scripts. The following project includes all three. After migration, we inspect their filenames, metadata, links, and retained content." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08610b6b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "115" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "migration_tmp = tempfile.TemporaryDirectory()\n", + "d = migration_tmp.name\n", + "nbs = Path(d)/'nbs'\n", + "(Path(d)/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\n')\n", + "(nbs/'blog/posts').mkdir(parents=True)\n", + "(nbs/'images').mkdir()\n", + "(nbs/'guide.qmd').write_text(r'''---\n", + "subtitle: A handy guide\n", + "toc-depth: 2\n", + "custom-key: keep-this\n", + "---\n", + "\n", + "See [blog](blog/index.qmd). The `.qmd` format is mentioned here.\n", + "\n", + "::: {.content-visible when-format=\"html\" style=\"text-align: center\"}\n", + "\n", + ":::\n", + "''')\n", + "(nbs/'blog/index.qmd').write_text(r'''---\n", + "title: Blog\n", + "subtitle: All posts\n", + "sidebar: false\n", + "listing:\n", + " categories: true\n", + "---\n", + "\n", + "Back to [guide](../guide.qmd).\n", + "''')" + ] + }, + { + "cell_type": "markdown", + "id": "866b1842", + "metadata": {}, + "source": [ + "Render scripts carry metadata in a Python docstring. Their program body and non-Markdown generated assets must remain unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "759a5b2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "30" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(nbs/'index.qmd.py').write_text(r'''\"\"\"---\n", + "title: Home\n", + "description: Write, test, document, and distribute software packages and technical articles — all in one place, your notebook.\n", + "page-layout: custom\n", + "toc: false\n", + "---\"\"\"\n", + "print(\"# Home\")\n", + "''')\n", + "(nbs/'blog/posts/render.qmd.py').write_text('\"\"\"---\\ntitle: Rendered\\ntoc-depth: 3\\ntoc_depth: 1\\n---\"\"\"\\nprint(\"# Rendered\")\\n')\n", + "(nbs/'images/circles.svg.py').write_text('\"\"\"---\\n---\"\"\"\\nprint(\"\")\\n')" + ] + }, + { + "cell_type": "markdown", + "id": "57b7b6dd", + "metadata": {}, + "source": [ + "This project includes both supported and unsupported navbar icons. Migration should report the unsupported entry without losing the supported links." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58b0450c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "371" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(nbs/'_quarto.yml').write_text(r'''website:\n", + " title: demo\n", + " favicon: favicon.png\n", + " navbar:\n", + " left:\n", + " - text: Guide\n", + " href: guide.qmd\n", + " right:\n", + " - icon: github\n", + " href: https://github.com/x/y\n", + " - icon: twitter\n", + " href: https://twitter.com/x\n", + " aria-label: Follow us\n", + " - icon: bug\n", + " href: https://x.y/issues\n", + "format:\n", + " html:\n", + " css: styles.css\n", + " toc-depth: 4\n", + "''')\n", + "(nbs/'styles.css').write_text('body { color: red; }')" + ] + }, + { + "cell_type": "markdown", + "id": "cb1a6d7c", + "metadata": {}, + "source": [ + "Notebook frontmatter and format-specific Markdown need the same conversion as standalone documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "add75676", + "metadata": {}, + "outputs": [], + "source": [ + "nb = new_nb([mk_cell(r'''---\n", + "summary: Notebook summary\n", + "categories: [tutorial]\n", + "toc-depth: 2\n", + "---\n", + "\n", + "[guide](guide.qmd) and [blog](blog/index.qmd)\n", + "\n", + "::: {.content-visible when-format=\"markdown\"}\n", + "![CI](badge.svg)\n", + ":::''', 'markdown')])\n", + "write_nb(nb, nbs/'index.ipynb')" + ] + }, + { + "cell_type": "markdown", + "id": "c7ab0d32", + "metadata": {}, + "source": [ + "Renaming includes nested documents and Markdown render scripts. Other generated asset types retain their filenames and content." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff857ee9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpcyu0zsjx/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n" + ] + } + ], + "source": [ + "quarto_before = (nbs/'_quarto.yml').read_bytes()\n", + "moves = _migrate_octavo(nbs)\n", + "test_eq({str(k.relative_to(nbs)):str(v.relative_to(nbs)) for k,v in moves.items()}, {\n", + " 'blog/index.qmd':'blog/index.md', 'blog/posts/render.qmd.py':'blog/posts/render.md.py',\n", + " 'guide.qmd':'guide.md', 'index.qmd.py':'index.md.py'})\n", + "assert not (nbs/'guide.qmd').exists() and (nbs/'guide.md').exists()\n", + "assert not (nbs/'blog/index.qmd').exists() and (nbs/'blog/index.md').exists()\n", + "assert not (nbs/'index.qmd.py').exists() and (nbs/'index.md.py').exists()\n", + "assert not (nbs/'blog/posts/render.qmd.py').exists() and (nbs/'blog/posts/render.md.py').exists()\n", + "test_eq((nbs/'images/circles.svg.py').read_text(), '\"\"\"---\\n---\"\"\"\\nprint(\"\")\\n')" + ] + }, + { + "cell_type": "markdown", + "id": "0a8eb0af", + "metadata": {}, + "source": [ + "Existing underscore keys take precedence in script metadata. `page-layout` is retained without creating `layout` or `page_layout`. Long descriptions must survive YAML serialization without inserted line continuations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2132c8f", + "metadata": {}, + "outputs": [], + "source": [ + "idx_txt = (nbs/'index.md.py').read_text()\n", + "idx_fm = yaml.safe_load(_script_fm.match(idx_txt).group('fm'))\n", + "test_eq(idx_fm['page-layout'], 'custom')\n", + "test_eq(idx_fm['toc_depth'], 0)\n", + "assert 'layout' not in idx_fm and 'page_layout' not in idx_fm and 'toc' not in idx_fm\n", + "assert 'technical\\\\\\n' not in idx_txt and '— all in one place' in idx_txt\n", + "render_txt = (nbs/'blog/posts/render.md.py').read_text()\n", + "render_fm = yaml.safe_load(_script_fm.match(render_txt).group('fm'))\n", + "test_eq(render_fm['toc_depth'], 1)\n", + "assert 'toc-depth' not in render_fm" + ] + }, + { + "cell_type": "markdown", + "id": "cacaa706", + "metadata": {}, + "source": [ + "The converted configuration retains the site identity and supported navigation. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b203a171", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq((nbs/'_quarto.yml').read_bytes(), quarto_before)\n", + "assert (nbs/'octavo.yml').exists()\n", + "cfg = yaml.safe_load((nbs/'octavo.yml').read_text())\n", + "test_eq(cfg['title'], 'demo')\n", + "assert 'output' not in cfg\n", + "test_eq(cfg['toc_depth'], 3)\n", + "assert 'css' not in cfg\n", + "test_eq((nbs/'styles.css').read_text(), 'body { color: red; }')\n", + "test_eq(cfg['favicon'], 'favicon.png')\n", + "test_eq(cfg['nav_links'], [{'text': 'Guide', 'href': 'guide.html'}])\n", + "test_eq(cfg['tools'], [{'icon': 'github', 'href': 'https://github.com/x/y', 'label': 'GitHub'},\n", + " {'icon': 'x', 'href': 'https://twitter.com/x', 'label': 'Follow us'}])" + ] + }, + { + "cell_type": "markdown", + "id": "de7fa745", + "metadata": {}, + "source": [ + "Frontmatter aliases change at the top level. Relative links still need to reach the renamed document from its own directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "958eb45c", + "metadata": {}, + "outputs": [], + "source": [ + "guide_fm,guide_body = frontmatter((nbs/'guide.md').read_text())\n", + "test_eq(guide_fm['description'], 'A handy guide')\n", + "assert 'subtitle' not in guide_fm\n", + "test_eq(guide_fm['toc_depth'], 2)\n", + "test_eq(guide_fm['custom_key'], 'keep-this')\n", + "assert 'toc-depth' not in guide_fm and 'custom-key' not in guide_fm\n", + "\n", + "blog_fm,blog_body = frontmatter((nbs/'blog/index.md').read_text())\n", + "test_eq(blog_fm['description'], 'All posts')\n", + "test_eq(blog_fm['sidebar'], False)\n", + "test_eq(_nav_item(dict(text='Help', menu=[dict(text='Guide', href='guide.qmd')])),\n", + " dict(text='Help', children=[dict(text='Guide', href='guide.html')]))\n", + "test_eq(blog_body.strip(), 'Back to [guide](../guide.md).')" + ] + }, + { + "cell_type": "markdown", + "id": "838adac0", + "metadata": {}, + "source": [ + "Notebook Markdown receives the same metadata conversion. A second migration must leave the converted metadata unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a823abd", + "metadata": {}, + "outputs": [], + "source": [ + "nb_fm,nb_body = frontmatter(read_nb(nbs/'index.ipynb').cells[0].source)\n", + "test_eq(nb_fm['description'], 'Notebook summary')\n", + "test_eq(nb_fm['tags'], ['tutorial'])\n", + "assert 'summary' not in nb_fm and 'categories' not in nb_fm\n", + "test_eq(nb_fm['toc_depth'], 2)\n", + "assert 'toc-depth' not in nb_fm\n", + "\n", + "test_eq(_migrate_octavo(nbs), {})\n", + "test_eq(frontmatter((nbs/'guide.md').read_text())[0], guide_fm)\n", + "migration_tmp.cleanup()" + ] + }, + { + "cell_type": "markdown", + "id": "74be3db5", + "metadata": {}, + "source": [ + "A destination collision must stop migration before an existing file can be overwritten." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84cacc34", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " p = Path(d)\n", + " (p/'page.qmd.py').touch(); (p/'page.md.py').touch()\n", + " with expect_fail(FileExistsError, contains='page.md.py'): _migrate_octavo(p)" + ] + }, + { + "cell_type": "markdown", + "id": "1540e82b", + "metadata": {}, + "source": [ + "Malformed YAML must fail before any source rename. This differs from a valid YAML value of the wrong type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26b3cfd1", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " p = Path(d)\n", + " (p/'guide.qmd').write_text('# Guide')\n", + " (p/'_quarto.yml').write_text('website: [invalid')\n", + " with expect_fail(yaml.YAMLError): _migrate_octavo(p)\n", + " assert (p/'guide.qmd').exists() and not (p/'guide.md').exists()" + ] + }, + { + "cell_type": "markdown", + "id": "09076322", + "metadata": {}, + "source": [ + "This project keeps notebooks in `docs/notebooks` and sets `doc_path` to `public`. Migration omits redundant output settings; the build still writes to the project’s `public` directory. A different Quarto destination is preserved." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d27559e2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "7" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nested_tmp = tempfile.TemporaryDirectory()\n", + "d = nested_tmp.name\n", + "project = Path(d)\n", + "nbs = project/'docs/notebooks'\n", + "nbs.mkdir(parents=True)\n", + "(project/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"docs/notebooks\"\\ndoc_path = \"public\"\\n')\n", + "(nbs/'_quarto.yml').write_text('website:\\n title: Demo\\nformat:\\n html:\\n toc: false\\n')\n", + "(nbs/'guide.qmd').write_text('# Guide')" + ] + }, + { + "cell_type": "markdown", + "id": "e4a5d888", + "metadata": {}, + "source": [ + "A project with nested notebooks should still publish to its configured project-level output directory. The installed CLI must find that directory without a `--path` override.\n", + "\n", + "Migration does not invent footer text. A project must supply its own footer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07930509", + "metadata": {}, + "outputs": [], + "source": [ + "with working_directory(project): run('nbdev-migrate-octavo')\n", + "settings = octavo_config(nbs)\n", + "assert 'output' not in settings\n", + "test_eq(settings['toc_depth'], 0)\n", + "assert 'footer' not in settings\n", + "assert (nbs/'guide.md').exists() and (nbs/'_quarto.yml').exists()\n", + "with working_directory(project): run('nbdev-docs')\n", + "assert (project/'public/guide.html').exists()\n", + "\n", + "test_eq(_octavo_output(nbs, {'project': {'output-dir': 'public'}}, {}), None)\n", + "test_eq(_octavo_output(nbs, {'project': {'output-dir': 'website'}}, {}), '../../website')\n", + "test_eq(_octavo_output(nbs, {'project': {'output-dir': 'website'}}, {'project': {'output-dir': 'public'}}), None)\n", + "test_eq(_octavo_output(nbs, {'project': {'output-dir': str(project/'elsewhere')}}, {}), '../../elsewhere')" + ] + }, + { + "cell_type": "markdown", + "id": "41760705", + "metadata": {}, + "source": [ + "An existing Octavo configuration belongs to the author. Repeating migration must preserve it rather than regenerate defaults." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a656bf", + "metadata": {}, + "outputs": [], + "source": [ + "before = (nbs/'octavo.yml').read_text()\n", + "test_eq(_migrate_octavo(nbs), {})\n", + "test_eq((nbs/'octavo.yml').read_text(), before)\n", + "nested_tmp.cleanup()" + ] + }, + { + "cell_type": "markdown", + "id": "58de7d96", + "metadata": {}, + "source": [ + "Valid YAML is not always valid configuration. Lists, booleans, and numbers must fail before source changes, whether they came from Quarto or an existing Octavo configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "215e4eeb", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as d:\n", + " nbs = Path(d)\n", + " (nbs/'guide.qmd').write_text('# Guide')\n", + " (nbs/'_quarto.yml').write_text('website:\\n title: Demo\\n')\n", + " for invalid in ('[]', 'false', '0'):\n", + " (nbs/'octavo.yml').write_text(invalid)\n", + " with expect_fail(ValueError, contains='mapping'): _migrate_octavo(nbs)\n", + " assert (nbs/'guide.qmd').exists() and (nbs/'_quarto.yml').exists()\n", + " (nbs/'octavo.yml').unlink()\n", + " for invalid in ('[]', 'false', '0'):\n", + " (nbs/'_quarto.yml').write_text(invalid)\n", + " with expect_fail(ValueError, contains='mapping'): _migrate_octavo(nbs)\n", + " assert (nbs/'guide.qmd').exists() and not (nbs/'guide.md').exists()" ] } ], diff --git a/pyproject.toml b/pyproject.toml index de8d02872..c6b09cc42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ nbdev-test = "nbdev.test:nbdev_test" nbdev-new = "nbdev.cli:nbdev_new" nbdev-migrate = "nbdev.migrate:nbdev_migrate" nbdev-migrate-config = "nbdev.migrate:nbdev_migrate_config" +nbdev-migrate-octavo = "nbdev.octavo:nbdev_migrate_octavo" nbdev-install-quarto = "nbdev.quarto:install_quarto" nbdev-install = "nbdev.quarto:install" nbdev-docs = "nbdev.quarto:nbdev_docs" From e18a116eedc82fc65cb71081e6f6119705a7c693 Mon Sep 17 00:00:00 2001 From: Rens Date: Wed, 7 Oct 2026 07:39:24 +0200 Subject: [PATCH 03/16] add octavo preview --- nbdev/_modidx.py | 1 + nbdev/octavo.py | 60 ++++++++++++++++++++++++++--- nbdev/quarto.py | 7 +++- nbs/api/14_quarto.ipynb | 15 ++++++-- nbs/api/21_octavo.ipynb | 84 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 155 insertions(+), 12 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index a7dc6ae20..fedafa057 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -238,6 +238,7 @@ 'nbdev.octavo.nbdev_migrate_octavo': ('api/octavo.html#nbdev_migrate_octavo', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_config': ('api/octavo.html#octavo_config', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_docs': ('api/octavo.html#octavo_docs', 'nbdev/octavo.py'), + 'nbdev.octavo.octavo_preview': ('api/octavo.html#octavo_preview', 'nbdev/octavo.py'), 'nbdev.octavo.octavo_readme': ('api/octavo.html#octavo_readme', 'nbdev/octavo.py'), 'nbdev.octavo.render_script': ('api/octavo.html#render_script', 'nbdev/octavo.py'), 'nbdev.octavo.repl_content_visible': ('api/octavo.html#repl_content_visible', 'nbdev/octavo.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index f824faaee..e2ffa37bd 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -5,16 +5,17 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. # %% auto #0 -__all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_readme', - 'repl_qmd_links', 'repl_content_visible', 'repl_frontmatter_keys', 'repl_script_frontmatter_keys', - 'migrate_nb_cells', 'migrate_md_file', 'nbdev_migrate_octavo'] +__all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_preview', + 'octavo_readme', 'repl_qmd_links', 'repl_content_visible', 'repl_frontmatter_keys', + 'repl_script_frontmatter_keys', 'migrate_nb_cells', 'migrate_md_file', 'nbdev_migrate_octavo'] # %% ../nbs/api/21_octavo.ipynb #7b098bb5 from contextlib import redirect_stdout from io import StringIO from fastcore.utils import * from fastcore.script import call_parse -import yaml +import yaml, shutil, subprocess, threading, webbrowser +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from octavo import * from mdhtml import md2gfm from .config import get_config, import_obj, _load_toml, read_nb @@ -23,7 +24,7 @@ from .processors import FilterDefaults, add_links from .serve import _is_qpy from .sync import write_nb -from .quarto import _doc_mtime_not_older +from .quarto import _doc_mtime_not_older, fs_watchdog # %% ../nbs/api/21_octavo.ipynb #1cc66460 class OctavoProc(FilterDefaults): @@ -92,6 +93,55 @@ def octavo_docs( if not out.is_absolute(): out = cfg.nbs_path/out import_obj(post)(out.resolve(), cfg) +# %% ../nbs/api/21_octavo.ipynb #db88c5c9 +def octavo_preview( + path:str=None, # Path to the project or its notebooks + port:int=None, # HTTP port (defaults to 8000) + host:str=None, # Bind address (defaults to localhost) + no_browser:bool=False, # Do not open a browser +): + "Serve documentation and rebuild when project files change" + cfg = get_config(path) + root = cfg.config_path.resolve() + settings = octavo_config(cfg.nbs_path) + out = (cfg.nbs_path/Path(settings.get('output', cfg.doc_path)).expanduser()).resolve() + if root.is_relative_to(out) or cfg.nbs_path.resolve().is_relative_to(out): raise ValueError('Output must not contain sources') + changed = threading.Event() + + def rebuild(): + try: subprocess.run(['nbdev-docs', '--path', str(root)], cwd=root, check=True) + except subprocess.CalledProcessError: + shutil.rmtree(out, ignore_errors=True) + out.mkdir(parents=True, exist_ok=True) + (out/'index.html').write_text('

Build failed - see terminal

') + print('Build failed; waiting for another edit.', flush=True) + + def on_change(event): + if event.event_type not in ('created', 'modified', 'deleted', 'moved'): return + if event.is_directory and event.event_type=='modified': return + for name in (event.src_path, getattr(event, 'dest_path', '')): + if not name: continue + src = Path(name).resolve() + if src.is_relative_to(out) or not src.is_relative_to(root): continue + if any(p.startswith('.') or p=='__pycache__' for p in src.relative_to(root).parts): continue + changed.set() + + handler = partial(SimpleHTTPRequestHandler, directory=str(out)) + with ThreadingHTTPServer((host or 'localhost', 8000 if port is None else port), handler) as server: + url = f'http://{host or "localhost"}:{server.server_port}' + with fs_watchdog(on_change, root): + rebuild() + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + print(f'{url} - refresh your browser after rebuilding', flush=True) + if not no_browser: webbrowser.open(url) + while True: + changed.wait() + changed.clear() + while changed.wait(0.2): changed.clear() + rebuild() + finally: server.shutdown() + # %% ../nbs/api/21_octavo.ipynb #0d38402b def octavo_readme( path:str=None, # Path to the project or its notebooks diff --git a/nbdev/quarto.py b/nbdev/quarto.py index a34f5a980..6200d8473 100644 --- a/nbdev/quarto.py +++ b/nbdev/quarto.py @@ -76,7 +76,7 @@ def _sort(a): _def_file_re = r'\.(?:ipynb|qmd|html)$' -@delegates(nbglob) +@delegates def _nbglob_docs( path:str=None, # Path to notebooks file_glob:str=None, # Only include files matching glob @@ -400,6 +400,11 @@ def nbdev_preview( n_workers:int=defaults.cpus, # Preprocessing and Quarto workers (0 or 1: serial) **kwargs): "Preview docs locally" + if backend:=_octavo(path): + params = signature(_nbglob_docs).parameters + unsupported = [k for k,v in kwargs.items() if k not in params or v != params[k].default] + if unsupported: raise ValueError(f'Octavo does not support these Quarto flags: {", ".join(unsupported)}') + return backend.octavo_preview(path, port=port, host=host, no_browser=no_browser) os.environ['QUARTO_PREVIEW']='1' cache,cfg,path = _pre_docs(path, n_workers=n_workers, **kwargs) site = cache/cfg.doc_path.name diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb index 96e71aa12..fc55bf98d 100644 --- a/nbs/api/14_quarto.ipynb +++ b/nbs/api/14_quarto.ipynb @@ -163,7 +163,7 @@ "\n", "_def_file_re = r'\\.(?:ipynb|qmd|html)$'\n", "\n", - "@delegates(nbglob)\n", + "@delegates\n", "def _nbglob_docs(\n", " path:str=None, # Path to notebooks\n", " file_glob:str=None, # Only include files matching glob \n", @@ -755,7 +755,7 @@ "source": [ "## Preview\n", "\n", - "`nbdev-preview` keeps its own copy of the site in the `_proc` cache, rendering new or changed pages on startup and as you edit; it doesn't change `doc_path`. When the cache has no site it starts from a copy of the one in `doc_path`, so unchanged pages from the last `nbdev-docs` run aren't rendered again. With neither, it builds all pages in parallel using the same `--n-workers` setting as `nbdev-docs`. Use `nbdev-preview --render` to rebuild everything, including changes to shared configuration or filters." + "For Quarto projects, `nbdev-preview` keeps its own copy of the site in the `_proc` cache, rendering new or changed pages on startup and as you edit; it doesn't change `doc_path`. When the cache has no site it starts from a copy of the one in `doc_path`, so unchanged pages from the last `nbdev-docs` run aren't rendered again. With neither, it builds all pages in parallel using the same `--n-workers` setting as `nbdev-docs`. Use `nbdev-preview --render` to rebuild everything, including changes to shared configuration or filters." ] }, { @@ -801,6 +801,11 @@ " n_workers:int=defaults.cpus, # Preprocessing and Quarto workers (0 or 1: serial)\n", " **kwargs):\n", " \"Preview docs locally\"\n", + " if backend:=_octavo(path):\n", + " params = signature(_nbglob_docs).parameters\n", + " unsupported = [k for k,v in kwargs.items() if k not in params or v != params[k].default]\n", + " if unsupported: raise ValueError(f'Octavo does not support these Quarto flags: {\", \".join(unsupported)}')\n", + " return backend.octavo_preview(path, port=port, host=host, no_browser=no_browser)\n", " os.environ['QUARTO_PREVIEW']='1'\n", " cache,cfg,path = _pre_docs(path, n_workers=n_workers, **kwargs)\n", " site = cache/cfg.doc_path.name\n", @@ -852,7 +857,11 @@ ] } ], - "metadata": {}, + "metadata": { + "solveit": { + "use_thinking": false + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index bd1e18e17..7c4cb450d 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -32,7 +32,7 @@ "\n", "Run `nbdev-export` before building to update the generated Python modules and symbol indexes. `nbdev-readme` and `nbdev-contributing` also use Octavo for these projects. `nbdev-prepare` still exports, tests, and cleans notebooks, but skips Quarto configuration generation.\n", "\n", - "For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Octavo support for `nbdev-preview` is not yet implemented." + "For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Use `nbdev-preview` to serve the site and rebuild after edits. Refresh the browser manually." ] }, { @@ -85,7 +85,8 @@ "from io import StringIO\n", "from fastcore.utils import *\n", "from fastcore.script import call_parse\n", - "import yaml\n", + "import yaml, shutil, subprocess, threading, webbrowser\n", + "from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer\n", "from octavo import *\n", "from mdhtml import md2gfm\n", "from nbdev.config import get_config, import_obj, _load_toml, read_nb\n", @@ -94,7 +95,7 @@ "from nbdev.processors import FilterDefaults, add_links\n", "from nbdev.serve import _is_qpy\n", "from nbdev.sync import write_nb\n", - "from nbdev.quarto import _doc_mtime_not_older" + "from nbdev.quarto import _doc_mtime_not_older, fs_watchdog" ] }, { @@ -522,6 +523,83 @@ "build_tmp.cleanup()" ] }, + { + "cell_type": "markdown", + "id": "5c1356ac", + "metadata": {}, + "source": [ + "## Local preview" + ] + }, + { + "cell_type": "markdown", + "id": "f568f1c6", + "metadata": {}, + "source": [ + "`nbdev-preview` builds and serves an Octavo site at `http://localhost:8000`. Use `--port`, `--host`, and `--no-browser` to change the server options. It watches the project for edits and rebuilds the whole site in a fresh process. Refresh the browser manually after each build.\n", + "\n", + "Preview writes directly to the configured documentation output, unlike Quarto’s separate preview cache. Pages can be unavailable during a rebuild. A failed build clears partial output and shows an error page; the server keeps watching so the next successful build restores the site. Errors appear in the terminal.\n", + "\n", + "Hidden paths, Python bytecode caches, and the output directory are ignored. Restart preview after changing the notebooks or output directory. Octavo always builds on startup; `--render` is redundant and `--n-workers` remains Quarto-only." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db88c5c9", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def octavo_preview(\n", + " path:str=None, # Path to the project or its notebooks\n", + " port:int=None, # HTTP port (defaults to 8000)\n", + " host:str=None, # Bind address (defaults to localhost)\n", + " no_browser:bool=False, # Do not open a browser\n", + "):\n", + " \"Serve documentation and rebuild when project files change\"\n", + " cfg = get_config(path)\n", + " root = cfg.config_path.resolve()\n", + " settings = octavo_config(cfg.nbs_path)\n", + " out = (cfg.nbs_path/Path(settings.get('output', cfg.doc_path)).expanduser()).resolve()\n", + " if root.is_relative_to(out) or cfg.nbs_path.resolve().is_relative_to(out): raise ValueError('Output must not contain sources')\n", + " changed = threading.Event()\n", + "\n", + " def rebuild():\n", + " try: subprocess.run(['nbdev-docs', '--path', str(root)], cwd=root, check=True)\n", + " except subprocess.CalledProcessError:\n", + " shutil.rmtree(out, ignore_errors=True)\n", + " out.mkdir(parents=True, exist_ok=True)\n", + " (out/'index.html').write_text('

Build failed - see terminal

')\n", + " print('Build failed; waiting for another edit.', flush=True)\n", + "\n", + " def on_change(event):\n", + " if event.event_type not in ('created', 'modified', 'deleted', 'moved'): return\n", + " if event.is_directory and event.event_type=='modified': return\n", + " for name in (event.src_path, getattr(event, 'dest_path', '')):\n", + " if not name: continue\n", + " src = Path(name).resolve()\n", + " if src.is_relative_to(out) or not src.is_relative_to(root): continue\n", + " if any(p.startswith('.') or p=='__pycache__' for p in src.relative_to(root).parts): continue\n", + " changed.set()\n", + "\n", + " handler = partial(SimpleHTTPRequestHandler, directory=str(out))\n", + " with ThreadingHTTPServer((host or 'localhost', 8000 if port is None else port), handler) as server:\n", + " url = f'http://{host or \"localhost\"}:{server.server_port}'\n", + " with fs_watchdog(on_change, root):\n", + " rebuild()\n", + " threading.Thread(target=server.serve_forever, daemon=True).start()\n", + " try:\n", + " print(f'{url} - refresh your browser after rebuilding', flush=True)\n", + " if not no_browser: webbrowser.open(url)\n", + " while True:\n", + " changed.wait()\n", + " changed.clear()\n", + " while changed.wait(0.2): changed.clear()\n", + " rebuild()\n", + " finally: server.shutdown()" + ] + }, { "cell_type": "markdown", "id": "b53bd453", From 9e578447e04151e6e86fd1dc46872dd0bac90c58 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 8 Oct 2026 06:23:16 +0200 Subject: [PATCH 04/16] fix _nglob_docs integration (is a PR to main too) --- nbdev/quarto.py | 2 +- nbs/api/14_quarto.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nbdev/quarto.py b/nbdev/quarto.py index 6200d8473..e6b0a36fa 100644 --- a/nbdev/quarto.py +++ b/nbdev/quarto.py @@ -76,7 +76,7 @@ def _sort(a): _def_file_re = r'\.(?:ipynb|qmd|html)$' -@delegates +@delegates(nbglob_cli) def _nbglob_docs( path:str=None, # Path to notebooks file_glob:str=None, # Only include files matching glob diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb index fc55bf98d..15a5384d7 100644 --- a/nbs/api/14_quarto.ipynb +++ b/nbs/api/14_quarto.ipynb @@ -163,7 +163,7 @@ "\n", "_def_file_re = r'\\.(?:ipynb|qmd|html)$'\n", "\n", - "@delegates\n", + "@delegates(nbglob_cli)\n", "def _nbglob_docs(\n", " path:str=None, # Path to notebooks\n", " file_glob:str=None, # Only include files matching glob \n", From 174e6dcef75e87eacdf675d009f43c4ac027b5dc Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 8 Oct 2026 13:18:47 +0200 Subject: [PATCH 05/16] fix octavo minver --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6b09cc42..f5fba4bed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,8 @@ dependencies = [ "fastcore>=2.2.28", "execnb>=0.3.3", "astunparse", "ghapi>=2.0. "setuptools", "build", "fastgit>=0.0.7", "pyyaml", "tomli; python_version < '3.11'", ] [project.optional-dependencies] -octavo = ["octavo>=0.0.1"] -dev = [ "ipywidgets", "nbdev-numpy", "nbdev-stdlib", "pandas", "matplotlib", "black", "svg.py", "pysym2md>=0.0.6", "llms-txt", "sphinx", "plum-dispatch", "nbclient", "ipykernel", "octavo>=0.0.1", ] +octavo = ["aai-octavo>=0.0.2"] +dev = [ "ipywidgets", "nbdev-numpy", "nbdev-stdlib", "pandas", "matplotlib", "black", "svg.py", "pysym2md>=0.0.6", "llms-txt", "sphinx", "plum-dispatch", "nbclient", "ipykernel", "aai-octavo>=0.0.2", ] [project.urls] Repository = "https://github.com/AnswerDotAI/nbdev" From e8cc5ff82169e75143ff45d9c3882729fe947248 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 8 Oct 2026 14:30:39 +0200 Subject: [PATCH 06/16] update octavo migration prompt --- nbdev/__init__.py | 1 + nbdev/octavo.py | 38 +++++++++++++++++++++++++++ nbs/api/21_octavo.ipynb | 57 ++++++++++++++++++----------------------- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/nbdev/__init__.py b/nbdev/__init__.py index f244760cc..0af6c802d 100644 --- a/nbdev/__init__.py +++ b/nbdev/__init__.py @@ -4,6 +4,7 @@ - `nbdev.extract_attachments`: A preprocessor that extracts all of the attachments from the notebook file. The extracted attachments are returned in the 'resources' dictionary. - `nbdev.moddocs`: # Module docs: creating them from existing notebooks +- `nbdev.octavo`: Build nbdev documentation with Octavo - `nbdev.skill`: Write, read and review nbdev notebooks as source, documentation, examples and tests. You MUST read this before any work on a project with notebooks.""" __version__ = "3.3.26" diff --git a/nbdev/octavo.py b/nbdev/octavo.py index e2ffa37bd..776f29ec1 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -1,5 +1,43 @@ """Build nbdev documentation with Octavo +Octavo is an alternative to Quarto for building a documentation website from nbdev notebooks. Install `nbdev[octavo]` to use it. A project selects Octavo by placing `octavo.yml` in its configured notebooks directory. Projects without that file continue to use Quarto and do not import Octavo. + +The usual `nbdev-docs` command builds the site. Octavo currently builds all pages together. File-selection options are rejected unless left at their defaults; `n_workers` affects only Quarto builds. + +Run `nbdev-export` before building to update the generated Python modules and symbol indexes. `nbdev-readme` and `nbdev-contributing` also use Octavo for these projects. `nbdev-prepare` still exports, tests, and cleans notebooks, but skips Quarto configuration generation. + +For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Use `nbdev-preview` to serve the site and rebuild after edits. Refresh the browser manually. + +## Migration instructions (share this with your AI) + +Before migrating, save a built reference site. Octavo builds replace the output directory. Retaining `_quarto.yml` does not retain the rendered Quarto site or keep converted sources compatible with Quarto. + +1. Ensure you have committed your latest changes or backed up your project +2. From the project root, run `nbdev-docs` while the project still uses Quarto. Confirm the build succeeds and inspect the site. Record existing failures rather than treating them as migration regressions. +3. Copy the complete generated site, including assets and search files, to a separate baseline directory outside the source and build directories. +4. Run `nbdev-migrate-octavo` from the project root. Review its changes and any executable scripts or hooks before attempting another build. +5. Run `nbdev-docs` again to build with Octavo. Keep the baseline untouched. If the build fails, retain its output for diagnosis. Use the manual migration tips below with the baseline path, new output path, and any errors. + +Read the retained _quarto.yml, nbdev.yml, pyproject.toml, and generated octavo.yml. Inspect page metadata, stylesheets, render scripts, custom HTML, build hooks, and deployment workflows. Automatic migration renames sources and updates selected document references, directives, metadata, and site settings. It does not port the site's design. It does not copy format.html.css into octavo.yml. It leaves stylesheet files and page-layout metadata unchanged. Existing page-level CSS references can still load; inspect those too. An existing octavo.yml is preserved rather than regenerated. + +Identify which checks apply and propose a small migration plan. Inspect optional features only where the project uses them, such as custom homepages, blog listings, embeds, and tabs: + +- Styles and custom pages: inspect the rendered Octavo HTML before adapting selectors. Check responsive behavior and dependencies on Bootstrap or Quarto classes. Reuse compatible CSS selectively. Do not enable an old stylesheet wholesale or copy a compatibility layer without checking the current DOM. +- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as any custom layouts. Compare toc and toc_depth settings with the original metadata, including explicit depth combined with toc: false. +- Embedded and conditional content: review the project's custom markup and format-conditional blocks. Compare when-format and unless-format behavior with the original sources in both HTML and Markdown output. Do not assume automatic conversion preserves exclusion conditions or renders markdown inside an HTML-targeted block. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the result rather than rewriting every block by pattern. +- Render scripts and processors: if the project uses these, check their generated content and assets after migration. +- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. +- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes. +- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. +- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions. +- Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow. + +After approval, make the agreed changes in small steps. Export notebook-authored modules before building. Run nbdev-docs and the relevant repository Markdown commands. Compare the resulting HTML, Markdown, assets, and additional build artifacts with the saved baseline. Check representative API signatures, symbol links, and GitHub source links. Separate pre-existing failures and expected generated differences from migration regressions. + +Serve the saved baseline and new output separately. Confirm that each server points to the intended directory. Inspect representative pages at desktop and narrow widths. Test navigation, mobile navigation, anchors, images, and the optional features identified above. Test search and follow result links from both the homepage and a nested page. Exercise the intended hosting subpath, such as /project/, rather than testing only at /. A successful build or static link check does not establish that browser interactions work. + +Report the changes, checks actually run, unresolved differences, omitted settings, and decisions requiring my review. Keep the original configuration and baseline until the migration has been reviewed. + Docs: https://nbdev.fast.ai/api/octavo.html.md""" # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index 7c4cb450d..6088ca905 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -26,51 +26,44 @@ "id": "56ff9208", "metadata": {}, "source": [ + "#| export \n", "Octavo is an alternative to Quarto for building a documentation website from nbdev notebooks. Install `nbdev[octavo]` to use it. A project selects Octavo by placing `octavo.yml` in its configured notebooks directory. Projects without that file continue to use Quarto and do not import Octavo.\n", "\n", "The usual `nbdev-docs` command builds the site. Octavo currently builds all pages together. File-selection options are rejected unless left at their defaults; `n_workers` affects only Quarto builds.\n", "\n", "Run `nbdev-export` before building to update the generated Python modules and symbol indexes. `nbdev-readme` and `nbdev-contributing` also use Octavo for these projects. `nbdev-prepare` still exports, tests, and cleans notebooks, but skips Quarto configuration generation.\n", "\n", - "For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Use `nbdev-preview` to serve the site and rebuild after edits. Refresh the browser manually." - ] - }, - { - "cell_type": "markdown", - "id": "7b66c605", - "metadata": {}, - "source": [ - "## Migration finishing prompt" - ] - }, - { - "cell_type": "markdown", - "id": "dc8e677d", - "metadata": {}, - "source": [ - "After running `nbdev-migrate-octavo`, use this prompt with an assistant that can inspect your project. `_quarto.yml` remains available for comparison. Migration changes sources in place; it does not keep them compatible with both renderers.\n", + "For an existing Quarto project, `nbdev-migrate-octavo` converts documentation sources in place. The converted sources are not intended to work unchanged with both builders. Use `nbdev-preview` to serve the site and rebuild after edits. Refresh the browser manually.\n", + "\n", + "## Migration instructions (share this with your AI)\n", + "\n", + "Before migrating, save a built reference site. Octavo builds replace the output directory. Retaining `_quarto.yml` does not retain the rendered Quarto site or keep converted sources compatible with Quarto.\n", "\n", - "```markdown\n", - "Help finish this project's migration from Quarto to Octavo. Inspect the project before proposing changes. Agree on the changes with me before editing. Preserve unrelated work. Do not reset a checkout, clean untracked files, commit, or deploy.\n", + "1. Ensure you have committed your latest changes or backed up your project \n", + "2. From the project root, run `nbdev-docs` while the project still uses Quarto. Confirm the build succeeds and inspect the site. Record existing failures rather than treating them as migration regressions.\n", + "3. Copy the complete generated site, including assets and search files, to a separate baseline directory outside the source and build directories.\n", + "4. Run `nbdev-migrate-octavo` from the project root. Review its changes and any executable scripts or hooks before attempting another build.\n", + "5. Run `nbdev-docs` again to build with Octavo. Keep the baseline untouched. If the build fails, retain its output for diagnosis. Use the manual migration tips below with the baseline path, new output path, and any errors.\n", "\n", - "Read the retained _quarto.yml, nbdev.yml, pyproject.toml, and generated octavo.yml. Inspect page metadata, stylesheets, render scripts, custom HTML, build hooks, and deployment workflows. Compare against the original site or screenshots where available. If no visual reference is available, say so rather than claiming visual parity.\n", + "Read the retained _quarto.yml, nbdev.yml, pyproject.toml, and generated octavo.yml. Inspect page metadata, stylesheets, render scripts, custom HTML, build hooks, and deployment workflows. Automatic migration renames sources and updates selected document references, directives, metadata, and site settings. It does not port the site's design. It does not copy format.html.css into octavo.yml. It leaves stylesheet files and page-layout metadata unchanged. Existing page-level CSS references can still load; inspect those too. An existing octavo.yml is preserved rather than regenerated.\n", "\n", - "Automatic migration renames sources and updates selected document references, directives, metadata, and site settings. It does not port the site's design. It does not copy format.html.css into octavo.yml. It leaves stylesheet files and page-layout metadata unchanged. Existing page-level CSS references can still load; inspect those too. An existing octavo.yml is preserved rather than regenerated.\n", + "Identify which checks apply and propose a small migration plan. Inspect optional features only where the project uses them, such as custom homepages, blog listings, embeds, and tabs:\n", "\n", - "Identify which of these checks apply, then propose a small migration plan:\n", + "- Styles and custom pages: inspect the rendered Octavo HTML before adapting selectors. Check responsive behavior and dependencies on Bootstrap or Quarto classes. Reuse compatible CSS selectively. Do not enable an old stylesheet wholesale or copy a compatibility layer without checking the current DOM.\n", + "- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as any custom layouts. Compare toc and toc_depth settings with the original metadata, including explicit depth combined with toc: false.\n", + "- Embedded and conditional content: review the project's custom markup and format-conditional blocks. Compare when-format and unless-format behavior with the original sources in both HTML and Markdown output. Do not assume automatic conversion preserves exclusion conditions or renders markdown inside an HTML-targeted block. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the result rather than rewriting every block by pattern.\n", + "- Render scripts and processors: if the project uses these, check their generated content and assets after migration.\n", + "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer.\n", + "- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes.\n", + "- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior.\n", + "- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions.\n", + "- Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow.\n", "\n", - "- Styles and custom pages: inspect the rendered Octavo HTML before adapting selectors. Check homepage grids, buttons, spacing, responsive behavior, and dependencies on Bootstrap or Quarto classes. Reuse compatible CSS selectively. Do not enable an old stylesheet wholesale or copy a compatibility layer without checking the current DOM.\n", - "- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as landing pages and blog listings.\n", - "- Embedded content: inspect iframes, videos, raw HTML, fenced divs, callouts, tabs, shortcodes, and format-conditional blocks used by this project. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the rendered result and relevant Markdown outputs rather than rewriting every block by pattern.\n", - "- Render scripts and documentation: inspect renamed .md.py scripts and generated assets. Update tutorials and examples that still describe Quarto-specific commands, file extensions, or intermediate files. Source renaming alone does not make those explanations correct. Review executable scripts and custom processors before running a build.\n", - "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and the project's actual footer where needed. Do not infer license text from a project name.\n", - "- Additional build outputs: identify artifacts previously generated by Quarto hooks or custom scripts. If the project needs API lists or LLM context files, adapt its own generator using nbdev's docs_post_build callback. Do not add those artifacts to projects that do not need them. Check llms.txt destinations against actual output files, including old .html.md links and malformed Markdown links.\n", - "- Deployment: workflows are not migrated. Check documentation dependencies, replace direct Quarto build commands where needed, and verify the published directory. Octavo's output is relative to the notebooks directory. Use the ordinary nbdev-docs command. Do not copy a test harness's import-path workarounds or obsolete builder calls into production.\n", + "After approval, make the agreed changes in small steps. Export notebook-authored modules before building. Run nbdev-docs and the relevant repository Markdown commands. Compare the resulting HTML, Markdown, assets, and additional build artifacts with the saved baseline. Check representative API signatures, symbol links, and GitHub source links. Separate pre-existing failures and expected generated differences from migration regressions.\n", "\n", - "After approval, make the agreed changes in small steps. Export notebook-authored modules before building. Run nbdev-docs and the relevant repository Markdown commands. Inspect representative pages at desktop and narrow widths. Check navigation, anchors, images, embedded content, generated assets, and any additional build artifacts. A successful build alone does not establish a successful migration.\n", + "Serve the saved baseline and new output separately. Confirm that each server points to the intended directory. Inspect representative pages at desktop and narrow widths. Test navigation, mobile navigation, anchors, images, and the optional features identified above. Test search and follow result links from both the homepage and a nested page. Exercise the intended hosting subpath, such as /project/, rather than testing only at /. A successful build or static link check does not establish that browser interactions work.\n", "\n", - "Report the changes, checks actually run, unresolved differences, and decisions requiring my review. Keep the original configuration until the migration has been reviewed. Do not copy test-only galleries or fixtures into the site unless I request them.\n", - "```" + "Report the changes, checks actually run, unresolved differences, omitted settings, and decisions requiring my review. Keep the original configuration and baseline until the migration has been reviewed." ] }, { From 00cc143eae93ca425dba6485688b7aa282a7b627 Mon Sep 17 00:00:00 2001 From: Rens Date: Fri, 9 Oct 2026 11:19:03 +0200 Subject: [PATCH 07/16] rewrite .html.md -> .md on octavo migration --- nbdev/_modidx.py | 1 + nbdev/octavo.py | 13 +++++++++---- nbs/api/21_octavo.ipynb | 27 ++++++++++++++++++--------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index fedafa057..6cda28bd2 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -243,6 +243,7 @@ 'nbdev.octavo.render_script': ('api/octavo.html#render_script', 'nbdev/octavo.py'), 'nbdev.octavo.repl_content_visible': ('api/octavo.html#repl_content_visible', 'nbdev/octavo.py'), 'nbdev.octavo.repl_frontmatter_keys': ('api/octavo.html#repl_frontmatter_keys', 'nbdev/octavo.py'), + 'nbdev.octavo.repl_html_md_links': ('api/octavo.html#repl_html_md_links', 'nbdev/octavo.py'), 'nbdev.octavo.repl_qmd_links': ('api/octavo.html#repl_qmd_links', 'nbdev/octavo.py'), 'nbdev.octavo.repl_script_frontmatter_keys': ( 'api/octavo.html#repl_script_frontmatter_keys', 'nbdev/octavo.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 776f29ec1..b166875d1 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -44,7 +44,7 @@ # %% auto #0 __all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_preview', - 'octavo_readme', 'repl_qmd_links', 'repl_content_visible', 'repl_frontmatter_keys', + 'octavo_readme', 'repl_qmd_links', 'repl_html_md_links', 'repl_content_visible', 'repl_frontmatter_keys', 'repl_script_frontmatter_keys', 'migrate_nb_cells', 'migrate_md_file', 'nbdev_migrate_octavo'] # %% ../nbs/api/21_octavo.ipynb #7b098bb5 @@ -227,6 +227,10 @@ def repl_qmd_links(txt, moves, src, root): for old,new in _qmd_repls(src, root, moves): txt = txt.replace(old, new) return txt +def repl_html_md_links(txt): + "Rewrite Quarto `.html.md` Markdown-copy targets to Octavo's `.md`" + return re.sub(r'\.html\.md\b', '.md', txt) + # %% ../nbs/api/21_octavo.ipynb #237aea2c _re_cv = re.compile(r'^:{3,}[^\S\r\n]*\{([^}]*\.content-visible[^}]*)\}[^\S\r\n]*\n(.*?)\n:{3,}[^\S\r\n]*$', re.MULTILINE | re.DOTALL) @@ -290,7 +294,7 @@ def migrate_nb_cells(path, procs): if changed: write_nb(nb, path) def migrate_md_file(path, procs): - "Apply `procs` to a Markdown/QMD file in-place" + "Apply `procs` to a Markdown, QMD or text file in-place" txt = path.read_text() new_txt = _apply_procs(txt, procs) if new_txt!=txt: path.write_text(new_txt) @@ -384,12 +388,13 @@ def _migrate_octavo(path): elif not qy.is_file(): raise FileNotFoundError(qy) cfg = _create_octavo_cfg(path) if qy.exists() and not (path/'octavo.yml').exists() else None - srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md') + srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md','.txt') and not any(o[0] in '._' for o in p.relative_to(path).parts)] scripts = [p for p in sorted(path.rglob('*.qmd.py')) if not any(o[0] in '._' for o in p.relative_to(path).parts)] for src in srcs: - procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_content_visible, repl_frontmatter_keys] + procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_html_md_links] + if src.suffix!='.txt': procs += [repl_content_visible, repl_frontmatter_keys] if src.suffix=='.ipynb': migrate_nb_cells(src, procs) else: migrate_md_file(src, procs) for src in scripts: migrate_md_file(src, [repl_script_frontmatter_keys]) diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index 6088ca905..e4e058f6b 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -898,7 +898,7 @@ "`nbdev-migrate-octavo` converts Quarto documentation sources in place. Commit or back up the project before running it. The command:\n", "\n", "1. Renames `.qmd` documents to `.md` and `.qmd.py` render scripts to `.md.py`.\n", - "2. Rewrites references to renamed documents.\n", + "2. Rewrites references to renamed documents, and to Markdown copies of pages: Quarto named the copy of `page.ipynb` `page.html.md`, while Octavo names it `page.md`. This includes plain-text files such as `llms.txt`.\n", "3. Replaces Quarto content-visible block markers with raw-format fences.\n", "4. Converts selected page-metadata names, such as `subtitle` to `description`.\n", "5. Creates `octavo.yml` from selected Quarto settings when no Octavo configuration exists.\n", @@ -932,7 +932,9 @@ "source": [ "Renaming a document also requires updating references to it. A page beside `guide.qmd` refers to it differently from a page in a subdirectory: `guide.qmd` versus `../guide.qmd`. Migration calculates replacement paths from the documentation root and from each source file’s directory.\n", "\n", - "The current implementation performs text replacement rather than parsing links. Matching text outside link destinations can also change." + "The current implementation performs text replacement rather than parsing links. Matching text outside link destinations can also change.\n", + "\n", + "The Markdown copy of each page also moves. Quarto published `page.html.md` beside `page.html`; Octavo publishes `page.md`. Hand-written indexes such as `llms.txt` often link these copies, so migration rewrites `.html.md` targets to `.md` in Markdown sources and `.txt` files." ] }, { @@ -955,7 +957,11 @@ "def repl_qmd_links(txt, moves, src, root):\n", " \"Rewrite relative and root-relative `.qmd` link targets to `.md`\"\n", " for old,new in _qmd_repls(src, root, moves): txt = txt.replace(old, new)\n", - " return txt" + " return txt\n", + "\n", + "def repl_html_md_links(txt):\n", + " \"Rewrite Quarto `.html.md` Markdown-copy targets to Octavo's `.md`\"\n", + " return re.sub(r'\\.html\\.md\\b', '.md', txt)" ] }, { @@ -1106,7 +1112,7 @@ " if changed: write_nb(nb, path)\n", "\n", "def migrate_md_file(path, procs):\n", - " \"Apply `procs` to a Markdown/QMD file in-place\"\n", + " \"Apply `procs` to a Markdown, QMD or text file in-place\"\n", " txt = path.read_text()\n", " new_txt = _apply_procs(txt, procs)\n", " if new_txt!=txt: path.write_text(new_txt)" @@ -1308,12 +1314,13 @@ " elif not qy.is_file(): raise FileNotFoundError(qy)\n", " cfg = _create_octavo_cfg(path) if qy.exists() and not (path/'octavo.yml').exists() else None\n", "\n", - " srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md')\n", + " srcs = [p for p in sorted(path.rglob('*')) if p.suffix in ('.ipynb','.qmd','.md','.txt')\n", " and not any(o[0] in '._' for o in p.relative_to(path).parts)]\n", " scripts = [p for p in sorted(path.rglob('*.qmd.py'))\n", " if not any(o[0] in '._' for o in p.relative_to(path).parts)]\n", " for src in srcs:\n", - " procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_content_visible, repl_frontmatter_keys]\n", + " procs = [partial(repl_qmd_links, moves=moves, src=src, root=path), repl_html_md_links]\n", + " if src.suffix!='.txt': procs += [repl_content_visible, repl_frontmatter_keys]\n", " if src.suffix=='.ipynb': migrate_nb_cells(src, procs)\n", " else: migrate_md_file(src, procs)\n", " for src in scripts: migrate_md_file(src, [repl_script_frontmatter_keys])\n", @@ -1402,7 +1409,8 @@ "---\n", "\n", "Back to [guide](../guide.qmd).\n", - "''')" + "''')\n", + "(nbs/'llms.txt').write_text('- [API](api.html.md): reference\\n- [Guide](guide.md)\\n')" ] }, { @@ -1617,7 +1625,7 @@ "id": "de7fa745", "metadata": {}, "source": [ - "Frontmatter aliases change at the top level. Relative links still need to reach the renamed document from its own directory." + "Frontmatter aliases change at the top level. Relative links still need to reach the renamed document from its own directory, and `llms.txt` now links Octavo's Markdown copies." ] }, { @@ -1639,7 +1647,8 @@ "test_eq(blog_fm['sidebar'], False)\n", "test_eq(_nav_item(dict(text='Help', menu=[dict(text='Guide', href='guide.qmd')])),\n", " dict(text='Help', children=[dict(text='Guide', href='guide.html')]))\n", - "test_eq(blog_body.strip(), 'Back to [guide](../guide.md).')" + "test_eq(blog_body.strip(), 'Back to [guide](../guide.md).')\n", + "test_eq((nbs/'llms.txt').read_text(), '- [API](api.md): reference\\n- [Guide](guide.md)\\n')" ] }, { From 8cc66c36706dce3d86924397f8ae041972ada4ba Mon Sep 17 00:00:00 2001 From: Rens Date: Fri, 9 Oct 2026 11:49:40 +0200 Subject: [PATCH 08/16] add instruction on excluding files --- nbdev/octavo.py | 14 +++++++++----- nbs/api/21_octavo.ipynb | 21 ++++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/nbdev/octavo.py b/nbdev/octavo.py index b166875d1..a9a9563bd 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -26,8 +26,9 @@ - Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as any custom layouts. Compare toc and toc_depth settings with the original metadata, including explicit depth combined with toc: false. - Embedded and conditional content: review the project's custom markup and format-conditional blocks. Compare when-format and unless-format behavior with the original sources in both HTML and Markdown output. Do not assume automatic conversion preserves exclusion conditions or renders markdown inside an HTML-targeted block. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the result rather than rewriting every block by pattern. - Render scripts and processors: if the project uses these, check their generated content and assets after migration. -- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. +- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. Markdown copies of notebook pages move from page.html.md to page.md, so external links to the old addresses break; Octavo does not redirect them. Report this change rather than working around it. - Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes. +- Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review. - Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. - Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions. - Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow. @@ -38,6 +39,8 @@ Report the changes, checks actually run, unresolved differences, omitted settings, and decisions requiring my review. Keep the original configuration and baseline until the migration has been reviewed. +After the review, delete _quarto.yml and the files it lists under metadata-files and syntax-definitions that Octavo does not use, such as nbdev.yml and sidebar.yml. + Docs: https://nbdev.fast.ai/api/octavo.html.md""" # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. @@ -93,12 +96,13 @@ def render_script(path): return (f'---\n{fm}\n---\n\n' if fm else '') + f.getvalue() # %% ../nbs/api/21_octavo.ipynb #e85c51f3 -def script_pages(root): +def script_pages(root, # Documentation directory + exclude=None): # Glob patterns of paths to skip, as in Octavo's `exclude` setting "Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s" root,pages,files = Path(root),L(),L() for o in sorted(root.rglob('*.py')): rel = o.relative_to(root) - if hidden(rel) or (txt:=render_script(o)) is None: continue + if hidden(rel, exclude) or (txt:=render_script(o)) is None: continue dest = rel.with_suffix('') if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest))) else: files.append(GeneratedFile(str(rel), str(dest), txt)) @@ -122,8 +126,8 @@ def octavo_docs( cfg = get_config(path) settings = {'output': cfg.doc_path, **octavo_config(cfg.nbs_path)} with working_directory(cfg.config_path): - pages,files = script_pages(cfg.nbs_path) - docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages + pages,files = script_pages(cfg.nbs_path, settings.get('exclude')) + docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg)), exclude=settings.get('exclude')) + pages site = Site(docs, root=cfg.nbs_path, files=files, **settings) if (miss := site.build()): print(miss) if (post := cfg.get('docs_post_build')): diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index e4e058f6b..e25fda5f3 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -53,8 +53,9 @@ "- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as any custom layouts. Compare toc and toc_depth settings with the original metadata, including explicit depth combined with toc: false.\n", "- Embedded and conditional content: review the project's custom markup and format-conditional blocks. Compare when-format and unless-format behavior with the original sources in both HTML and Markdown output. Do not assume automatic conversion preserves exclusion conditions or renders markdown inside an HTML-targeted block. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the result rather than rewriting every block by pattern.\n", "- Render scripts and processors: if the project uses these, check their generated content and assets after migration.\n", - "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer.\n", + "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. Markdown copies of notebook pages move from page.html.md to page.md, so external links to the old addresses break; Octavo does not redirect them. Report this change rather than working around it.\n", "- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes.\n", + "- Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review.\n", "- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior.\n", "- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions.\n", "- Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow.\n", @@ -63,7 +64,9 @@ "\n", "Serve the saved baseline and new output separately. Confirm that each server points to the intended directory. Inspect representative pages at desktop and narrow widths. Test navigation, mobile navigation, anchors, images, and the optional features identified above. Test search and follow result links from both the homepage and a nested page. Exercise the intended hosting subpath, such as /project/, rather than testing only at /. A successful build or static link check does not establish that browser interactions work.\n", "\n", - "Report the changes, checks actually run, unresolved differences, omitted settings, and decisions requiring my review. Keep the original configuration and baseline until the migration has been reviewed." + "Report the changes, checks actually run, unresolved differences, omitted settings, and decisions requiring my review. Keep the original configuration and baseline until the migration has been reviewed.\n", + "\n", + "After the review, delete _quarto.yml and the files it lists under metadata-files and syntax-definitions that Octavo does not use, such as nbdev.yml and sidebar.yml." ] }, { @@ -231,7 +234,9 @@ "\n", "A `.md.py` script can put YAML page metadata in its opening docstring. nbdev prepends that metadata to the printed Markdown.\n", "\n", - "Render-script source files are not published. Ordinary single-suffix `.py` files are copied as assets rather than executed. An exception in a render script stops the build." + "Render-script source files are not published. Ordinary single-suffix `.py` files are copied as assets rather than executed. An exception in a render script stops the build.\n", + "\n", + "Scripts under paths matched by the `exclude` setting in `octavo.yml` are not run." ] }, { @@ -260,12 +265,13 @@ "outputs": [], "source": [ "#| export\n", - "def script_pages(root):\n", + "def script_pages(root, # Documentation directory\n", + " exclude=None): # Glob patterns of paths to skip, as in Octavo's `exclude` setting\n", " \"Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s\"\n", " root,pages,files = Path(root),L(),L()\n", " for o in sorted(root.rglob('*.py')):\n", " rel = o.relative_to(root)\n", - " if hidden(rel) or (txt:=render_script(o)) is None: continue\n", + " if hidden(rel, exclude) or (txt:=render_script(o)) is None: continue\n", " dest = rel.with_suffix('')\n", " if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest)))\n", " else: files.append(GeneratedFile(str(rel), str(dest), txt))\n", @@ -292,6 +298,7 @@ " test_eq(pages[0].meta['title'], 'Hi')\n", " test_eq(pages[1].text, '# No frontmatter\\n')\n", " test_eq([(f.src,f.dest,f.content) for f in files], [('images/dot.svg.py','images/dot.svg','\\n')])\n", + " test_eq(script_pages(d, exclude=['images'])[1], [])\n", " test_is(render_script(d/'ordinary.py'), None)\n", " (d/'bad.md.py').write_text('1/0')\n", " with expect_fail(ZeroDivisionError, contains='division'): script_pages(d)" @@ -396,8 +403,8 @@ " cfg = get_config(path)\n", " settings = {'output': cfg.doc_path, **octavo_config(cfg.nbs_path)}\n", " with working_directory(cfg.config_path):\n", - " pages,files = script_pages(cfg.nbs_path)\n", - " docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg))) + pages\n", + " pages,files = script_pages(cfg.nbs_path, settings.get('exclude'))\n", + " docs = scan(cfg.nbs_path, nbproc=OctavoProc(local_lib=local_libraries(cfg)), exclude=settings.get('exclude')) + pages\n", " site = Site(docs, root=cfg.nbs_path, files=files, **settings)\n", " if (miss := site.build()): print(miss)\n", " if (post := cfg.get('docs_post_build')):\n", From 4dd06d8624d964e85f1410ea146e94509658d914 Mon Sep 17 00:00:00 2001 From: Rens Date: Fri, 9 Oct 2026 12:22:54 +0200 Subject: [PATCH 09/16] support custom sidebar migration --- nbdev/_modidx.py | 2 + nbdev/octavo.py | 30 +++++++++++++- nbs/api/21_octavo.ipynb | 91 +++++++++++++++++++++++++++++++++-------- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index 6cda28bd2..9491d0154 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -227,10 +227,12 @@ 'nbdev.octavo._nav_item': ('api/octavo.html#_nav_item', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_nav': ('api/octavo.html#_octavo_nav', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_output': ('api/octavo.html#_octavo_output', 'nbdev/octavo.py'), + 'nbdev.octavo._octavo_sidebar': ('api/octavo.html#_octavo_sidebar', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_moves': ('api/octavo.html#_qmd_moves', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_repls': ('api/octavo.html#_qmd_repls', 'nbdev/octavo.py'), 'nbdev.octavo._quarto_config': ('api/octavo.html#_quarto_config', 'nbdev/octavo.py'), 'nbdev.octavo._repl_cv_match': ('api/octavo.html#_repl_cv_match', 'nbdev/octavo.py'), + 'nbdev.octavo._sidebar_item': ('api/octavo.html#_sidebar_item', 'nbdev/octavo.py'), 'nbdev.octavo._tool_item': ('api/octavo.html#_tool_item', 'nbdev/octavo.py'), 'nbdev.octavo.local_libraries': ('api/octavo.html#local_libraries', 'nbdev/octavo.py'), 'nbdev.octavo.migrate_md_file': ('api/octavo.html#migrate_md_file', 'nbdev/octavo.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index a9a9563bd..2651a31fb 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -305,8 +305,10 @@ def migrate_md_file(path, procs): # %% ../nbs/api/21_octavo.ipynb #01ed763e def _nav_href(h): + "Quarto link `h`, with a documentation source path changed to its published HTML page" if not h or '://' in h or h.startswith(('#','mailto:')): return h - return re.sub(r'\.(?:ipynb|qmd|md)(?=[?#]|$)', '.html', h) + m = re.fullmatch(r'(.*?)([^/?#]+)\.(?:ipynb|qmd|md)([?#].*)?', h) + return f'{m[1]}{page_nm(m[2])}.html{m[3] or ""}' if m else h def _nav_item(item): res = {'text': item['text']} @@ -357,6 +359,31 @@ def _nav(k): return nested_idx(qy,'website','navbar',k) or nested_idx(ny,'websit if (tools := tools.filter()): cfg['tools'] = list(tools) return cfg +# %% ../nbs/api/21_octavo.ipynb #e0912069 +def _sidebar_item(o, drop): + "An Octavo `sidebar_links` item from Quarto sidebar entry `o`; `drop` reports entries without an equivalent" + if isinstance(o, str): o = dict(href=o) + res = {} + if (text := o.get('section') or o.get('text')) == '---': return None + if text: res['text'] = text + if href := o.get('href'): + if href=='auto' or '*' in href: return drop(href) + res['href'] = _nav_href(href) + if 'contents' in o: res['children'] = [i for c in listify(o['contents']) if (i := _sidebar_item(c, drop))] + return res or None + +def _octavo_sidebar(nbs_path): + "Convert a hand-written `sidebar.yml` into Octavo `sidebar_links`" + p = Path(nbs_path)/'sidebar.yml' + if not get_config(nbs_path).custom_sidebar or not p.exists(): return {} + sb = nested_idx(yaml.safe_load(p.read_text()) or {}, 'website', 'sidebar') or {} + if not isinstance(sb, dict): + print(f'{p}: not converted: Octavo supports one sidebar. Set sidebar_links in octavo.yml by hand.') + return {} + def drop(h): print(f'{p}: dropped sidebar entry {h!r}: Octavo has no equivalent. List those pages explicitly.') + links = [i for o in listify(sb.get('contents')) if (i := _sidebar_item(o, drop))] + return {'sidebar_links': links} if links else {} + # %% ../nbs/api/21_octavo.ipynb #2f2d4b3b def _create_octavo_cfg(nbs_path): "Return Octavo settings derived from `_quarto.yml` and `nbdev.yml`" @@ -377,6 +404,7 @@ def _create_octavo_cfg(nbs_path): favicon = ws_n.get('favicon') or ws_q.get('favicon') if favicon: cfg['favicon'] = favicon cfg.update(_octavo_nav(nbs_path, qy, ny)) + cfg.update(_octavo_sidebar(nbs_path)) return cfg # %% ../nbs/api/21_octavo.ipynb #c4b4478c diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index e25fda5f3..0900f9415 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -726,14 +726,14 @@ "```python\n", "{ 'cells': [ { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': '03d65b6e',\n", + " 'id': 'b9be9d6e',\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", " 'source': '# Example\\n\\n> Repository document'},\n", " { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': 'bcf12a9e',\n", + " 'id': '7e3ae9fb',\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -744,7 +744,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': '0b2fc6cb',\n", + " 'id': '524050f8',\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -756,7 +756,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': 'ba6db483',\n", + " 'id': '64a587d3',\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -774,14 +774,14 @@ " 'metadata': {},\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", - " 'id': '03d65b6e'},\n", + " 'id': 'b9be9d6e'},\n", " {'cell_type': 'markdown',\n", " 'source': 'Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md',\n", " 'directives_': {},\n", " 'metadata': {},\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", - " 'id': 'bcf12a9e'},\n", + " 'id': '7e3ae9fb'},\n", " {'cell_type': 'code',\n", " 'source': '#| echo: false\\nraise RuntimeError(\"must not execute\")',\n", " 'directives_': {},\n", @@ -792,7 +792,7 @@ " 'execution_count': None,\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", - " 'id': '0b2fc6cb'},\n", + " 'id': '524050f8'},\n", " {'cell_type': 'code',\n", " 'source': '#| include: false\\n\"excluded cell\"',\n", " 'directives_': {},\n", @@ -801,7 +801,7 @@ " 'execution_count': None,\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", - " 'id': 'ba6db483'}],\n", + " 'id': '64a587d3'}],\n", " 'metadata': {},\n", " 'nbformat': 4,\n", " 'nbformat_minor': 5}" @@ -1134,8 +1134,10 @@ "source": [ "#| export\n", "def _nav_href(h):\n", + " \"Quarto link `h`, with a documentation source path changed to its published HTML page\"\n", " if not h or '://' in h or h.startswith(('#','mailto:')): return h\n", - " return re.sub(r'\\.(?:ipynb|qmd|md)(?=[?#]|$)', '.html', h)\n", + " m = re.fullmatch(r'(.*?)([^/?#]+)\\.(?:ipynb|qmd|md)([?#].*)?', h)\n", + " return f'{m[1]}{page_nm(m[2])}.html{m[3] or \"\"}' if m else h\n", "\n", "def _nav_item(item):\n", " res = {'text': item['text']}\n", @@ -1253,6 +1255,47 @@ " return cfg" ] }, + { + "cell_type": "markdown", + "id": "6d3b3ba1", + "metadata": {}, + "source": [ + "Octavo derives its sidebar from directories, which matches the sidebar nbdev generates. A project with `custom_sidebar` writes `sidebar.yml` by hand instead, often grouping pages from one directory into sections. Migration converts that file into `sidebar_links`: pages become links and sections become groups. Entries without text take the page's title at build time. Quarto's `auto` entries and glob patterns have no Octavo equivalent; migration reports and omits them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0912069", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _sidebar_item(o, drop):\n", + " \"An Octavo `sidebar_links` item from Quarto sidebar entry `o`; `drop` reports entries without an equivalent\"\n", + " if isinstance(o, str): o = dict(href=o)\n", + " res = {}\n", + " if (text := o.get('section') or o.get('text')) == '---': return None\n", + " if text: res['text'] = text\n", + " if href := o.get('href'):\n", + " if href=='auto' or '*' in href: return drop(href)\n", + " res['href'] = _nav_href(href)\n", + " if 'contents' in o: res['children'] = [i for c in listify(o['contents']) if (i := _sidebar_item(c, drop))]\n", + " return res or None\n", + "\n", + "def _octavo_sidebar(nbs_path):\n", + " \"Convert a hand-written `sidebar.yml` into Octavo `sidebar_links`\"\n", + " p = Path(nbs_path)/'sidebar.yml'\n", + " if not get_config(nbs_path).custom_sidebar or not p.exists(): return {}\n", + " sb = nested_idx(yaml.safe_load(p.read_text()) or {}, 'website', 'sidebar') or {}\n", + " if not isinstance(sb, dict):\n", + " print(f'{p}: not converted: Octavo supports one sidebar. Set sidebar_links in octavo.yml by hand.')\n", + " return {}\n", + " def drop(h): print(f'{p}: dropped sidebar entry {h!r}: Octavo has no equivalent. List those pages explicitly.')\n", + " links = [i for o in listify(sb.get('contents')) if (i := _sidebar_item(o, drop))]\n", + " return {'sidebar_links': links} if links else {}" + ] + }, { "cell_type": "markdown", "id": "e1cd81ec", @@ -1290,6 +1333,7 @@ " favicon = ws_n.get('favicon') or ws_q.get('favicon')\n", " if favicon: cfg['favicon'] = favicon\n", " cfg.update(_octavo_nav(nbs_path, qy, ny))\n", + " cfg.update(_octavo_sidebar(nbs_path))\n", " return cfg" ] }, @@ -1380,7 +1424,7 @@ { "data": { "text/plain": [ - "115" + "52" ] }, "execution_count": null, @@ -1392,7 +1436,7 @@ "migration_tmp = tempfile.TemporaryDirectory()\n", "d = migration_tmp.name\n", "nbs = Path(d)/'nbs'\n", - "(Path(d)/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\n')\n", + "(Path(d)/'pyproject.toml').write_text('[project]\\nname = \"demo\"\\n[tool.nbdev]\\nnbs_path = \"nbs\"\\ncustom_sidebar = true\\n')\n", "(nbs/'blog/posts').mkdir(parents=True)\n", "(nbs/'images').mkdir()\n", "(nbs/'guide.qmd').write_text(r'''---\n", @@ -1463,7 +1507,7 @@ "id": "57b7b6dd", "metadata": {}, "source": [ - "This project includes both supported and unsupported navbar icons. Migration should report the unsupported entry without losing the supported links." + "This project includes both supported and unsupported navbar icons. Migration should report the unsupported entry without losing the supported links. The project sets `custom_sidebar`, and its hand-written `sidebar.yml` has a section and an `auto` entry, which migration reports." ] }, { @@ -1475,7 +1519,7 @@ { "data": { "text/plain": [ - "371" + "163" ] }, "execution_count": null, @@ -1504,7 +1548,17 @@ " css: styles.css\n", " toc-depth: 4\n", "''')\n", - "(nbs/'styles.css').write_text('body { color: red; }')" + "(nbs/'styles.css').write_text('body { color: red; }')\n", + "(nbs/'sidebar.yml').write_text(r'''website:\n", + " sidebar:\n", + " contents:\n", + " - guide.qmd\n", + " - section: More\n", + " contents:\n", + " - blog/index.qmd\n", + " - 01_extra.ipynb\n", + " - auto\n", + "''')" ] }, { @@ -1554,7 +1608,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpcyu0zsjx/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n" + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpjp32wa6p/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n", + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpjp32wa6p/nbs/sidebar.yml: dropped sidebar entry 'auto': Octavo has no equivalent. List those pages explicitly.\n" ] } ], @@ -1603,7 +1658,7 @@ "id": "cacaa706", "metadata": {}, "source": [ - "The converted configuration retains the site identity and supported navigation. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." + "The converted configuration retains the site identity and supported navigation. The sidebar keeps its order and section; `01_extra.ipynb` links to its published name, `extra.html`. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." ] }, { @@ -1624,7 +1679,9 @@ "test_eq(cfg['favicon'], 'favicon.png')\n", "test_eq(cfg['nav_links'], [{'text': 'Guide', 'href': 'guide.html'}])\n", "test_eq(cfg['tools'], [{'icon': 'github', 'href': 'https://github.com/x/y', 'label': 'GitHub'},\n", - " {'icon': 'x', 'href': 'https://twitter.com/x', 'label': 'Follow us'}])" + " {'icon': 'x', 'href': 'https://twitter.com/x', 'label': 'Follow us'}])\n", + "test_eq(cfg['sidebar_links'], [{'href': 'guide.html'},\n", + " {'text': 'More', 'children': [{'href': 'blog/index.html'}, {'href': 'extra.html'}]}])" ] }, { From 23404f280d6346b21c320da05df406e34c59ddd7 Mon Sep 17 00:00:00 2001 From: Rens Date: Mon, 12 Oct 2026 14:10:07 +0200 Subject: [PATCH 10/16] default to dollar mode on octavo migration --- nbdev/octavo.py | 4 +++- nbs/api/21_octavo.ipynb | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 2651a31fb..4106c87fd 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -28,6 +28,7 @@ - Render scripts and processors: if the project uses these, check their generated content and assets after migration. - Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. Markdown copies of notebook pages move from page.html.md to page.md, so external links to the old addresses break; Octavo does not redirect them. Report this change rather than working around it. - Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes. +- Code that reads documentation sources: migration renames .qmd sources to .md but does not update code that finds them by name. Search the whole project, including build scripts, tests, generators and contributor instructions, for .qmd, and update paths, globs and extension checks. A glob that no longer matches anything fails silently, so run the project's full test suite, not only nbdev-docs, and check that tests that read documentation pages still find them. Saved notebook outputs that contain .qmd links need to be regenerated after the fix. - Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review. - Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. - Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions. @@ -391,7 +392,8 @@ def _create_octavo_cfg(nbs_path): qy,ny = _quarto_config(nbs_path) ws_q,ws_n = qy.get('website', {}) or {},ny.get('website', {}) or {} fmt_html = (qy.get('format', {}) or {}).get('html', {}) or {} - cfg = {} + # Quarto reads `$…$` as inline maths, and Octavo does only with this setting. + cfg = {'math': 'dollars'} if (output := _octavo_output(nbs_path, qy, ny)) is not None: cfg['output'] = output title = ws_n.get('title') or ws_q.get('title') if title: cfg['title'] = title diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index 0900f9415..e23d4c1f6 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -55,6 +55,7 @@ "- Render scripts and processors: if the project uses these, check their generated content and assets after migration.\n", "- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. Markdown copies of notebook pages move from page.html.md to page.md, so external links to the old addresses break; Octavo does not redirect them. Report this change rather than working around it.\n", "- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes.\n", + "- Code that reads documentation sources: migration renames .qmd sources to .md but does not update code that finds them by name. Search the whole project, including build scripts, tests, generators and contributor instructions, for .qmd, and update paths, globs and extension checks. A glob that no longer matches anything fails silently, so run the project's full test suite, not only nbdev-docs, and check that tests that read documentation pages still find them. Saved notebook outputs that contain .qmd links need to be regenerated after the fix.\n", "- Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review.\n", "- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior.\n", "- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions.\n", @@ -1320,7 +1321,8 @@ " qy,ny = _quarto_config(nbs_path)\n", " ws_q,ws_n = qy.get('website', {}) or {},ny.get('website', {}) or {}\n", " fmt_html = (qy.get('format', {}) or {}).get('html', {}) or {}\n", - " cfg = {}\n", + " # Quarto reads `$…$` as inline maths, and Octavo does only with this setting.\n", + " cfg = {'math': 'dollars'}\n", " if (output := _octavo_output(nbs_path, qy, ny)) is not None: cfg['output'] = output\n", " title = ws_n.get('title') or ws_q.get('title')\n", " if title: cfg['title'] = title\n", @@ -1658,7 +1660,7 @@ "id": "cacaa706", "metadata": {}, "source": [ - "The converted configuration retains the site identity and supported navigation. The sidebar keeps its order and section; `01_extra.ipynb` links to its published name, `extra.html`. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." + "The converted configuration retains the site identity and supported navigation. The sidebar keeps its order and section; `01_extra.ipynb` links to its published name, `extra.html`. It sets `math: dollars`, so `$…$` stays inline maths as in Quarto. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." ] }, { @@ -1672,6 +1674,7 @@ "assert (nbs/'octavo.yml').exists()\n", "cfg = yaml.safe_load((nbs/'octavo.yml').read_text())\n", "test_eq(cfg['title'], 'demo')\n", + "test_eq(cfg['math'], 'dollars')\n", "assert 'output' not in cfg\n", "test_eq(cfg['toc_depth'], 3)\n", "assert 'css' not in cfg\n", From 98da6d30ed6f1bb2c2fe429b0c0e32eefca5ac71 Mon Sep 17 00:00:00 2001 From: Rens Date: Mon, 12 Oct 2026 16:14:35 +0200 Subject: [PATCH 11/16] add page metadata and twitter cards --- nbdev/_modidx.py | 1 + nbdev/octavo.py | 19 +++++++++++++++- nbs/api/21_octavo.ipynb | 50 +++++++++++++++++++++++++++++++---------- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index 9491d0154..f7953ebf1 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -228,6 +228,7 @@ 'nbdev.octavo._octavo_nav': ('api/octavo.html#_octavo_nav', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_output': ('api/octavo.html#_octavo_output', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_sidebar': ('api/octavo.html#_octavo_sidebar', 'nbdev/octavo.py'), + 'nbdev.octavo._octavo_social': ('api/octavo.html#_octavo_social', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_moves': ('api/octavo.html#_qmd_moves', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_repls': ('api/octavo.html#_qmd_repls', 'nbdev/octavo.py'), 'nbdev.octavo._quarto_config': ('api/octavo.html#_quarto_config', 'nbdev/octavo.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 4106c87fd..bddec199a 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -30,7 +30,7 @@ - Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes. - Code that reads documentation sources: migration renames .qmd sources to .md but does not update code that finds them by name. Search the whole project, including build scripts, tests, generators and contributor instructions, for .qmd, and update paths, globs and extension checks. A glob that no longer matches anything fails silently, so run the project's full test suite, not only nbdev-docs, and check that tests that read documentation pages still find them. Saved notebook outputs that contain .qmd links need to be regenerated after the fix. - Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review. -- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. +- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including Open Graph and Twitter card options the migration reported as dropped, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. - Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions. - Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow. @@ -386,6 +386,20 @@ def drop(h): print(f'{p}: dropped sidebar entry {h!r}: Octavo has no equivalent. return {'sidebar_links': links} if links else {} # %% ../nbs/api/21_octavo.ipynb #2f2d4b3b +_IMAGE_KEYS = {'title', 'description', 'image', 'image-alt', 'image-width', 'image-height'} +_SOCIAL_KEYS = {'open-graph': _IMAGE_KEYS | {'site-name', 'locale'}, 'twitter-card': _IMAGE_KEYS | {'card-style', 'creator', 'site'}} + +def _octavo_social(ws): + "Octavo `open_graph` and `twitter_card` settings from Quarto `website` settings `ws`, reporting options Octavo lacks" + res = {} + for key,keys in _SOCIAL_KEYS.items(): + if not (v := ws.get(key)): continue + if isinstance(v, dict): + if drop := sorted(set(v)-keys): print(f'{key}: dropped {", ".join(drop)}: Octavo has no equivalent.') + v = {k.replace('-','_'):o for k,o in v.items() if k in keys} or True + res[key.replace('-','_')] = v + return res + def _create_octavo_cfg(nbs_path): "Return Octavo settings derived from `_quarto.yml` and `nbdev.yml`" nbs_path = Path(nbs_path).resolve() @@ -405,8 +419,11 @@ def _create_octavo_cfg(nbs_path): else: cfg['toc_depth'] = 0 if fmt_html.get('toc') is False else 3 favicon = ws_n.get('favicon') or ws_q.get('favicon') if favicon: cfg['favicon'] = favicon + desc = ws_n.get('description') or ws_q.get('description') + if desc: cfg['description'] = desc cfg.update(_octavo_nav(nbs_path, qy, ny)) cfg.update(_octavo_sidebar(nbs_path)) + cfg.update(_octavo_social({**ws_q, **ws_n})) return cfg # %% ../nbs/api/21_octavo.ipynb #c4b4478c diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index e23d4c1f6..3ae180ed2 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -57,7 +57,7 @@ "- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes.\n", "- Code that reads documentation sources: migration renames .qmd sources to .md but does not update code that finds them by name. Search the whole project, including build scripts, tests, generators and contributor instructions, for .qmd, and update paths, globs and extension checks. A glob that no longer matches anything fails silently, so run the project's full test suite, not only nbdev-docs, and check that tests that read documentation pages still find them. Saved notebook outputs that contain .qmd links need to be regenerated after the fix.\n", "- Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review.\n", - "- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including social metadata, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior.\n", + "- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including Open Graph and Twitter card options the migration reported as dropped, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior.\n", "- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions.\n", "- Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow.\n", "\n", @@ -727,14 +727,14 @@ "```python\n", "{ 'cells': [ { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': 'b9be9d6e',\n", + " 'id': 'c2daa09c',\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", " 'source': '# Example\\n\\n> Repository document'},\n", " { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': '7e3ae9fb',\n", + " 'id': 'e050d919',\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -745,7 +745,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': '524050f8',\n", + " 'id': '61c6a532',\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -757,7 +757,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': '64a587d3',\n", + " 'id': '3af1d12c',\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -775,14 +775,14 @@ " 'metadata': {},\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", - " 'id': 'b9be9d6e'},\n", + " 'id': 'c2daa09c'},\n", " {'cell_type': 'markdown',\n", " 'source': 'Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md',\n", " 'directives_': {},\n", " 'metadata': {},\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", - " 'id': '7e3ae9fb'},\n", + " 'id': 'e050d919'},\n", " {'cell_type': 'code',\n", " 'source': '#| echo: false\\nraise RuntimeError(\"must not execute\")',\n", " 'directives_': {},\n", @@ -793,7 +793,7 @@ " 'execution_count': None,\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", - " 'id': '524050f8'},\n", + " 'id': '61c6a532'},\n", " {'cell_type': 'code',\n", " 'source': '#| include: false\\n\"excluded cell\"',\n", " 'directives_': {},\n", @@ -802,7 +802,7 @@ " 'execution_count': None,\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", - " 'id': '64a587d3'}],\n", + " 'id': '3af1d12c'}],\n", " 'metadata': {},\n", " 'nbformat': 4,\n", " 'nbformat_minor': 5}" @@ -1315,6 +1315,20 @@ "outputs": [], "source": [ "#| export\n", + "_IMAGE_KEYS = {'title', 'description', 'image', 'image-alt', 'image-width', 'image-height'}\n", + "_SOCIAL_KEYS = {'open-graph': _IMAGE_KEYS | {'site-name', 'locale'}, 'twitter-card': _IMAGE_KEYS | {'card-style', 'creator', 'site'}}\n", + "\n", + "def _octavo_social(ws):\n", + " \"Octavo `open_graph` and `twitter_card` settings from Quarto `website` settings `ws`, reporting options Octavo lacks\"\n", + " res = {}\n", + " for key,keys in _SOCIAL_KEYS.items():\n", + " if not (v := ws.get(key)): continue\n", + " if isinstance(v, dict):\n", + " if drop := sorted(set(v)-keys): print(f'{key}: dropped {\", \".join(drop)}: Octavo has no equivalent.')\n", + " v = {k.replace('-','_'):o for k,o in v.items() if k in keys} or True\n", + " res[key.replace('-','_')] = v\n", + " return res\n", + "\n", "def _create_octavo_cfg(nbs_path):\n", " \"Return Octavo settings derived from `_quarto.yml` and `nbdev.yml`\"\n", " nbs_path = Path(nbs_path).resolve()\n", @@ -1334,8 +1348,11 @@ " else: cfg['toc_depth'] = 0 if fmt_html.get('toc') is False else 3\n", " favicon = ws_n.get('favicon') or ws_q.get('favicon')\n", " if favicon: cfg['favicon'] = favicon\n", + " desc = ws_n.get('description') or ws_q.get('description')\n", + " if desc: cfg['description'] = desc\n", " cfg.update(_octavo_nav(nbs_path, qy, ny))\n", " cfg.update(_octavo_sidebar(nbs_path))\n", + " cfg.update(_octavo_social({**ws_q, **ws_n}))\n", " return cfg" ] }, @@ -1533,6 +1550,12 @@ "(nbs/'_quarto.yml').write_text(r'''website:\n", " title: demo\n", " favicon: favicon.png\n", + " description: A demo site\n", + " open-graph: true\n", + " twitter-card:\n", + " site: \"@demo\"\n", + " image: https://x.y/card.png\n", + " image-width: 1200\n", " navbar:\n", " left:\n", " - text: Guide\n", @@ -1610,8 +1633,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpjp32wa6p/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n", - "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpjp32wa6p/nbs/sidebar.yml: dropped sidebar entry 'auto': Octavo has no equivalent. List those pages explicitly.\n" + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpbb06_xm6/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n", + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpbb06_xm6/nbs/sidebar.yml: dropped sidebar entry 'auto': Octavo has no equivalent. List those pages explicitly.\n" ] } ], @@ -1660,7 +1683,7 @@ "id": "cacaa706", "metadata": {}, "source": [ - "The converted configuration retains the site identity and supported navigation. The sidebar keeps its order and section; `01_extra.ipynb` links to its published name, `extra.html`. It sets `math: dollars`, so `$…$` stays inline maths as in Quarto. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." + "The converted configuration retains the site identity and supported navigation. The sidebar keeps its order and section; `01_extra.ipynb` links to its published name, `extra.html`. The site description, Open Graph and Twitter card settings carry over, with keys such as `image-width` renamed to `image_width`; the migration reports any key Octavo doesn't support as dropped. It sets `math: dollars`, so `$…$` stays inline maths as in Quarto. Quarto CSS is not enabled in `octavo.yml`. The original configuration and stylesheet remain unchanged for manual adaptation." ] }, { @@ -1675,6 +1698,9 @@ "cfg = yaml.safe_load((nbs/'octavo.yml').read_text())\n", "test_eq(cfg['title'], 'demo')\n", "test_eq(cfg['math'], 'dollars')\n", + "test_eq(cfg['description'], 'A demo site')\n", + "test_eq(cfg['open_graph'], True)\n", + "test_eq(cfg['twitter_card'], {'site': '@demo', 'image': 'https://x.y/card.png', 'image_width': 1200})\n", "assert 'output' not in cfg\n", "test_eq(cfg['toc_depth'], 3)\n", "assert 'css' not in cfg\n", From b5c98715a8978ebd53744f7c77b43870a9c644e9 Mon Sep 17 00:00:00 2001 From: Rens Date: Tue, 13 Oct 2026 09:19:43 +0200 Subject: [PATCH 12/16] refactor octavo file exclusion --- nbdev/octavo.py | 5 ++--- nbs/api/21_octavo.ipynb | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nbdev/octavo.py b/nbdev/octavo.py index bddec199a..20a205e9e 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -101,9 +101,8 @@ def script_pages(root, # Documentation directory exclude=None): # Glob patterns of paths to skip, as in Octavo's `exclude` setting "Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s" root,pages,files = Path(root),L(),L() - for o in sorted(root.rglob('*.py')): - rel = o.relative_to(root) - if hidden(rel, exclude) or (txt:=render_script(o)) is None: continue + for rel in source_files(root, exclude).map(Path).filter(lambda o: o.suffix=='.py'): + if (txt:=render_script(root/rel)) is None: continue dest = rel.with_suffix('') if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest))) else: files.append(GeneratedFile(str(rel), str(dest), txt)) diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/21_octavo.ipynb index 3ae180ed2..afda72868 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/21_octavo.ipynb @@ -270,9 +270,8 @@ " exclude=None): # Glob patterns of paths to skip, as in Octavo's `exclude` setting\n", " \"Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s\"\n", " root,pages,files = Path(root),L(),L()\n", - " for o in sorted(root.rglob('*.py')):\n", - " rel = o.relative_to(root)\n", - " if hidden(rel, exclude) or (txt:=render_script(o)) is None: continue\n", + " for rel in source_files(root, exclude).map(Path).filter(lambda o: o.suffix=='.py'):\n", + " if (txt:=render_script(root/rel)) is None: continue\n", " dest = rel.with_suffix('')\n", " if dest.suffix=='.md': pages.append(Page(str(rel), txt, nm=page_nm(dest)))\n", " else: files.append(GeneratedFile(str(rel), str(dest), txt))\n", From 97c3d63787233f5925f2675bbc14ec72fba4cdea Mon Sep 17 00:00:00 2001 From: Rens Date: Tue, 13 Oct 2026 09:27:23 +0200 Subject: [PATCH 13/16] rename octavo nb from 21 to 20 --- nbdev/octavo.py | 50 +++++++++--------- nbs/api/{21_octavo.ipynb => 20_octavo.ipynb} | 53 +++++++++++++++----- 2 files changed, 65 insertions(+), 38 deletions(-) rename nbs/api/{21_octavo.ipynb => 20_octavo.ipynb} (98%) diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 20a205e9e..71a7e75f0 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -44,14 +44,14 @@ Docs: https://nbdev.fast.ai/api/octavo.html.md""" -# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/21_octavo.ipynb. +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/20_octavo.ipynb. # %% auto #0 __all__ = ['OctavoProc', 'local_libraries', 'render_script', 'script_pages', 'octavo_config', 'octavo_docs', 'octavo_preview', 'octavo_readme', 'repl_qmd_links', 'repl_html_md_links', 'repl_content_visible', 'repl_frontmatter_keys', 'repl_script_frontmatter_keys', 'migrate_nb_cells', 'migrate_md_file', 'nbdev_migrate_octavo'] -# %% ../nbs/api/21_octavo.ipynb #7b098bb5 +# %% ../nbs/api/20_octavo.ipynb #7b098bb5 from contextlib import redirect_stdout from io import StringIO from fastcore.utils import * @@ -68,7 +68,7 @@ from .sync import write_nb from .quarto import _doc_mtime_not_older, fs_watchdog -# %% ../nbs/api/21_octavo.ipynb #1cc66460 +# %% ../nbs/api/20_octavo.ipynb #1cc66460 class OctavoProc(FilterDefaults): "Prepare notebook content, metadata, and symbol links for Octavo" def __init__(self, local_lib=None, strip_libs=None): self.nl = NbdevLookup(strip_libs=strip_libs or local_lib, local_lib=local_lib) @@ -80,13 +80,13 @@ def __call__(self, nb): super().__call__(nb) if getattr(nb, 'frontmatter_', None) and nb.cells[0].cell_type=='raw': nb.cells[0].cell_type = 'markdown' -# %% ../nbs/api/21_octavo.ipynb #9a0258ee +# %% ../nbs/api/20_octavo.ipynb #9a0258ee def local_libraries(cfg): "Names of the project's declared nbdev symbol indexes" proj = _load_toml(cfg.config_file).get('project', {}) return tuple(proj.get('entry-points', {}).get('nbdev', {})) -# %% ../nbs/api/21_octavo.ipynb #f9f85ebf +# %% ../nbs/api/20_octavo.ipynb #f9f85ebf def render_script(path): "Execute a double-suffix render script and return its printed output with optional page metadata" path = Path(path) @@ -96,7 +96,7 @@ def render_script(path): with redirect_stdout(f): exec(compile(path.read_text(encoding='utf-8'), str(path), 'exec'), {}) return (f'---\n{fm}\n---\n\n' if fm else '') + f.getvalue() -# %% ../nbs/api/21_octavo.ipynb #e85c51f3 +# %% ../nbs/api/20_octavo.ipynb #e85c51f3 def script_pages(root, # Documentation directory exclude=None): # Glob patterns of paths to skip, as in Octavo's `exclude` setting "Render scripts under `root`: `*.md.py` as `Page`s, anything else as `GeneratedFile`s" @@ -108,7 +108,7 @@ def script_pages(root, # Documentation directory else: files.append(GeneratedFile(str(rel), str(dest), txt)) return pages,files -# %% ../nbs/api/21_octavo.ipynb #d6904375 +# %% ../nbs/api/20_octavo.ipynb #d6904375 def octavo_config(root): "Read `octavo.yml` as a settings dictionary; return an empty dictionary for an empty file" root = Path(root) @@ -118,7 +118,7 @@ def octavo_config(root): if not isinstance(settings, dict): raise ValueError(f'{root / "octavo.yml"} must contain a YAML mapping') return settings -# %% ../nbs/api/21_octavo.ipynb #f6830357 +# %% ../nbs/api/20_octavo.ipynb #f6830357 def octavo_docs( path:str=None, # Path to the project or its notebooks ): @@ -135,7 +135,7 @@ def octavo_docs( if not out.is_absolute(): out = cfg.nbs_path/out import_obj(post)(out.resolve(), cfg) -# %% ../nbs/api/21_octavo.ipynb #db88c5c9 +# %% ../nbs/api/20_octavo.ipynb #db88c5c9 def octavo_preview( path:str=None, # Path to the project or its notebooks port:int=None, # HTTP port (defaults to 8000) @@ -184,7 +184,7 @@ def on_change(event): rebuild() finally: server.shutdown() -# %% ../nbs/api/21_octavo.ipynb #0d38402b +# %% ../nbs/api/20_octavo.ipynb #0d38402b def octavo_readme( path:str=None, # Path to the project or its notebooks chk_time:bool=False, # Only build if the source is newer @@ -208,7 +208,7 @@ def octavo_readme( if desc:=doc.meta.get('description'): head += f'{desc}\n\n' md2gfm(head + doc.body, dest=dest, imgdir=cfg.config_path/f'{src.stem}_files', raw=('md', 'html')) -# %% ../nbs/api/21_octavo.ipynb #267d0150 +# %% ../nbs/api/20_octavo.ipynb #267d0150 def _qmd_moves(path): "`.qmd` documents and `.qmd.py` render scripts under `path`, mapped to Octavo names" path = Path(path) @@ -216,7 +216,7 @@ def _qmd_moves(path): return {p:p.with_name(re.sub(r'\.qmd(?=\.py$|$)', '.md', p.name)) for p in sorted(ps) if not any(o[0] in '._' for o in p.relative_to(path).parts)} -# %% ../nbs/api/21_octavo.ipynb #a6688f7e +# %% ../nbs/api/20_octavo.ipynb #a6688f7e def _qmd_repls(src, path, moves): "Old and new spellings of each moved document, as seen from `src`" def _rel(p, start): return os.path.relpath(p, start).replace(os.sep, '/') @@ -235,7 +235,7 @@ def repl_html_md_links(txt): "Rewrite Quarto `.html.md` Markdown-copy targets to Octavo's `.md`" return re.sub(r'\.html\.md\b', '.md', txt) -# %% ../nbs/api/21_octavo.ipynb #237aea2c +# %% ../nbs/api/20_octavo.ipynb #237aea2c _re_cv = re.compile(r'^:{3,}[^\S\r\n]*\{([^}]*\.content-visible[^}]*)\}[^\S\r\n]*\n(.*?)\n:{3,}[^\S\r\n]*$', re.MULTILINE | re.DOTALL) def _repl_cv_match(m): @@ -252,7 +252,7 @@ def repl_content_visible(s): "Convert Quarto `::: {.content-visible when-format=...}` to ````{=md}```` / ````{=html}````" return _re_cv.sub(_repl_cv_match, s) -# %% ../nbs/api/21_octavo.ipynb #04d5a738 +# %% ../nbs/api/20_octavo.ipynb #04d5a738 def _dump_fm(fm): return yaml.dump(fm, sort_keys=False, allow_unicode=True, width=10**9) def repl_frontmatter_keys(s): @@ -271,7 +271,7 @@ def repl_frontmatter_keys(s): if changed: return f'---\n{_dump_fm(fm)}---\n\n{body.lstrip(chr(10))}' return s -# %% ../nbs/api/21_octavo.ipynb #0ceec29d +# %% ../nbs/api/20_octavo.ipynb #0ceec29d _script_fm = re.compile(r'\A(?P"{3}|\'{3})---\n(?P.*?)\n---(?P=q)', re.DOTALL) def repl_script_frontmatter_keys(s): @@ -283,7 +283,7 @@ def repl_script_frontmatter_keys(s): fm,_ = frontmatter(new) return f"{m.group('q')}---\n{_dump_fm(fm)}---{m.group('q')}" + s[m.end():] -# %% ../nbs/api/21_octavo.ipynb #0c93c529 +# %% ../nbs/api/20_octavo.ipynb #0c93c529 def _apply_procs(txt, procs): for p in procs: txt = p(txt) return txt @@ -303,7 +303,7 @@ def migrate_md_file(path, procs): new_txt = _apply_procs(txt, procs) if new_txt!=txt: path.write_text(new_txt) -# %% ../nbs/api/21_octavo.ipynb #01ed763e +# %% ../nbs/api/20_octavo.ipynb #01ed763e def _nav_href(h): "Quarto link `h`, with a documentation source path changed to its published HTML page" if not h or '://' in h or h.startswith(('#','mailto:')): return h @@ -316,7 +316,7 @@ def _nav_item(item): elif 'href' in item: res['href'] = _nav_href(item['href']) return res -# %% ../nbs/api/21_octavo.ipynb #acd395a1 +# %% ../nbs/api/20_octavo.ipynb #acd395a1 _ICON_MAP = {'twitter':'x', 'twitter-x':'x'} _ICON_LABELS = {'github':'GitHub', 'rss':'RSS'} @@ -327,7 +327,7 @@ def _tool_item(item): if ico not in ICONS: return None return {'icon': ico, 'href': _nav_href(item['href']), 'label': item.get('aria-label') or item.get('text') or _ICON_LABELS.get(ico, ico.title())} -# %% ../nbs/api/21_octavo.ipynb #deb7ee06 +# %% ../nbs/api/20_octavo.ipynb #deb7ee06 def _quarto_config(nbs_path): qy_p,ny_p = nbs_path/'_quarto.yml',nbs_path/'nbdev.yml' qy = yaml.safe_load(qy_p.read_text()) if qy_p.exists() else {} @@ -336,7 +336,7 @@ def _quarto_config(nbs_path): if settings is not None and not isinstance(settings, dict): raise ValueError(f'{fname} must contain a YAML mapping') return qy or {},ny or {} -# %% ../nbs/api/21_octavo.ipynb #127739bd +# %% ../nbs/api/20_octavo.ipynb #127739bd def _octavo_output(nbs_path, qy, ny): "Return a Quarto output override relative to notebooks, or None when doc_path suffices" out_dir = (ny.get('project', {}) or {}).get('output-dir') or (qy.get('project', {}) or {}).get('output-dir') @@ -346,7 +346,7 @@ def _octavo_output(nbs_path, qy, ny): if out_dir == cfg.doc_path.resolve(): return return os.path.relpath(out_dir, Path(nbs_path).resolve()).replace(os.sep, '/') -# %% ../nbs/api/21_octavo.ipynb #201d0976 +# %% ../nbs/api/20_octavo.ipynb #201d0976 def _octavo_nav(nbs_path, qy, ny): "Convert Quarto navbar settings into Octavo navigation links and icon tools" cfg = {} @@ -359,7 +359,7 @@ def _nav(k): return nested_idx(qy,'website','navbar',k) or nested_idx(ny,'websit if (tools := tools.filter()): cfg['tools'] = list(tools) return cfg -# %% ../nbs/api/21_octavo.ipynb #e0912069 +# %% ../nbs/api/20_octavo.ipynb #e0912069 def _sidebar_item(o, drop): "An Octavo `sidebar_links` item from Quarto sidebar entry `o`; `drop` reports entries without an equivalent" if isinstance(o, str): o = dict(href=o) @@ -384,7 +384,7 @@ def drop(h): print(f'{p}: dropped sidebar entry {h!r}: Octavo has no equivalent. links = [i for o in listify(sb.get('contents')) if (i := _sidebar_item(o, drop))] return {'sidebar_links': links} if links else {} -# %% ../nbs/api/21_octavo.ipynb #2f2d4b3b +# %% ../nbs/api/20_octavo.ipynb #2f2d4b3b _IMAGE_KEYS = {'title', 'description', 'image', 'image-alt', 'image-width', 'image-height'} _SOCIAL_KEYS = {'open-graph': _IMAGE_KEYS | {'site-name', 'locale'}, 'twitter-card': _IMAGE_KEYS | {'card-style', 'creator', 'site'}} @@ -425,7 +425,7 @@ def _create_octavo_cfg(nbs_path): cfg.update(_octavo_social({**ws_q, **ws_n})) return cfg -# %% ../nbs/api/21_octavo.ipynb #c4b4478c +# %% ../nbs/api/20_octavo.ipynb #c4b4478c def _migrate_octavo(path): "Migrate documentation sources in place, create missing Octavo configuration, and return the rename mapping" path = Path(path).expanduser() @@ -454,7 +454,7 @@ def _migrate_octavo(path): if cfg: (path/'octavo.yml').write_text(yaml.dump(cfg, sort_keys=False)) return moves -# %% ../nbs/api/21_octavo.ipynb #e332ece4 +# %% ../nbs/api/20_octavo.ipynb #e332ece4 @call_parse def nbdev_migrate_octavo( path:str=None, # Documentation directory (defaults to configured nbs_path) diff --git a/nbs/api/21_octavo.ipynb b/nbs/api/20_octavo.ipynb similarity index 98% rename from nbs/api/21_octavo.ipynb rename to nbs/api/20_octavo.ipynb index afda72868..e458fc102 100644 --- a/nbs/api/21_octavo.ipynb +++ b/nbs/api/20_octavo.ipynb @@ -207,7 +207,16 @@ "execution_count": null, "id": "86824851", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rensdimmendaal/aai/ws/execnb/execnb/shell.py:172: UserWarning: Failed loading extension dialoghelper.bootstrap: Local skills already enabled for /Users/rensdimmendaal/aai/ws/nbdev/nbs/api\n", + " for w in warns: warnings.warn(w)\n" + ] + } + ], "source": [ "with tempfile.TemporaryDirectory() as d:\n", " project = Path(d)\n", @@ -493,7 +502,16 @@ "execution_count": null, "id": "bd46348b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rensdimmendaal/aai/ws/execnb/execnb/shell.py:172: UserWarning: Failed loading extension dialoghelper.bootstrap: Local skills already enabled for /Users/rensdimmendaal/aai/ws/nbdev/nbs/api\n", + " for w in warns: warnings.warn(w)\n" + ] + } + ], "source": [ "with working_directory(project): run(r'nbdev-docs --file-re \"\\.(?:ipynb|qmd|html)$\"')\n", "nbdev_docs(project, file_glob=None)" @@ -726,14 +744,14 @@ "```python\n", "{ 'cells': [ { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': 'c2daa09c',\n", + " 'id': 'a5015544',\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", " 'source': '# Example\\n\\n> Repository document'},\n", " { 'cell_type': 'markdown',\n", " 'directives_': {},\n", - " 'id': 'e050d919',\n", + " 'id': 'ecf07297',\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -744,7 +762,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': '61c6a532',\n", + " 'id': 'dcab9124',\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -756,7 +774,7 @@ " { 'cell_type': 'code',\n", " 'directives_': {},\n", " 'execution_count': None,\n", - " 'id': '3af1d12c',\n", + " 'id': 'bd88f480',\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", " 'metadata': {},\n", @@ -774,14 +792,14 @@ " 'metadata': {},\n", " 'idx_': 0,\n", " 'lang_': 'python',\n", - " 'id': 'c2daa09c'},\n", + " 'id': 'a5015544'},\n", " {'cell_type': 'markdown',\n", " 'source': 'Use `ModuleMaker`. [Guide](guide.md). [Reference][ref]\\n\\n[ref]: guide.md',\n", " 'directives_': {},\n", " 'metadata': {},\n", " 'idx_': 1,\n", " 'lang_': 'python',\n", - " 'id': 'e050d919'},\n", + " 'id': 'ecf07297'},\n", " {'cell_type': 'code',\n", " 'source': '#| echo: false\\nraise RuntimeError(\"must not execute\")',\n", " 'directives_': {},\n", @@ -792,7 +810,7 @@ " 'execution_count': None,\n", " 'idx_': 2,\n", " 'lang_': 'python',\n", - " 'id': '61c6a532'},\n", + " 'id': 'dcab9124'},\n", " {'cell_type': 'code',\n", " 'source': '#| include: false\\n\"excluded cell\"',\n", " 'directives_': {},\n", @@ -801,7 +819,7 @@ " 'execution_count': None,\n", " 'idx_': 3,\n", " 'lang_': 'python',\n", - " 'id': '3af1d12c'}],\n", + " 'id': 'bd88f480'}],\n", " 'metadata': {},\n", " 'nbformat': 4,\n", " 'nbformat_minor': 5}" @@ -829,7 +847,16 @@ "execution_count": null, "id": "7b518cd0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rensdimmendaal/aai/ws/execnb/execnb/shell.py:172: UserWarning: Failed loading extension dialoghelper.bootstrap: Local skills already enabled for /Users/rensdimmendaal/aai/ws/nbdev/nbs/api\n", + " for w in warns: warnings.warn(w)\n" + ] + } + ], "source": [ "cwd = Path.cwd()\n", "octavo_readme(project)\n", @@ -1632,8 +1659,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpbb06_xm6/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n", - "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmpbb06_xm6/nbs/sidebar.yml: dropped sidebar entry 'auto': Octavo has no equivalent. List those pages explicitly.\n" + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmp9ykzj8ii/nbs/octavo.yml: dropped tool icon 'bug' (https://x.y/issues): not bundled by octavo. Add it back with `icon: ` to your own SVG.\n", + "/private/var/folders/0h/5ktbjhg17ns6j8zgdsgvzvy80000gn/T/tmp9ykzj8ii/nbs/sidebar.yml: dropped sidebar entry 'auto': Octavo has no equivalent. List those pages explicitly.\n" ] } ], From 2f4fc414724aab864527171a16f33ba73d069a02 Mon Sep 17 00:00:00 2001 From: Rens Date: Wed, 14 Oct 2026 12:37:07 +0200 Subject: [PATCH 14/16] update prompt and keep quarto naming in frontmatter --- nbdev/_modidx.py | 4 +- nbdev/octavo.py | 178 ++++++++++----------- nbs/api/20_octavo.ipynb | 342 ++++++++++++++++++++++++---------------- 3 files changed, 298 insertions(+), 226 deletions(-) diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py index f7953ebf1..6083f391c 100644 --- a/nbdev/_modidx.py +++ b/nbdev/_modidx.py @@ -228,12 +228,13 @@ 'nbdev.octavo._octavo_nav': ('api/octavo.html#_octavo_nav', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_output': ('api/octavo.html#_octavo_output', 'nbdev/octavo.py'), 'nbdev.octavo._octavo_sidebar': ('api/octavo.html#_octavo_sidebar', 'nbdev/octavo.py'), - 'nbdev.octavo._octavo_social': ('api/octavo.html#_octavo_social', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_moves': ('api/octavo.html#_qmd_moves', 'nbdev/octavo.py'), 'nbdev.octavo._qmd_repls': ('api/octavo.html#_qmd_repls', 'nbdev/octavo.py'), 'nbdev.octavo._quarto_config': ('api/octavo.html#_quarto_config', 'nbdev/octavo.py'), 'nbdev.octavo._repl_cv_match': ('api/octavo.html#_repl_cv_match', 'nbdev/octavo.py'), + 'nbdev.octavo._script_md': ('api/octavo.html#_script_md', 'nbdev/octavo.py'), 'nbdev.octavo._sidebar_item': ('api/octavo.html#_sidebar_item', 'nbdev/octavo.py'), + 'nbdev.octavo._src_fm': ('api/octavo.html#_src_fm', 'nbdev/octavo.py'), 'nbdev.octavo._tool_item': ('api/octavo.html#_tool_item', 'nbdev/octavo.py'), 'nbdev.octavo.local_libraries': ('api/octavo.html#local_libraries', 'nbdev/octavo.py'), 'nbdev.octavo.migrate_md_file': ('api/octavo.html#migrate_md_file', 'nbdev/octavo.py'), @@ -250,6 +251,7 @@ 'nbdev.octavo.repl_qmd_links': ('api/octavo.html#repl_qmd_links', 'nbdev/octavo.py'), 'nbdev.octavo.repl_script_frontmatter_keys': ( 'api/octavo.html#repl_script_frontmatter_keys', 'nbdev/octavo.py'), + 'nbdev.octavo.report_page_keys': ('api/octavo.html#report_page_keys', 'nbdev/octavo.py'), 'nbdev.octavo.script_pages': ('api/octavo.html#script_pages', 'nbdev/octavo.py')}, 'nbdev.process': { 'nbdev.process.NBProcessor': ('api/process.html#nbprocessor', 'nbdev/process.py'), 'nbdev.process.NBProcessor.__init__': ('api/process.html#nbprocessor.__init__', 'nbdev/process.py'), diff --git a/nbdev/octavo.py b/nbdev/octavo.py index 71a7e75f0..0dd8cdf44 100644 --- a/nbdev/octavo.py +++ b/nbdev/octavo.py @@ -1,4 +1,4 @@ -"""Build nbdev documentation with Octavo +r"""Build nbdev documentation with Octavo Octavo is an alternative to Quarto for building a documentation website from nbdev notebooks. Install `nbdev[octavo]` to use it. A project selects Octavo by placing `octavo.yml` in its configured notebooks directory. Projects without that file continue to use Quarto and do not import Octavo. @@ -10,46 +10,61 @@ ## Migration instructions (share this with your AI) -Before migrating, save a built reference site. Octavo builds replace the output directory. Retaining `_quarto.yml` does not retain the rendered Quarto site or keep converted sources compatible with Quarto. +Octavo builds into the same output directory as Quarto and replaces what's there. The migration also converts the sources in place, and they won't build with Quarto afterwards, even if you keep `_quarto.yml`. So save the Quarto site first: -1. Ensure you have committed your latest changes or backed up your project -2. From the project root, run `nbdev-docs` while the project still uses Quarto. Confirm the build succeeds and inspect the site. Record existing failures rather than treating them as migration regressions. -3. Copy the complete generated site, including assets and search files, to a separate baseline directory outside the source and build directories. -4. Run `nbdev-migrate-octavo` from the project root. Review its changes and any executable scripts or hooks before attempting another build. -5. Run `nbdev-docs` again to build with Octavo. Keep the baseline untouched. If the build fails, retain its output for diagnosis. Use the manual migration tips below with the baseline path, new output path, and any errors. +1. Commit or back up the project. +2. From the project root, run `nbdev-docs` while the project still uses Quarto. Check that the build works, and note any failures it already has, so you don't mistake them for migration problems later. +3. Copy the whole built site, including assets and search files, to a directory outside the project's source and output directories. This copy is the baseline you compare against. +4. Run `nbdev-migrate-octavo` from the project root. Read its changes and everything it prints: it reports the settings it couldn't carry over. +5. Run `nbdev-docs` again to build with Octavo. If the build fails, keep its output for diagnosis. -Read the retained _quarto.yml, nbdev.yml, pyproject.toml, and generated octavo.yml. Inspect page metadata, stylesheets, render scripts, custom HTML, build hooks, and deployment workflows. Automatic migration renames sources and updates selected document references, directives, metadata, and site settings. It does not port the site's design. It does not copy format.html.css into octavo.yml. It leaves stylesheet files and page-layout metadata unchanged. Existing page-level CSS references can still load; inspect those too. An existing octavo.yml is preserved rather than regenerated. +`nbdev-migrate-octavo` renames `.qmd` sources to `.md`, updates references to them, converts directives, and moves a page's `subtitle` to `description`. Octavo reads Quarto's other front matter and settings names as they are. The migration writes `octavo.yml` from `_quarto.yml` and `nbdev.yml`, and leaves an existing `octavo.yml` alone. It doesn't port the site's design. It leaves stylesheet files unchanged, and it doesn't add the stylesheets listed under `format.html.css` in `_quarto.yml` to `octavo.yml`. Stylesheets that a page lists in its own front matter still load. -Identify which checks apply and propose a small migration plan. Inspect optional features only where the project uses them, such as custom homepages, blog listings, embeds, and tabs: +Read the original `_quarto.yml`, `nbdev.yml` and `pyproject.toml`, and the new `octavo.yml`. Go through the checks below that apply to the project, skipping features it doesn't use, and propose a short migration plan. Ask me before accepting anything that loses behavior. -- Styles and custom pages: inspect the rendered Octavo HTML before adapting selectors. Check responsive behavior and dependencies on Bootstrap or Quarto classes. Reuse compatible CSS selectively. Do not enable an old stylesheet wholesale or copy a compatibility layer without checking the current DOM. -- Layouts and themes: review page-layout, sidebars, titles, TOCs, and theme settings. Choose Octavo settings explicitly. Do not assume that renaming a layout preserves its meaning. Octavo's layout: custom is an explicit choice for a custom page, not a general Quarto layout adapter. Check ordinary documentation pages as well as any custom layouts. Compare toc and toc_depth settings with the original metadata, including explicit depth combined with toc: false. -- Embedded and conditional content: review the project's custom markup and format-conditional blocks. Compare when-format and unless-format behavior with the original sources in both HTML and Markdown output. Do not assume automatic conversion preserves exclusion conditions or renders markdown inside an HTML-targeted block. Where needed, adapt HTML-only content to raw {=html} blocks. Verify the result rather than rewriting every block by pattern. -- Render scripts and processors: if the project uses these, check their generated content and assets after migration. -- Navigation and identity: check navbar destinations, dropdowns, icons, favicon, footer, and existing public URLs. Restore omitted icons and preserve the existing footer. Markdown copies of notebook pages move from page.html.md to page.md, so external links to the old addresses break; Octavo does not redirect them. Report this change rather than working around it. -- Additional build outputs: if the project uses custom build hooks, preserve their required outputs using docs_post_build. Check links in any generated indexes. -- Code that reads documentation sources: migration renames .qmd sources to .md but does not update code that finds them by name. Search the whole project, including build scripts, tests, generators and contributor instructions, for .qmd, and update paths, globs and extension checks. A glob that no longer matches anything fails silently, so run the project's full test suite, not only nbdev-docs, and check that tests that read documentation pages still find them. Saved notebook outputs that contain .qmd links need to be regenerated after the fix. -- Published files: Octavo publishes every visible file in the notebooks directory (no path part starting with . or _), either as a page or as a copied file. Quarto published only referenced files and declared resources. Compare the output file list with the baseline. Add paths that should not be published, such as stray READMEs or notes, to exclude in octavo.yml: glob patterns relative to the notebooks directory, where a pattern also excludes everything below a matching directory. Do not exclude Quarto-only files; they are deleted after the review. -- Omitted settings: compare the original configuration with Octavo's configuration. List settings that were not carried over, including Open Graph and Twitter card options the migration reported as dropped, repository actions, theme options, and preview preferences where present. For each, propose a replacement or identify it as intentionally dropped or unsupported. Ask me to approve losses of behavior. -- Installation and deployment: workflows are not migrated. Verify an Octavo-capable nbdev version and the aai-octavo distribution; Python imports remain octavo. Check dependencies required by custom hooks. Verify the documented installation and docs build in a clean environment. Replace direct Quarto build commands where needed with nbdev-docs, including in contributor instructions. -- Output paths: an explicit relative output in octavo.yml is relative to the notebooks directory. Without that setting, nbdev uses its project-relative doc_path. Verify the resolved output directory and the directory published by the deployment workflow. +- **Styles and custom pages.** Octavo's HTML has no Bootstrap or Quarto classes. Look at the rendered Octavo HTML before reusing old CSS, and copy over only the rules that still apply. Don't enable an old stylesheet wholesale or add a compatibility layer. Check narrow screens too. +- **Wide content.** Octavo has no equivalent of Quarto's `page-layout: full` or column classes such as `.column-page` and `.column-screen-inset-right`. Widen those pages with your own CSS, in a stylesheet listed in the page's `css` front matter. +- **Layouts and tables of contents.** Octavo reads Quarto's `page-layout`, `toc` and `toc-depth` with the same meanings, but of the `page-layout` values it supports only `custom`, for a hand-built page. Check pages that used `article` or `full`, and compare tables of contents with the baseline on ordinary pages as well as custom ones. +- **Sidebar.** If the project has a hand-written `sidebar.yml` (`custom_sidebar` in `pyproject.toml`), the migration turns it into `sidebar-links` in `octavo.yml`. Compare its sections and order with the baseline. Add the pages behind any entries it reported as dropped, such as `auto` or globs, as explicit links. +- **Scripts.** The migration doesn't carry over scripts that Quarto added through `include-in-header`, `include-after-body` or raw HTML. Load them with Octavo's `js` setting, in `octavo.yml` for the whole site or in a page's front matter. Octavo loads them as module scripts (`