diff --git a/README.md b/README.md index fcc9159a..6dc37306 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,31 @@ The site uses [DocFX](https://dotnet.github.io/docfx/) and GitHub flavoured mark All contributions are welcome. We will review all pull requests submitted. -To test your changes locally: -- Make sure [DocFX](https://dotnet.github.io/docfx/) and Python 3.11+ are installed. -- Run `python build-docs.py --serve` in the root of the project. +For convenience for typical contributions, we have built a simple wrapper around the build process. +Unless you are working specifically on localization or the build process itself, you should be able to get by with the `run` script. -# Build Script Usage +Getting started: +1. Make sure you have Bash installed (included in most Linux distros, macOS, and [Git for Windows](https://git-scm.com/install/windows)) +2. From the repo root, run the setup check and install any tools it reports as missing + - (Linux distros and macOS): `./run setup` + - (Windows): `bash run setup` (after this run everything from Git Bash) +3. To iterate on docs and see a preview in your browser: + - in one terminal: `./run serve`: launches a localhost server that renders the docs at http://localhost:8080 where you can see the docs as they will be rendered on the docs website + - in another terminal: `./run watch`: regenerates the rendered site every time you save a change to a markdown document, so you can refresh your browser to see it + +These commands all build and work only on the English language docs, as the English markdown is our canonical version of documentation and the language we expect contributions in. + +For more info, try `./run help` and `./run help`. +If you'd like more detail, [check out the run script README](build_scripts/run_scripts/README.md). + +If you want to have more control over the build process, continue reading below about the `build-docs.py` script. + +# Advanced build Script Usage The `build-docs.py` script handles all documentation building tasks including multi-language support. +Make sure you have Python >=3.11 and docfx installed, the latter either globally or locally. -## Quick Start +## Using build-docs.py directly ```bash # Build and serve locally (English only, for development) @@ -50,6 +66,8 @@ swa start _site | `--serve` | Build and serve locally (English only, for development) | | `--skip-gen` | Skip running gen_redirects.py (use existing configs) | | `--no-api-copy` | Skip copying API docs to localized sites | +| `--skip-api` | Reuse existing API metadata in content/api, ~30-40% faster (local markdown iteration only, requires `--serve`/`--lang`; never for testing/CI/CD/releases) | +| `--permissive` | Don't treat English DocFX warnings as build failures (for local iteration; full/CI builds stay strict) | | `--sync` | Sync English fallback for missing/outdated translations (for local dev) | ## What the Build Script Does @@ -68,16 +86,24 @@ swa start _site # Project Structure ``` -TEDoc/ +/ ├── build-docs.py # Main build script +├── run # Task runner for common dev tasks (see build_scripts/run_scripts/README.md) ├── build_scripts/ # Helper scripts -│ ├── gen_redirects.py # Generates docfx.json configs +│ ├── check_links.py # Dead-link checker for the generated _site +│ ├── config_loader.py # Shared configuration loader for build scripts +│ ├── csharp_doctest.py # Validates annotated C# code blocks in docs against the te CLI │ ├── gen_languages.py # Generates language manifest -│ ├── gen_staticwebapp_config.py -│ ├── inject_seo_tags.py -│ ├── sync-localized-content.py +│ ├── gen_redirects.py # Generates docfx.json configs +│ ├── gen_sitemap_index.py # Post-processes the English sitemap; generates the sitemap index +│ ├── gen_staticwebapp_config.py # Generates Azure Static Web Apps routing config +│ ├── inject_seo_tags.py # Adds hreflang and canonical tags to built HTML │ ├── normalize-localized-alerts.py # Repairs Crowdin-collapsed DocFX alerts -│ └── normalize-localized-heading-anchors.py # Injects English-slug bookmark anchors into translations +│ ├── normalize-localized-heading-anchors.py # Injects English-slug bookmark anchors into translations +│ ├── sync-localized-content.py # Syncs English content into localized build dirs +│ ├── te_script_runner.py # Runs C# snippets against a throwaway model via the te CLI +│ ├── test-fixtures/ # Fixtures for the build-script tests +│ └── run_scripts/ # ./run subcommand scripts and shared lib.sh (see its README) ├── content/ # English source content (tracked in git) │ └── _ui-strings.json # English UI strings (header, footer, banners) ├── localizedContent/ # Build directories for all languages @@ -222,4 +248,3 @@ If a key is missing from a language's file, or no `_ui-strings.json` exists at a | `tableOfContents` | `Table of Contents` | Mobile TOC offcanvas title | | `selectLanguage` | `Select language` | Language picker label | | `copyCode` | `Copy code` | Code block copy button aria-label | - diff --git a/build_scripts/check_links.py b/build_scripts/check_links.py index 78ac8c71..a48b8c64 100644 --- a/build_scripts/check_links.py +++ b/build_scripts/check_links.py @@ -348,7 +348,7 @@ def fetch_external(url: str, need_body: bool) -> FetchResult: `_GET_ONLY_HOSTS`). GET downloads the body only when a fragment must be verified.""" target = "https:" + url if url.startswith("//") else url # protocol-relative //host needs a scheme get_only = need_body or _host_of(url) in _GET_ONLY_HOSTS - for method in (("GET",) if get_only else ("HEAD", "GET")): + for method in ("GET",) if get_only else ("HEAD", "GET"): request = urllib.request.Request(target, method=method, headers={"User-Agent": _USER_AGENT}) try: with urllib.request.urlopen(request, timeout=20) as response: @@ -642,7 +642,7 @@ def map_to_source(built: str, root: Path) -> str | None: def _anchor_member(type_stem: str, fragment: str) -> str: """Best-effort member name from a DocFX member anchor; empty when the anchor scheme does not match.""" prefix = type_stem.replace(".", "_") + "_" - return fragment[len(prefix):].split("_", 1)[0] if fragment.startswith(prefix) else "" + return fragment[len(prefix) :].split("_", 1)[0] if fragment.startswith(prefix) else "" def _fragment_suffix(target: Target) -> str: @@ -826,16 +826,20 @@ def rank(item: tuple[str, HostStats]) -> tuple[int, int, int, float]: return outcome.broken, outcome.partial, stat.rate_limited, stat.wait_seconds print(f"\n--- stats ({elapsed:.1f}s, {len(active)} hosts) ---") - print(f"{'total':>5} {'ok':>5} {'frag':>5} {'bad':>5} {'reqs':>5} {'429':>4} {'401':>4} {'403':>4} {'404':>4} " - f"{'oth':>4} {'net':>4} {'wait(s)':>8} {'fb':>4} {'cap':>4} host [retry-after seen]") + print( + f"{'total':>5} {'ok':>5} {'frag':>5} {'bad':>5} {'reqs':>5} {'429':>4} {'401':>4} {'403':>4} {'404':>4} " + f"{'oth':>4} {'net':>4} {'wait(s)':>8} {'fb':>4} {'cap':>4} host [retry-after seen]" + ) for host, stat in sorted(active.items(), key=rank, reverse=True): oc = outcomes.get(host, empty) seen = sorted(stat.retry_after_seen) tail = f" {seen}" if stat.rate_limited else "" - print(f"{oc.total:>5} {oc.ok:>5} {oc.partial:>5} {oc.broken:>5} {stat.requests:>5} {stat.rate_limited:>4} " - f"{oc.status.get(401, 0):>4} {oc.status.get(403, 0):>4} {oc.status.get(404, 0):>4} " - f"{oc.other_http:>4} {oc.transport:>4} {stat.wait_seconds:>8.1f} " - f"{stat.head_fallbacks:>4} {stat.final_cap:>4} {host}{tail}") + print( + f"{oc.total:>5} {oc.ok:>5} {oc.partial:>5} {oc.broken:>5} {stat.requests:>5} {stat.rate_limited:>4} " + f"{oc.status.get(401, 0):>4} {oc.status.get(403, 0):>4} {oc.status.get(404, 0):>4} " + f"{oc.other_http:>4} {oc.transport:>4} {stat.wait_seconds:>8.1f} " + f"{stat.head_fallbacks:>4} {stat.final_cap:>4} {host}{tail}" + ) def cmd_validate(args: list[str], progress: _Progress, base_url: "Callable[[], str]") -> int: @@ -851,10 +855,20 @@ def cmd_validate(args: list[str], progress: _Progress, base_url: "Callable[[], s under = next((arg[len("under=") :] for arg in args if arg.startswith("under=")), None) positional = [arg for arg in args if arg not in flags and not arg.startswith("under=")] root = Path(positional[0]) if positional else Path("_site") + if not root.is_dir(): + # A missing root would otherwise scan zero pages and "pass"; that + # silence would hide a forgotten or failed build. + print(f"error: site root not found: {root} (build the site first)", file=sys.stderr) + return 2 source_prefix = os.path.normpath(root / under) if under else None progress.phase = "enumerating" refs, anchors_by_file = enumerate_site(root, source_prefix, progress) + if progress.pages == 0: + # Same silence risk as a missing root: a site with no HTML means + # nothing was validated, not that nothing is broken. + print(f"error: no HTML pages found under {root} (build the site first)", file=sys.stderr) + return 2 existing = _existing_files(root) progress.phase = "resolving" # Resolve the base URL only when needed: local mode never checks live, so it stays offline and config-free. diff --git a/build_scripts/csharp_doctest.py b/build_scripts/csharp_doctest.py index 220c1be5..3b348790 100644 --- a/build_scripts/csharp_doctest.py +++ b/build_scripts/csharp_doctest.py @@ -352,8 +352,7 @@ def _classify_fence(fence: Fence, following: Fence | None) -> Block | None: first = fence.annotation.split(None, 1)[0] if fence.annotation else "" if first in ("compile", "run"): raise ValueError( - f"{first} annotation is only valid on a csharp block " - f"(line {fence.line}, lang={fence.lang or 'plain'})" + f"{first} annotation is only valid on a csharp block (line {fence.line}, lang={fence.lang or 'plain'})" ) return None annotation = fence.annotation diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md new file mode 100644 index 00000000..6926f5a1 --- /dev/null +++ b/build_scripts/run_scripts/README.md @@ -0,0 +1,481 @@ +# Docs tooling runner (`./run`) + +`./run` in the repo root dispatches to the command scripts in this directory. +Run `./run` (or `./run help`) for the full command list, and +`./run help` for details on one command. + +- macOS / Linux: `./run ` +- Windows: use Git Bash (installed with [Git for Windows](https://gitforwindows.org/)): + `./run `, or `bash run ` from any shell (e.g., PowerShell) that can invoke bash. + +`./run setup` checks that everything below is installed and reports all +problems in one pass. It installs nothing. + +**Most common commands**: +- once: `./run setup` +- when editing (in two terminals): + - `./run serve`: http://localhost:8080 with rendered docs + - `./run watch`: regenerate docs whenever a file changes (manually refresh in browser to see new docs rendered) + +## Requirements + +| Tool | Needed by | Notes | +|--------|-----------------------------------------------------|-------------------------------------------------------------| +| bash | everything | 3.2+; Git Bash on Windows; included on most other platforms | +| Python | `build`, `serve`, `watch`, `check-links`, `doctest` | 3.11 or newer | +| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | +| docfx | `build`, `serve`, `watch` | local dotnet tool or global | +| te | `doctest` (all but validate) | Tabular Editor CLI | + +Working on these scripts (python and shell scripts) needs a few more tools; +see [Run script development](#run-script-development) under Contributing below. + +### Installation + +We provide the most common and script-friendly installation methods per platform in each section below. +If you prefer, though, you can follow these links to each tool's official installation docs + +- Bash (Git for Windows): https://git-scm.com/install/windows +- Bash (non-windows): https://www.gnu.org/software/bash/ +- Python: https://www.python.org/ +- docfx: https://dotnet.github.io/docfx/ +- te (Tabular Editor CLI): https://tabulareditor.com/download-tabular-editor-cli + +#### Linux + +```bash +# Python from your distro, e.g. Debian/Ubuntu: +sudo apt-get install python3 +# .NET SDK (for docfx): see https://learn.microsoft.com/dotnet/core/install/linux +``` + +#### macOS + +```bash +brew install python3 +brew install --cask dotnet-sdk # .NET SDK (for docfx) +``` + +#### Windows + +Install [Git for Windows](https://gitforwindows.org/) for Git Bash, then: + +```powershell +winget install Python.PythonInstallManager +winget install Microsoft.DotNet.SDK.10 +``` + +Note: winget has no unversioned .NET SDK id; docfx needs SDK 8.0 or newer. + +#### docfx (all platforms) + +With the .NET SDK installed (see your platform above), install docfx as a +repo-local dotnet tool. From the repo root: + +```bash +dotnet tool install docfx +dotnet tool restore +``` + +If you are running a `dotnet` older than 10.0, run `dotnet new tool-manifest` first. + +#### te CLI (all platforms) + +The te CLI is installed the same way on every platform: +sign in at [tabulareditor.com](https://tabulareditor.com/download-tabular-editor-cli), +download the archive for your platform and architecture, extract it, +and put `te` on your PATH (or point `RUN_TE` at it). +Full instructions, including verification, live in this repo's own docs: +[te CLI install guide](../../content/features/te-cli/te-cli-install.md). + +## Usage + +The most common commands are provided below for reference. +All sub-commands provide help text at the command line. + +- Check all dependencies are installed: + - `./run setup`: run only to start and ensure you have the correct tools + - `RUN_TE=/path/to/te RUN_PYTHON=/path/to/python ./run setup`: (advanced usage) confirm that tools at custom paths are resolved correctly +- Hot reloading iteration on markdown content (run in two separate terminals); likely all you need for most content contribution: + - `./run serve`: builds docs site and launches a localhost server at http://localhost:8080 where you can browse rendered docs + - `./run watch`: watches for changes in template files and markdown files, runs a fast rebuild upon detecting changes +- One-off build (pick one): + - `./run build`: required after a `./run clean` or a fresh clone of the repo + - `./run build fast`: skip re-generating the API docs; what you probably want most of the time and saves 1/3-1/2 build time +- Check the built site for broken links (needs a built site: run `./run build` first): + - `./run check-links`: full check; fetches every unique external URL once, so it needs network access and can take a while + - `./run check-links local`: offline; on-disk files and anchors only +- Test the annotated C# code blocks in the docs (needs the te CLI); without file args, automatically discovers all markdown files with annotated C# code blocks: + - `./run doctest`: compile and run all annotated code blocks; compare results to `**Output**` blocks, but do not update files + - `./run doctest file1.md file2.md`: same, for exactly the named files + - `./run doctest update`: compile, run, and replace `**Output**` blocks in all markdown files with annotated code blocks + - `./run doctest update file1.md path/to/file2.md`: same, but only for the named files +- Clean build artifacts: `./run clean` +- Get help: + - `./run help`: print top-level help for the `run` dispatcher and list all subcommands + - `./run help`: print full help for `` + +### Dependency overrides + +Every external tool can be pinned to a specific executable with a `RUN_` environment variable: +`RUN_PYTHON`, `RUN_TE` (and, for the tooling developers' tools, `RUN_UVX`, `RUN_SHELLCHECK`, `RUN_SHFMT`). +When set, that path is the only candidate checked; there is no fallback to your `PATH` environment variable. +Example: + +```bash +RUN_PYTHON=/opt/python3.12/bin/python3 ./run build +RUN_PYTHON=/opt/python3.12/bin/python3 RUN_SHELLCHECK=/path/to/shellcheck ./run build +``` + +These environment variables are intended for testing specific versions or for users with these binaries not available in their `PATH`. + +docfx is resolved by build-docs.py itself: +a `DOCFX` environment variable, then a repo-local dotnet tool manifest, then a global `docfx` on PATH. + +## Contributing + +This section is specifically about contributing to the run scripts themselves. +For guidance on contributing documentation content, [see the project README](../../README.md#bookmark-links-and-translations) +and the authoring guidance below that section. + +### Run script development + +`./run scripts` and its nested commands are the gateway to run scripts about run scripts. + +- Check the script-development tools are installed: `./run scripts setup` +- Lint and verify formatting, without modifying anything (the check a PR should pass): + - `./run scripts check`: everything (all shell tooling, the qualified Python sources) + - `./run scripts check file1.sh file2.py ...`: specific files, routed by kind +- Apply the formatters (writes files): `./run scripts format [file ...]` +- Scaffold a new run script: `./run scripts new ` (creates `build_scripts/run_scripts/.sh` from the anatomy below, ready to edit) + +Additional requirements beyond the doc-contributor table above: + +| Tool | Needed by | Notes | +|------------|------------------------------|------------------------------------------| +| uvx | `scripts` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | +| shellcheck | `scripts` (shell sources) | | +| shfmt | `scripts` (shell formatting) | [mvdan/sh](https://github.com/mvdan/sh) | + +The first `./run scripts check` needs network access: uvx downloads ruff and mypy into its cache on first use. + +**Installation:** + +```bash +# Linux (Debian/Ubuntu): +sudo apt-get install shellcheck shfmt +curl -LsSf https://astral.sh/uv/install.sh | sh # uv provides uvx + +# macOS: +brew install uv shellcheck shfmt +``` + +```powershell +# Windows: +winget install astral-sh.uv +winget install koalaman.shellcheck +winget install mvdan.shfmt +``` + +### Conventions for these scripts + +- bash 3.2 compatible (macOS default bash), ASCII only, long-form CLI flags where a long form exists +- Shell formatted by `./run scripts format` + (shfmt -s: tab indentation, `name() {` function style, simplifications applied) +- Python tooling formatted with ruff format (also via `./run scripts format`); + lint rules, line length, strict mode, and the qualified-file lists live in pyproject.toml at the repo root + (the ruff `include` and mypy `files` lists sit adjacent there and must stay in sync) +- Each script: banner help block, per-function comment docs, awk-extracted help, `dispatch "$@"` at the bottom +- Shared helpers live in `lib.sh`; command-specific state lives in the command's script +- All of this tooling must pass `./run scripts check` + (shellcheck + shfmt diff for the shell sources; ruff, ruff format diff, and mypy for the Python sources) +- Any internal function not intended for re-use is prefixed with an underscore + +### General architecture + +Everything flows through the `run` command, +which is a dispatcher to various special-purpose scripts, exposed as subcommands to `run`. +Every subcommand is a script in this directory (run_scripts/). +The script must be named `name.sh` and `name` becomes the name of the sub-command; +there is no other registration for sub-commands, if the script exists, it's a subcommand. + +#### `run` dispatcher + +`run` is the script in the repo root, through which we invoke all run scripts. +It is intentionally minimal, and does only these things: + +**Happy path**: + +1. `cd` to the repo root, so every run script starts from the same working directory no matter where you invoke it from +2. resolve `` to `build_scripts/run_scripts/.sh` and invoke it with `bash`, passing all remaining arguments through verbatim + +**`help`**: Print help (`./run`, `./run help`) and the command list (`./run commands`, `./run --list`) + +**Error path**: Print an error plus the command list when the first argument does not resolve to a script + +The command list is discovered by scanning this directory for `*.sh` files (excluding `lib.sh`); +the one-line description for each command is pulled from its banner comment with `banner_title`. +Anything beyond the above belongs in a run script, not in `run`. + +#### `lib.sh` + +Shared helpers live in [lib.sh](./lib.sh); these are intended to be used for boilerplate and common operations across sub-commands: +- `info`: print informational messages onto stderr for the user +- `error`: print error messages prefixed with "ERROR: " +- `log_and_run`: prints the command being run in bold on stderr; use for all commands that actually get run for a scripts work +- `log_cmd`: the printing half of `log_and_run` on its own; + use when the command's execution is indirected (e.g. parallel workers whose output is captured, as in `doctest.sh`) +- `require`: automatically handle the env var for tool overrides and fail with info if the tool can't be found +- `run_var_name`: print the `RUN_` env var name for a tool name (the naming convention, defined in one place) +- `print_resolved`: print each given tool with the executable path its `RUN_` resolved to (for setup-style reports; call `require` first) +- `dispatch`: provide a way to take a default function with args (see usage in commands for examples) +- `banner_title`: print the first one-line summary of a comment banner at the top of a script file (for command lists) +- `banner_help`: print the whole comment banner at the top of a script file (for help and usage) +- `command_help`: print doc comments for each function in a script (for help output) + +If you are writing or editing a run script, you should use these helpers. +If there's some new generic requirement, that's a decent candidate to be added to `lib.sh`. + +#### Anatomy of a run script + +All run scripts must: +- be named `.sh` in this directory; they are automatically discovered based on that naming convention. +- have a `#!/usr/bin/env bash` shebang, and are marked executable with `chmod +x`. +- have a comment banner with a one-line summary, help text, and usage examples +- contain the header below +- use shell functions for all functionality +- contain a standard `help` function +- use the `dispatch` pattern + +##### Banner comment + +A banner comment is of the form below. +This becomes the help text for this script. + +```shell +###################################### (at least 10 octothorpes starting at character 0) +# One line command summary +# +# all banner comment lines must start with an octothorpe. +# provide descriptive help text +# explain what this run script does +# give any relevant details for a user running this command +# not information about the development of the command +# +# Usage: +# ./run command # what this invocation does +# ./run command arg # what *this* one does +###################################### (at least 10 octothorpes starting at character 0) +``` + +##### Script header + +This: +1. sets up some safety defaults for Bash +2. makes sure that all scripts are executing in the same working directory context +3. sources `lib.sh` so we have the shared helpers + +```shell +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." +``` + +##### Implementation functions + +Anything that looks like an argument or a sub-subcommand is implemented as a Bash function. +The very first thing in the function definition must be comments that become the help text for this function. + +Example: + +```shell +fn() { + # the help text for this function + # can span many lines + require ... + +``` + +These functions must declare their required tools upfront with `require`, e.g., `require uvx`; +do not use a path here, but only the exact binary name. +The sole exception is Python, which must always be written as "python" verbatim (`require` handles platform specifics for this). + +Examples: +- `require uvx` +- `require python` +- `require python shellcheck uvx` + +Using `require` means we get automatic environment variable handling for these things, and helpful error messages when the binaries are not available. + +##### Using the `RUN_` variables that `require` sets up + +`require` does two jobs. +1. It checks that each named tool is available, + honoring the user's `RUN_` override as described in [Dependency overrides](#dependency-overrides), +2. It exports a `RUN_` environment variable holding the winner. + The variable name is the tool name uppercased, with dashes turned into underscores: + `require uvx` exports `RUN_UVX`, `require shellcheck` exports `RUN_SHELLCHECK`, `require python` exports `RUN_PYTHON`. + +Always invoke a required tool through its variable, never by its bare name: + +```shell +python () { + # ruff on the Python tooling + require uvx + log_and_run "$RUN_UVX" ruff check --quiet "$@" +} +``` + +Invoking `"$RUN_UVX"` (quotes included; the path may contain spaces) is what makes a user's override actually take effect. +A bare `uvx` on that line would search the user's `PATH`, but if they provided `RUN_UVX=/other/path/to/uvx ./run scripts check`, +then they are explicitly telling us, "Use the `uvx` at the provided path, not from `PATH`," +so we honor that. +These exports also flow into any script the current one delegates to (e.g., `watch.sh` runs `build.sh fast`, which sees the same `RUN_PYTHON`), +so an override applies consistently across a whole command. + +###### How `require` failures behave + +On failure, `require` prints what is missing and a pointer to this README. +When there are multiple arguments to `require`, e.g., `require python uvx`, it resolves and sets all variables. +If one or more tools are not found, then the error printed to the terminal includes details about all missing tools. +When it doesn't find all required tools then it **returns** nonzero; it never exits the script itself. +In the normal placement, at the top of a function that the user invoked directly, that is all you need: +the script header's `set -euo pipefail` turns the non-0 `return` into a clean stop. + +The exception is a function that gets called inside a conditional (`if`, `!`, `&&`, `||`). +Bash suspends `set -e` for everything running inside a condition, including called functions, so a failing `require` there does not stop anything; +execution falls through to the next line as if nothing happened. +In that placement, you must propagate the failure explicitly: + +```shell +require dotnet || return 1 +``` + +`setup.sh` is the resident example: its `check` function probes helpers inside `if !` conditionals, so those helpers use the explicit form. + +##### Standard `help` function + +This uses `lib.sh` helpers to give a standardized help text. +These pull help text out of the banner comment and all leading comments in functions. +They are automatically formatted as you see when you run `./run help` or `./run subcommand help`. + +```shell +help() { + # show this help + banner_help "$0" + command_help "$0" +} +``` + +##### `dispatch` pattern + +The last line of every run script is a call to `dispatch`: + +```shell +dispatch "$@" +``` + +Read it as: "look at what the user typed after `./run `, and call the function with that name". + +Take `build.sh`, which ends with `dispatch full "$@"`, as a worked example. +`"$@"` stands for "everything the user typed after `./run build`", so: + +- When a user submits `./run build`, there is nothing after the subcommand, so we end up with `dispatch full` and nothing in `"$@"`. + With no user input to look at, `dispatch` falls back to the default it was given and calls the `full` function. +- When a user submits `./run build fast`, we end up with `dispatch full fast`. + `dispatch` sees the user's `fast`, finds a function with that name in the script, and calls `fast`; the `full` default is ignored. +- When a user submits `./run build nonsense`, we end up with `dispatch full nonsense`. + There is no `nonsense` function in the script, so dispatch prints an error, then the script's `help`, and exits with a usage error. +- Anything after the function name is handed to that function as its arguments. + In `scripts.sh` (default `check`), `./run scripts format a.py b.sh` ends up as `dispatch check format a.py b.sh`; + dispatch calls the `format` function with `a.py b.sh` as its arguments. + +Name the default function after what it does (`full`, `all`, `check`), not a generic `main`; +function names are user interface here, since they are exactly what a user types after `./run `. + +There are exactly two correct ways to write the dispatch line: + +1. The standard form shown above: `dispatch "$@"`. + The default function name is required; `dispatch` always needs to know what to run when the user types nothing. +2. A wrapper around form 1, for a script that wants to handle unrecognized arguments itself instead of treating them as an error. + For example, `scripts.sh` treats an unrecognized first argument as a list of files to check: + +```shell +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" +else + check "$@" +fi +``` + +Two mistakes to avoid: + +- `dispatch "$@"` (leaving out the default) is broken, not just incomplete: + `dispatch` would mistake the user's second argument for the name of the function to call, or crash the script when there are no arguments at all. + If a script has no obvious default action, make `help` its default. +- `"$@"` must always be written with the quotes. + Without them, an argument containing a space (such as a file path) gets split into pieces before `dispatch` ever sees it. + + +### Shell script or python? + +We use shell scripts primarily for orchestration of other tools. +If you just want to provide a friendly wrapper for some more complex command, +or a bunch of reasonable defaults, then a shell script in this directory is the right choice. +If there is significant logic, data processing, or any sort of parsing, then a python script is probably the right choice. +If you build a python script that is a reasonable tool for most contributors to use, +then you should also give it a friendly run script wrapper here. +A good example of a run script wrapper improving contributor convenience is [doctest.sh](./doctest.sh); +this wraps a python tool that can only operate on one file, +and gives a run script interface that lets you pass many files, +calling the python script once for each. + +### Style guidelines + +Write everything as a series of small functions, +using dispatch patterns as in these run scripts and demonstrated in [`check_links.py`](../check_links.py), +[`te_script_runner.py`](../te_script_runner.py), and [`csharp_doctest.py`](../csharp_doctest.py). +This allows incremental testing and validation: +small pieces are built that do a part of the work and their logic is demonstrated to be sound; +then downstream components compose and orchestrate this functionality. +This style supports incremental validation, as well as ad hoc reuse of components. +We do not follow TDD (there are not test suites for these scripts), +but following this approach still keeps the code test*able*, which is critical. + + +General conventions and guidelines. + +- Output routing: stdout carries only a command's real output; + all chatter (progress, status, errors) goes to stderr via `info`/`error`, + so output stays clean for piping and redirection. +- Success is quiet: pick tool flags that silence success output + (e.g. ruff `--quiet`, mypy `--no-error-summary`), + or capture a step's report and print it only on failure (as `doctest.sh` does). + Anything printed should mean something needs attention. +- Ensure failures have non-0 exit stati; + for anything in a pipeline or other automation, this is the sole reliable indicator of failure. +- `log_and_run` executed commands: every command that performs a script's actual work runs through `log_and_run`, + or announces itself with `log_cmd` when its execution is indirected (see the parallel workers in `doctest.sh`); + silent probes and internal plumbing do not. + When delegating to another script, let the delegate echo its own commands: see `watch.sh` and `build.sh` +- Report everything at once: collect all failures (missing tools, broken files) and report them together, + one per line, before failing; this allows a user to remediate as much as possible at once, + rather than chasing one error at a time. + If there is useful documentation to point to in an error context, do so; this README is a good choice. +- No silent false successes: a checker that found nothing to check must fail or say so loudly, + never pass quietly (see the missing-root and zero-pages guards in `check_links.py`). +- Explicit user input always overrides defaults: if a user gives us filenames or sets an env var, + we use exactly what they have provided, rather than automatic discovery or default file lists. + See `RUN_` handling and all commands that accept file args. +- Run scripts should always accept multiple files, and route these files intelligently; + see the mixed .py and .sh handling in `scripts.sh`. +- A wrapper is a chance to fix warts in what it wraps, not to reproduce them; + when porting or wrapping existing behavior surfaces a bug, fix it. +- Use constructs portable across the BSD and GNU userlands: + plain `tr` ranges under `LC_ALL=C` rather than `[:class:]`-plus-literal sets, + and streaming filters rather than `sed -i` (whose flags differ between the two). +- Keep docs, comments, and implementation in sync. +- Ensure python scripts remain cleanly importable by using setup functions guarded in `if __name__ == '__main__':`. diff --git a/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE b/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE new file mode 100644 index 00000000..0cec5ee0 --- /dev/null +++ b/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +################################################################################ +# TODO one-line summary, shown by './run commands' and './run help' +# +# TODO describe what this command does for the user running it. +# +# TODO list all RUN_ environment vars applicable to this command. +# +# Usage: +# ./run TODO +# ./run TODO help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +TODO_FUNCTION_NAME() { + # TODO help text for this function + # TODO declare tools upfront: require + require TODO + error 'not implemented' + return 1 #TODO return 0 on success +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch TODO_FUNCTION_DEFAULT "$@" diff --git a/build_scripts/run_scripts/build.sh b/build_scripts/run_scripts/build.sh new file mode 100755 index 00000000..d9402ee3 --- /dev/null +++ b/build_scripts/run_scripts/build.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +################################################################################ +# Build the English docs for local iteration +# +# Wraps the canonical build-docs.py orchestration; no parallel build logic. +# Never touches the localized es/zh content. +# +# full (default) regenerate the API reference; reuse the generated +# docfx.json configs; fail on DocFX warnings +# fast also reuse existing API metadata and tolerate DocFX warnings; +# the same build that `watch` runs +# +# Tool overrides (env vars): RUN_PYTHON. +# +# Usage: +# ./run build [full|fast] +# ./run build help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +common_flags() { + # Set FLAGS to the markdown-iteration base: English only, and reuse the + # generated docfx.json configs (--skip-gen) when they exist. After a + # `clean` they do not, so the generation step runs on the next build. + FLAGS=(--lang en) + if [ -f localizedContent/en/docfx.json ] && [ -f metadata/languages.json ]; then + FLAGS+=(--skip-gen) + fi +} + +full() { + # regenerate the API reference; fail on DocFX warnings + require python + common_flags + log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" +} + +fast() { + # reuse existing API metadata (--skip-api); tolerate DocFX warnings + # (--permissive); the build that watch runs after each save + require python + common_flags + log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" --skip-api --permissive +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch full "$@" diff --git a/build_scripts/run_scripts/check-links.sh b/build_scripts/run_scripts/check-links.sh new file mode 100755 index 00000000..b0973716 --- /dev/null +++ b/build_scripts/run_scripts/check-links.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +################################################################################ +# Check the built site for broken links +# +# Wraps build_scripts/check_links.py: walks the built HTML in _site, resolves +# every href/src to a local file or external URL, and reports references to +# broken targets. Local failures (missing files or anchors) are errors and +# fail the run; external failures are warnings, listed for verification by +# hand. Needs a built site: run `./run build` first. +# +# The full check fetches every unique external URL once, so it needs network +# access and can take a while; `local` stays entirely on disk. +# +# Arguments pass through to check_links.py validate: +# local skip external fetching (on-disk checks only; offline) +# all include generated API and localized pages +# (default: authored content only) +# stats print per-host fetch diagnostics +# under= only check links from pages under _site/ +# +# Tool overrides (env vars): RUN_PYTHON. +# +# Usage: +# ./run check-links # full check (fetches external URLs) +# ./run check-links local # offline: on-disk checks only +# ./run check-links local all # offline, including generated pages +# ./run check-links help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +check() { + # validate the built site; args pass through to check_links.py validate + require python + log_and_run "$RUN_PYTHON" build_scripts/check_links.py validate "$@" +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else passes through +# to `check` as validate arguments, so `./run check-links local` just works. +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" +else + check "$@" +fi diff --git a/build_scripts/run_scripts/clean.sh b/build_scripts/run_scripts/clean.sh new file mode 100755 index 00000000..7b299cb9 --- /dev/null +++ b/build_scripts/run_scripts/clean.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +################################################################################ +# Remove generated build artifacts +# +# Deletes the DocFX build outputs and generated configs: _site, obj, the +# generated docfx.json files, generated API reference yml, language metadata, +# and the localized-content staging outputs. Everything removed is gitignored; +# tracked source is never touched. +# +# Usage: +# ./run clean +# ./run clean help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +clean() { + # remove all generated build artifacts (gitignored); tracked source is + # left intact + log_and_run rm -rf _site obj docfx.json + log_and_run rm -f content/api/*.yml content/api/.manifest + log_and_run rm -f metadata/languages.json + log_and_run rm -rf localizedContent/en + log_and_run find localizedContent -maxdepth 2 -name docfx.json -delete || true + # Generated files only: each localized content/api keeps a tracked + # index.md, so the whole directory must not be removed. + log_and_run rm -f localizedContent/*/content/api/*.yml localizedContent/*/content/api/.manifest + log_and_run rm -rf localizedContent/*/content/assets + info "Clean complete." +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch clean "$@" diff --git a/build_scripts/run_scripts/doctest.sh b/build_scripts/run_scripts/doctest.sh new file mode 100755 index 00000000..0bf3318b --- /dev/null +++ b/build_scripts/run_scripts/doctest.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +################################################################################ +# Test the annotated C# code blocks in the docs against the te CLI +# +# Wraps build_scripts/csharp_doctest.py. Code blocks in content markdown opt +# in with a fenced-code annotation (```csharp {compile} or +# ```csharp {run ...}); see the header of csharp_doctest.py for the +# annotation reference. With no file arguments, every annotated markdown +# file under content/ is processed; having none is fine (reported, exit 0). +# +# Files are processed in parallel, at most DOCTEST_JOBS at a time (default +# 4; each invocation runs against its own throwaway model, and update only +# writes its own markdown file). Results print in the order files finish: +# every file's command line, plus the full report (including te's own +# stderr) only when that file failed. The run exits nonzero if any file +# failed. +# +# Needs the Tabular Editor CLI for everything except validate: `te` on PATH, +# or RUN_TE=/path/to/te. update REWRITES the documented outputs inside the +# markdown files. +# +# NOTE: The correctness of the results of doctest` depend entirely on the +# alignment of your te CLI version with the scripting surface area +# being documented. If your te version is out of date, you may see false +# error reports from this command. +# +# Usage: +# ./run doctest [file.md ...] # compare (the default) +# ./run doctest validate [file.md ...] # annotation coverage; no te needed +# ./run doctest compile [file.md ...] # compile-only (te --dry-run) +# ./run doctest update [file.md ...] # execute and rewrite doc outputs +# DOCTEST_JOBS=N ./run doctest # run at most N jobs at once +# ./run doctest help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +_annotated_files() { + # print the content markdown files containing annotated csharp fences + grep --recursive --files-with-matches --include='*.md' -- '```csharp {' content || true +} + +_run_one() { + # xargs worker: run one file, echo its command line on completion, and + # print its captured report (stdout and stderr, so parallel te chatter + # never interleaves) only on failure, as a single write so concurrent + # reports stay intact. + local -a cmd=("$RUN_PYTHON" build_scripts/csharp_doctest.py "$1" "$2") + local out + if out="$("${cmd[@]}" 2>&1)"; then + log_cmd "${cmd[@]}" + else + log_cmd "${cmd[@]}" + printf '%s\n' "$out" + return 1 + fi +} + +_run_mode() { + # run csharp_doctest.py once per file (it takes exactly one file + # per invocation), at most DOCTEST_JOBS files at a time. Each file's + # own blocks already run in parallel inside csharp_doctest.py (one te + # per block), so an unbounded file-level fan-out would oversubscribe + # the machine. Explicit file arguments always win; discovery of + # annotated content files fills in only when no files are given. + local mode="$1" + shift + local files=("$@") f + if [ "${#files[@]}" -eq 0 ]; then + while IFS= read -r f; do + files+=("$f") + done < <(_annotated_files) + fi + if [ "${#files[@]}" -eq 0 ]; then + info 'doctest: no annotated C# blocks found under content/; nothing to do' + return 0 + fi + + # The xargs-spawned worker shells receive these through the environment. + export -f _run_one log_cmd + + # xargs runs the workers and refills free slots as files finish, so + # results print in finish order. Embedding $mode is safe: dispatch + # already resolved it to one of this script's function names. xargs + # exits 123 when any invocation failed (BSD and GNU alike). + local status=0 + printf '%s\0' "${files[@]}" | + xargs -0 -n 1 -P "${DOCTEST_JOBS:-4}" bash -c "_run_one $mode \"\$1\"" _ || status=1 + return "$status" +} + +validate() { + # check annotation correctness: report on malformed annotations. + # no te requirement + require python + _run_mode validate "$@" +} + +compile() { + # compile every annotated block via te --dry-run; catches API drift + # without executing anything + require python te + _run_mode compile "$@" +} + +compare() { + # execute annotated blocks and compare their documented outputs + require python te + _run_mode compare "$@" +} + +update() { + # execute annotated blocks and REWRITE the documented outputs in the + # markdown files in place + require python te + _run_mode update "$@" +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else passes through +# to `compare` as file paths, so `./run doctest some-doc.md` just works. +if declare -F "${1:-compare}" >/dev/null; then + dispatch compare "$@" +else + compare "$@" +fi diff --git a/build_scripts/run_scripts/lib.sh b/build_scripts/run_scripts/lib.sh new file mode 100755 index 00000000..2bfcfc49 --- /dev/null +++ b/build_scripts/run_scripts/lib.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# Shared helpers for the ./run command scripts. +# +# Sourced as a library by the command scripts in this directory, or executed +# directly to exercise a single helper: +# bash build_scripts/run_scripts/lib.sh require git python3 + +# Status/progress chatter, on stderr so stdout stays clean for command output. +info() { printf '%s\n' "$*" >&2; } + +# Error message with a uniform prefix, on stderr. +error() { printf 'ERROR: %s\n' "$*" >&2; } + +# Print a command line without running it, bold (like just's recipe echo) +# with a "$ " prefix. Goes to stderr so a command's stdout stays clean for +# its own output; escape codes only when stderr is a terminal and NO_COLOR +# is unset, so redirected output stays plain. Use directly when the command +# itself runs indirected (e.g. backgrounded with captured output). +log_cmd() { + if [ -t 2 ] && [ -z "${NO_COLOR:-}" ]; then + printf '\033[1m$ %s\033[0m\n' "$*" >&2 + else + printf '$ %s\n' "$*" >&2 + fi +} + +# Print a command then execute it. +log_and_run() { + log_cmd "$@" + "$@" +} + +run_var_name() { + # Print the RUN_ variable name for a tool (uppercased, dashes to + # underscores: uvx -> RUN_UVX). The single source of the require naming + # convention. LC_ALL=C pins ASCII range semantics regardless of locale. + printf 'RUN_%s' "$(printf '%s' "$1" | LC_ALL=C tr 'a-z-' 'A-Z_')" +} + +require() { + # Check the given tools and resolve which executable to use for each. + # Takes bare tool names, not paths. A pre-set RUN_ env var (name + # uppercased, dashes to underscores: RUN_UVX for uvx, RUN_SHELLCHECK + # for shellcheck) is a user override and is the only candidate checked + # otherwise the tool must be on PATH. The winner is exported as + # RUN_ for callers to invoke and for delegated scripts to + # inherit. Failures are reported all at once. + # The name `python` is special: it delegates to _require_python, which + # version-gates the interpreter and reports its own detail. + local missing=() cmd var val failed=0 + for cmd in "$@"; do + if [ "$cmd" = "python" ]; then + _require_python || failed=1 + continue + fi + var="$(run_var_name "$cmd")" + val="${!var:-$cmd}" + if command -v "$val" >/dev/null 2>&1; then + export "$var=$val" + elif [ "$val" = "$cmd" ]; then + missing+=("$cmd") + else + missing+=("$cmd ($var=$val not found)") + fi + done + if [ "${#missing[@]}" -gt 0 ]; then + error 'missing required command(s):' + for cmd in "${missing[@]}"; do + info " $cmd" + done + info 'See build_scripts/run_scripts/README.md for installation instructions.' + return 1 + fi + return "$failed" +} + +_python_ok() { + # True if the given interpreter exists and is Python >= 3.11 (the + # version floor for the repo's build tooling, matching CI). + command -v "$1" >/dev/null 2>&1 && + "$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' >/dev/null 2>&1 +} + +_require_python() { + # Locate a Python >= 3.11 and export it as RUN_PYTHON for commands to + # invoke. When RUN_PYTHON is already set (contributor override) it is + # the ONLY candidate checked, so a broken override fails loudly instead + # of silently falling back. Otherwise probe python3, python, then the + # Windows launcher (py -3, resolved to its underlying executable so + # RUN_PYTHON stays a single quotable word). On failure, reports the + # version of every candidate that was present. + local cand tried="" + if [ -n "${RUN_PYTHON:-}" ]; then + if _python_ok "$RUN_PYTHON"; then + export RUN_PYTHON + return 0 + fi + error "RUN_PYTHON=$RUN_PYTHON is not a Python >= 3.11." + if command -v "$RUN_PYTHON" >/dev/null 2>&1; then + info " found: $("$RUN_PYTHON" --version 2>&1 || true)" + fi + info 'See build_scripts/run_scripts/README.md for installation instructions.' + return 1 + fi + + for cand in python3 python; do + if _python_ok "$cand"; then + RUN_PYTHON="$cand" + export RUN_PYTHON + return 0 + fi + if command -v "$cand" >/dev/null 2>&1; then + tried="$tried + $cand: $("$cand" --version 2>&1 || true)" + fi + done + + if command -v py >/dev/null 2>&1; then + cand="$(py -3 -c 'import sys; print(sys.executable)' 2>/dev/null || true)" + if [ -n "$cand" ] && _python_ok "$cand"; then + RUN_PYTHON="$cand" + export RUN_PYTHON + return 0 + fi + tried="$tried + py -3: $(py -3 --version 2>&1 || true)" + fi + + error "Python >= 3.11 is required and was not found." + if [ -n "$tried" ]; then + info "Candidates checked:$tried" + fi + info 'Set RUN_PYTHON to a suitable interpreter, or see build_scripts/run_scripts/README.md for installation instructions.' + return 1 +} + +print_resolved() { + # Print each tool name with the executable path its RUN_ variable + # resolved to. Call require on the tools first. + local tool var + for tool in "$@"; do + var="$(run_var_name "$tool")" + printf ' %-11s %s\n' "$tool" "$(command -v "${!var}")" + done +} + +dispatch() { + # dispatch [args...]: call the function named by the first + # remaining arg, or default_fn when none is given. Unknown names print + # the calling script's help and fail with a usage error. + local fn="${2:-$1}" + shift + if [ "$#" -gt 0 ]; then + shift + fi + if declare -F "$fn" >/dev/null; then + "$fn" "$@" + else + error "unknown command: $fn" + if declare -F help >/dev/null; then + help + fi + return 2 + fi +} + +banner_title() { + # Print only the first line of the #### banner block of the given script + # (used by ./run for its one-line-per-command index). + awk ' + /^#{10,}/ { if (inIntro) exit; inIntro=1; next } + inIntro && /^#/ { gsub(/^# ?/, ""); print; exit } + ' "$1" +} + +banner_help() { + # Print the #### banner block at the top of the given script, unprefixed. + awk ' + BEGIN { inIntro=0; introDone=0 } + /^#{10,}/ && introDone==0 && inIntro==0 { inIntro=1; next } + /^#{10,}/ && inIntro==1 { inIntro=0; introDone=1; print ""; next } + inIntro==1 { gsub(/^# ?/, ""); print } + ' "$1" +} + +command_help() { + # Print each function in the given script with its leading comment lines. + # Underscore-prefixed functions are internal and excluded. + printf 'Commands:\n' + awk ' + /^[a-z][a-z_]* *\(\) *\{/ { fn=$1; sub(/\(\)$/, "", fn); inCmd=1; next } + inCmd==1 && /^[ \t]+#/ { + if (fn != "") { print " " fn; fn="" } + gsub(/^[ \t]+# ?/, " ") + print + next + } + inCmd==1 { inCmd=0 } + ' "$1" +} + +# Dispatch only when executed directly; stay silent when sourced. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + set -euo pipefail + "$@" +fi diff --git a/build_scripts/run_scripts/scripts.sh b/build_scripts/run_scripts/scripts.sh new file mode 100755 index 00000000..65fad25b --- /dev/null +++ b/build_scripts/run_scripts/scripts.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +################################################################################ +# Develop the repo tooling: check, format, and scaffold the scripts themselves +# +# Commands for working on the repo's own tooling: the run scripts and the +# qualified Python build scripts. Contributing docs does not need any of +# this; see `./run setup` for the doc-contributor checks. +# +# Python rules, line length, strict mode, and the qualified-file lists live +# in pyproject.toml at the repo root; the shell file list is defined at the +# top of this script. File arguments are routed by kind (.py to the Python +# tools; .sh and the `run` dispatcher to the shell tools) and always trump +# the configured lists. +# +# Tool overrides (env vars): RUN_UVX, RUN_SHELLCHECK, RUN_SHFMT. +# +# Usage: +# ./run scripts # check: linters + formatter diffs +# ./run scripts check [file ...] # same, explicitly +# ./run scripts format [file ...] # apply the formatters (writes files) +# ./run scripts new # scaffold a new run script +# ./run scripts setup # check the script-development tools +# ./run scripts help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +# Default shell source list: the tooling that meets the check bar. The Python +# equivalent lives in pyproject.toml (ruff include / mypy files). +SHELL_SOURCES=(run build_scripts/run_scripts/*.sh) + +_check_python() { + # non-mutating: ruff lint, ruff format diff, mypy (slowest last) + require uvx + log_and_run "$RUN_UVX" ruff check --quiet "$@" + log_and_run "$RUN_UVX" ruff format --check --quiet "$@" + # --no-error-summary drops only the trailing tally line ("Found N errors + # ..."/"Success: ..."); per-error diagnostics and the exit code are + # unaffected, so success is silent like shellcheck. + log_and_run "$RUN_UVX" mypy --no-error-summary "$@" +} + +_check_shell() { + # non-mutating: shellcheck, then shfmt as a diff + require shellcheck shfmt + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${SHELL_SOURCES[@]}") + fi + log_and_run "$RUN_SHELLCHECK" --external-sources "${files[@]}" + # -s simplifies the code where possible; -d prints a diff and fails + # when formatting is off (shfmt has no long-form flags) + log_and_run "$RUN_SHFMT" -s -d "${files[@]}" +} + +_format_python() { + require uvx + log_and_run "$RUN_UVX" ruff format "$@" +} + +_format_shell() { + require shfmt + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${SHELL_SOURCES[@]}") + fi + log_and_run "$RUN_SHFMT" -s -w "${files[@]}" +} + +_route() { + # Split file args by kind and hand them to the given python/shell + # functions. With no files, call both with none, so each falls back to + # its configured or default list. + local python_fn="$1" shell_fn="$2" + shift 2 + if [ "$#" -eq 0 ]; then + "$python_fn" + "$shell_fn" + return + fi + local py=() sh=() path + for path in "$@"; do + case "$path" in + *.py) py+=("$path") ;; + *.sh | run | */run) sh+=("$path") ;; + *) info "scripts: ignoring path of unrecognized kind: $path" ;; + esac + done + if [ "${#py[@]}" -eq 0 ] && [ "${#sh[@]}" -eq 0 ]; then + error "no recognized files among: $*" + return 2 + fi + if [ "${#py[@]}" -gt 0 ]; then + "$python_fn" "${py[@]}" + fi + if [ "${#sh[@]}" -gt 0 ]; then + "$shell_fn" "${sh[@]}" + fi +} + +check() { + # lint and verify formatting without modifying anything: shellcheck + + # shfmt diff on the shell tooling, ruff + format diff + mypy on the + # Python tooling + _route _check_python _check_shell "$@" +} + +format() { + # apply the formatters (WRITES files): shfmt on the shell tooling, + # ruff format on the Python tooling + _route _format_python _format_shell "$@" +} + +new() { + # scaffold a new run script with the standard anatomy and mark it + # executable; the name becomes the subcommand (./run ) + local name="${1:-}" + case "$name" in + "" | *[!a-z0-9-]* | -* | *-) + error 'usage: ./run scripts new (lowercase letters, digits, and inner dashes)' + return 2 + ;; + help | commands) + # the run dispatcher answers these itself, so a script by either + # name would be unreachable + error "$name is reserved by the run dispatcher" + return 2 + ;; + esac + local target="$SCRIPT_DIR/$name.sh" + if [ -e "$target" ]; then + error "$target already exists" + return 1 + fi + local fn + fn="$(printf '%s' "$name" | LC_ALL=C tr '-' '_')" + cat "$SCRIPT_DIR"/RUN_SCRIPT_TEMPLATE >"$target" + chmod +x "$target" + info "Created $target." + info "Replace every TODO (suggested function name: $fn); ./run picks the command up automatically." +} + +setup() { + # check that the script-development tools are installed and print what + # each tool resolved to + require shellcheck shfmt uvx || return 1 + print_resolved shellcheck shfmt uvx + info 'All script-development tools found.' +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else is treated as +# a file list for `check`, so `./run scripts file1.sh file2.py` just works. +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" +else + check "$@" +fi diff --git a/build_scripts/run_scripts/serve.sh b/build_scripts/run_scripts/serve.sh new file mode 100755 index 00000000..4a4af4ac --- /dev/null +++ b/build_scripts/run_scripts/serve.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +################################################################################ +# Serve a local English preview at http://localhost:8080 +# +# Does a full English build first (including the API reference), so it works +# from a clean tree with no prior build, then serves the result. Permissive: +# transient DocFX warnings do not block the preview. Ctrl-C stops the server. +# +# Run `./run watch` in another terminal for live rebuilds as you edit, then +# refresh the browser after each rebuild. +# +# Tool overrides (env vars): RUN_PYTHON. +# +# Usage: +# ./run serve +# ./run serve help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +serve() { + # full English build, then serve at http://localhost:8080 (Ctrl-C stops) + require python + log_and_run "$RUN_PYTHON" build-docs.py --serve --permissive +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch serve "$@" diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh new file mode 100755 index 00000000..5c223b1a --- /dev/null +++ b/build_scripts/run_scripts/setup.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +################################################################################ +# Check that the tools needed to contribute docs are installed +# +# Verifies every dependency a doc contributor needs: Python >= 3.11, the +# te CLI, plus docfx (either the repo-local dotnet tool or a global +# install). Reports everything that is missing in one pass; installs +# nothing. The script-development tools have their own check: +# `./run scripts setup`. +# +# Usage: +# ./run setup +# ./run setup help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +_docfx_ok() { + # True if docfx is invocable in either form build-docs.py resolves: + # the repo-local dotnet tool, or a docfx on PATH. + if docfx --version >/dev/null 2>&1; then + return 0 + fi + + # This function runs inside a conditional (check's `if ! _docfx_ok`), + # which suspends set -e; a failing require does not stop execution + # here, so its failure must be propagated explicitly. + require dotnet || return 1 + "$RUN_DOTNET" docfx --version >/dev/null 2>&1 +} + +_docfx_desc() { + # Describe which docfx form resolved, mirroring _docfx_ok's probe order: + # a docfx on PATH, else the repo-local dotnet tool (RUN_DOTNET is set in + # that case, by _docfx_ok's require). + if command -v docfx >/dev/null 2>&1; then + command -v docfx + else + printf '%s docfx (repo-local dotnet tool)\n' "$(command -v "$RUN_DOTNET")" + fi +} + +check() { + # verify all tooling the run commands need and print what each tool + # resolved to; report problems all at once + local failed=0 + # dotnet is not in this list: _docfx_ok requires it only when no global + # docfx exists, so it is reported exactly once and only when relevant. + require python te || failed=1 + if ! _docfx_ok; then + error "docfx not found: neither 'dotnet docfx --version' nor 'docfx --version' succeeded." + info 'See build_scripts/run_scripts/README.md for installation instructions.' + failed=1 + fi + if [ "$failed" -eq 0 ]; then + print_resolved python te + printf ' %-11s %s\n' docfx "$(_docfx_desc)" + info 'All dependencies found.' + fi + return "$failed" +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch check "$@" diff --git a/build_scripts/run_scripts/watch.sh b/build_scripts/run_scripts/watch.sh new file mode 100755 index 00000000..78c59486 --- /dev/null +++ b/build_scripts/run_scripts/watch.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +################################################################################ +# Rebuild the English docs on every save +# +# Polling loop: watches content/ and templates/ and runs the fast build +# (build.sh fast) whenever a file changes. Zero dependencies beyond the build +# itself; the ~1s poll latency is nothing next to the build time, and rapid +# saves coalesce into one rebuild. Ctrl-C to stop. +# +# Run alongside `./run serve` in another terminal, then refresh the browser +# after each rebuild. +# +# Tool overrides (env vars): RUN_PYTHON. +# +# Usage: +# ./run watch +# ./run watch help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +watch() { + # poll content/ and templates/; run the fast build on each change + require python + info "Watching content/ and templates/ for changes (Ctrl-C to stop)..." + # stamp is intentionally a script global (not local) so the EXIT trap + # can still reference it when the loop is interrupted. + stamp="$(mktemp)" + trap 'rm -f "$stamp"' EXIT + while true; do + if find content templates -type f -newer "$stamp" -print -quit | grep -q .; then + touch "$stamp" + bash "$SCRIPT_DIR/build.sh" fast || true + info '' + info "--- rebuilt $(date +%T) - refresh your browser ---" + fi + sleep 1 + done +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch watch "$@" diff --git a/build_scripts/te_script_runner.py b/build_scripts/te_script_runner.py index 98f7f2f2..74acfb88 100755 --- a/build_scripts/te_script_runner.py +++ b/build_scripts/te_script_runner.py @@ -28,6 +28,7 @@ """ import json +import os import shutil import subprocess import sys @@ -36,7 +37,10 @@ from pathlib import Path from typing import Any, NamedTuple -DEFAULT_TE_BIN = "te" +# RUN_TE is the ./run tooling's override convention (see +# build_scripts/run_scripts/README.md); honoring it here means a pinned te +# applies to everything built on this module. +DEFAULT_TE_BIN = os.environ.get("RUN_TE", "te") _WORKDIR_PREFIX = "te-script-run." # Emitted between the next-to-last and last script when last_only is set, so the runner # can report only the final script's output. Chosen to be absent from any real output. @@ -80,11 +84,7 @@ def _parse_json(stdout: str) -> dict[str, Any] | None: def _output_lines(data: dict[str, Any]) -> list[str]: """The text of every output-level message, in order. Pure.""" messages = data.get("messages") or [] - return [ - str(m.get("text", "")) - for m in messages - if isinstance(m, dict) and m.get("level") == "output" - ] + return [str(m.get("text", "")) for m in messages if isinstance(m, dict) and m.get("level") == "output"] def _diagnostic_lines(data: dict[str, Any]) -> list[str]: @@ -135,11 +135,7 @@ def summarize(stdout: str, exit_code: int) -> Result: """ data = _parse_json(stdout) if data is None: - diagnostics = ( - [f"[te] failed with exit {exit_code} and no json output"] - if exit_code != 0 - else [] - ) + diagnostics = [f"[te] failed with exit {exit_code} and no json output"] if exit_code != 0 else [] return Result(exit_code, exit_code == 0, "", diagnostics) if data.get("dryRun"): return _summarize_dry_run(data, exit_code) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..941fc2b6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +# Configuration for the repo's Python tooling (build_scripts/*.py). +# +# The ruff `include` and mypy `files` lists below are the same list and must +# stay in sync: the build scripts that meet the bar enforced by +# `./run scripts check`. Add scripts to both lists as they qualify. + +[tool.ruff] +line-length = 120 +target-version = "py311" +lint.select = ["F", "B", "SIM", "I", "UP"] +include = [ + "build_scripts/check_links.py", + "build_scripts/csharp_doctest.py", + "build_scripts/te_script_runner.py", +] + +[tool.mypy] +files = [ + "build_scripts/check_links.py", + "build_scripts/csharp_doctest.py", + "build_scripts/te_script_runner.py", +] +strict = true diff --git a/run b/run new file mode 100755 index 00000000..d0d5133c --- /dev/null +++ b/run @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +################################################################################ +# Task runner for the Tabular Editor docs repo +# +# Dispatches to the command scripts in build_scripts/run_scripts/. Always runs +# from the repo root, regardless of the caller's working directory, so file +# arguments are repo-root-relative. +# +# Usage: +# ./run # this documentation plus the command list (alias: help) +# ./run [args] # run a command +# ./run help # detailed help for one command +# ./run commands # just the command list (alias: --list) +# +# On Windows, invoke from Git Bash (`./run `, or `bash run `). +################################################################################ + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS="$ROOT/build_scripts/run_scripts" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPTS/lib.sh" +cd "$ROOT" + +commands() { + # List every command script with the first line of its banner. + printf 'Commands:\n' + local script name + for script in "$SCRIPTS"/*.sh; do + name="$(basename "$script" .sh)" + if [ "$name" = "lib" ]; then + continue + fi + printf ' %-12s %s\n' "$name" "$(banner_title "$script")" + done + printf '\nRun ./run help for details on one command.\n' + printf 'Run ./run help for full runner documentation.\n' +} + +help() { + # Full runner documentation (the banner above) plus the command index. + banner_help "$0" + commands +} + +cmd="${1:-help}" +if [ "$#" -gt 0 ]; then + shift +fi + +case "$cmd" in +commands | --list) + commands + exit 0 + ;; +help | --help | -h) + help + exit 0 + ;; +esac + +script="$SCRIPTS/$cmd.sh" +if [ ! -f "$script" ]; then + error "unknown command: $cmd" + commands >&2 + exit 2 +fi + +exec bash "$script" "$@"