From 398681c3a9ab881b18454df30151b27b43faf640 Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Wed, 15 Jul 2026 21:01:54 +0200 Subject: [PATCH 1/6] Add `run` script for contributor conveniences See build_scripts/run_scripts/README.md --- README.md | 49 +++- build_scripts/run_scripts/README.md | 357 ++++++++++++++++++++++++++++ build_scripts/run_scripts/build.sh | 57 +++++ build_scripts/run_scripts/clean.sh | 44 ++++ build_scripts/run_scripts/lib.sh | 182 ++++++++++++++ build_scripts/run_scripts/lint.sh | 103 ++++++++ build_scripts/run_scripts/serve.sh | 37 +++ build_scripts/run_scripts/setup.sh | 59 +++++ build_scripts/run_scripts/watch.sh | 51 ++++ run | 71 ++++++ 10 files changed, 998 insertions(+), 12 deletions(-) create mode 100644 build_scripts/run_scripts/README.md create mode 100755 build_scripts/run_scripts/build.sh create mode 100755 build_scripts/run_scripts/clean.sh create mode 100755 build_scripts/run_scripts/lib.sh create mode 100755 build_scripts/run_scripts/lint.sh create mode 100755 build_scripts/run_scripts/serve.sh create mode 100755 build_scripts/run_scripts/setup.sh create mode 100755 build_scripts/run_scripts/watch.sh create mode 100755 run diff --git a/README.md b/README.md index fcc9159a..6dc37306 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,31 @@ The site uses [DocFX](https://dotnet.github.io/docfx/) and GitHub flavoured mark All contributions are welcome. We will review all pull requests submitted. -To test your changes locally: -- Make sure [DocFX](https://dotnet.github.io/docfx/) and Python 3.11+ are installed. -- Run `python build-docs.py --serve` in the root of the project. +For convenience for typical contributions, we have built a simple wrapper around the build process. +Unless you are working specifically on localization or the build process itself, you should be able to get by with the `run` script. -# Build Script Usage +Getting started: +1. Make sure you have Bash installed (included in most Linux distros, macOS, and [Git for Windows](https://git-scm.com/install/windows)) +2. From the repo root, run the setup check and install any tools it reports as missing + - (Linux distros and macOS): `./run setup` + - (Windows): `bash run setup` (after this run everything from Git Bash) +3. To iterate on docs and see a preview in your browser: + - in one terminal: `./run serve`: launches a localhost server that renders the docs at http://localhost:8080 where you can see the docs as they will be rendered on the docs website + - in another terminal: `./run watch`: regenerates the rendered site every time you save a change to a markdown document, so you can refresh your browser to see it + +These commands all build and work only on the English language docs, as the English markdown is our canonical version of documentation and the language we expect contributions in. + +For more info, try `./run help` and `./run help`. +If you'd like more detail, [check out the run script README](build_scripts/run_scripts/README.md). + +If you want to have more control over the build process, continue reading below about the `build-docs.py` script. + +# Advanced build Script Usage The `build-docs.py` script handles all documentation building tasks including multi-language support. +Make sure you have Python >=3.11 and docfx installed, the latter either globally or locally. -## Quick Start +## Using build-docs.py directly ```bash # Build and serve locally (English only, for development) @@ -50,6 +66,8 @@ swa start _site | `--serve` | Build and serve locally (English only, for development) | | `--skip-gen` | Skip running gen_redirects.py (use existing configs) | | `--no-api-copy` | Skip copying API docs to localized sites | +| `--skip-api` | Reuse existing API metadata in content/api, ~30-40% faster (local markdown iteration only, requires `--serve`/`--lang`; never for testing/CI/CD/releases) | +| `--permissive` | Don't treat English DocFX warnings as build failures (for local iteration; full/CI builds stay strict) | | `--sync` | Sync English fallback for missing/outdated translations (for local dev) | ## What the Build Script Does @@ -68,16 +86,24 @@ swa start _site # Project Structure ``` -TEDoc/ +/ ├── build-docs.py # Main build script +├── run # Task runner for common dev tasks (see build_scripts/run_scripts/README.md) ├── build_scripts/ # Helper scripts -│ ├── gen_redirects.py # Generates docfx.json configs +│ ├── check_links.py # Dead-link checker for the generated _site +│ ├── config_loader.py # Shared configuration loader for build scripts +│ ├── csharp_doctest.py # Validates annotated C# code blocks in docs against the te CLI │ ├── gen_languages.py # Generates language manifest -│ ├── gen_staticwebapp_config.py -│ ├── inject_seo_tags.py -│ ├── sync-localized-content.py +│ ├── gen_redirects.py # Generates docfx.json configs +│ ├── gen_sitemap_index.py # Post-processes the English sitemap; generates the sitemap index +│ ├── gen_staticwebapp_config.py # Generates Azure Static Web Apps routing config +│ ├── inject_seo_tags.py # Adds hreflang and canonical tags to built HTML │ ├── normalize-localized-alerts.py # Repairs Crowdin-collapsed DocFX alerts -│ └── normalize-localized-heading-anchors.py # Injects English-slug bookmark anchors into translations +│ ├── normalize-localized-heading-anchors.py # Injects English-slug bookmark anchors into translations +│ ├── sync-localized-content.py # Syncs English content into localized build dirs +│ ├── te_script_runner.py # Runs C# snippets against a throwaway model via the te CLI +│ ├── test-fixtures/ # Fixtures for the build-script tests +│ └── run_scripts/ # ./run subcommand scripts and shared lib.sh (see its README) ├── content/ # English source content (tracked in git) │ └── _ui-strings.json # English UI strings (header, footer, banners) ├── localizedContent/ # Build directories for all languages @@ -222,4 +248,3 @@ If a key is missing from a language's file, or no `_ui-strings.json` exists at a | `tableOfContents` | `Table of Contents` | Mobile TOC offcanvas title | | `selectLanguage` | `Select language` | Language picker label | | `copyCode` | `Copy code` | Code block copy button aria-label | - diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md new file mode 100644 index 00000000..779e26d0 --- /dev/null +++ b/build_scripts/run_scripts/README.md @@ -0,0 +1,357 @@ +# Docs tooling runner (`./run`) + +`./run` in the repo root dispatches to the command scripts in this directory. +Run `./run` (or `./run help`) for the full command list, and +`./run help` for details on one command. + +- macOS / Linux: `./run ` +- Windows: use Git Bash (installed with [Git for Windows](https://gitforwindows.org/)): + `./run `, or `bash run ` from any shell (e.g., PowerShell) that can invoke bash. + +`./run setup` checks that everything below is installed and reports all +problems in one pass. It installs nothing. + +**Most common commands**: +- once: `./run setup` +- when editing (in two terminals): + - `./run serve`: http://localhost:8080 with rendered docs + - `./run watch`: regenerate docs whenever a file changes (manually refresh in browser to see new docs rendered) + +## Requirements + +| Tool | Needed by | Notes | +|------------|---------------------------|-----------------------------------------------------------------------------| +| bash | everything | 3.2+; Git Bash on Windows, but available by default on most other platforms | +| Python | `build`, `serve`, `watch` | 3.11 or newer | +| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | +| docfx | `build`, `serve`, `watch` | local dotnet tool or global | +| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | +| shellcheck | `lint` (shell sources) | | + +The first `./run lint` needs network access: uvx downloads ruff and mypy into +its cache on first use. + +## Installation + +We provide the most common and script-friendly installation methods per platform in each section below. +If you prefer, though, you can follow these links to each tool's official installation docs + +- Bash (Git for Windows): https://git-scm.com/install/windows +- Bash (non-windows): https://www.gnu.org/software/bash/ +- Python: https://www.python.org/ +- docfx: https://dotnet.github.io/docfx/ +- uv: https://docs.astral.sh/uv/#installation +- shellcheck: https://github.com/koalaman/shellcheck#user-content-installing + +### Linux + +```bash +# Python and shellcheck from your distro, e.g. Debian/Ubuntu: +sudo apt-get install python3 shellcheck +# uv (provides uvx): +curl -LsSf https://astral.sh/uv/install.sh | sh +# .NET SDK (for docfx): see https://learn.microsoft.com/dotnet/core/install/linux +``` + +### macOS + +```bash +brew install python3 uv shellcheck +brew install --cask dotnet-sdk # .NET SDK (for docfx) +``` + +### Windows + +Install [Git for Windows](https://gitforwindows.org/) for Git Bash, then: + +```powershell +winget install Python.PythonInstallManager +winget install astral-sh.uv +winget install koalaman.shellcheck +winget install Microsoft.DotNet.SDK.10 +``` + +Note: winget has no unversioned .NET SDK id; docfx needs SDK 8.0 or newer. + +### docfx (all platforms) + +With the .NET SDK installed (see your platform above), install docfx as a +repo-local dotnet tool. From the repo root: + +```bash +dotnet tool install docfx +dotnet tool restore +``` + +If you are running a `dotnet` older than 10.0, run `dotnet new tool-manifest` first. + +## Usage + +The most common commands are provided below for reference. +All sub-commands provide help text at the command line. + +- Check all dependencies are installed: + - `./run setup` + - `RUN_UVX=/path/to/uvx RUN_PYTHON=/path/to/python ./run setup`: confirm that tools at custom paths are resolved correctly +- Hot reloading iteration on markdown content (run in two separate terminals) + - `./run serve`: builds docs site and launches a localhost server at http://localhost:8080 where you can browse rendered docs + - `./run watch`: watches for changes in template files and markdown files, runs a fast rebuild upon detecting changes +- One-off build (pick one) + - `./run build`: required after a `./run clean` or a fresh clone of the repo + - `./run build fast`: skip re-generating the API docs; what you probably want most of the time and saves 1/3-1/2 build time +- Lint build scripts and run scripts: + - `./run lint`: lint everything that is known to already conform to linting standards (all shell scripts and a subset of build scripts) + - `./run lint file1.sh file2.py ...`: lint specific shell scripts or python scripts (pass as many filenames of either or both types as you like) +- Clean build artifacts: `./run clean` +- Get help: + - `./run help`: print top-level help for the `run` dispatcher and list all subcommands + - `./run help`: print full help for `` + +### Dependency overrides + +Every external tool (e.g., python, uvx, shellcheck) can be pinned to a specific executable with a `RUN_` environment variable: +`RUN_PYTHON`, `RUN_UVX`, `RUN_SHELLCHECK`. +When set, that path is the only candidate checked; there is no fallback to PATH. Example: + +```bash +RUN_PYTHON=/opt/python3.12/bin/python3 ./run build +RUN_PYTHON=/opt/python3.12/bin/python3 RUN_SHELLCHECK=/path/to/shellcheck ./run build +``` + +These environment variables are intended for testing specific versions or for users with these binaries not available in their `PATH`. + +docfx is resolved by build-docs.py itself: +a `DOCFX` environment variable, then a repo-local dotnet tool manifest, then a global `docfx` on PATH. + +## Contributing + +### Conventions for these scripts + +- bash 3.2 compatible (macOS default bash), ASCII only, long-form CLI flags. +- Each script: banner help block, per-function comment docs, awk-extracted help, `dispatch "$@"` at the bottom. +- Shared helpers live in `lib.sh`; command-specific state lives in the command's script. +- All of this tooling is linted by `./run lint` (shellcheck for the shell scripts, ruff + mypy for the Python sources). +- Any internal function not intended for re-use is prefixed with an underscore + +### General architecture + +Everything flows through the `run` command, which is a dispatcher to various special-purpose scripts, exposed as subcommands to `run`. +Every subcommand is a script in this directory (run_scripts/). +The script must be named `name.sh` and `name` becomes the name of the sub-command; +there is no other registration for sub-commands, if the script exists, it's a subcommand. + +#### `run` dispatcher + +`run` is the script in the repo root, through which we invoke all run scripts. +It is intentionally minimal, and does only these things: + +**Happy path**: + +1. `cd` to the repo root, so every run script starts from the same working directory no matter where you invoke it from +2. resolve `` to `build_scripts/run_scripts/.sh` and invoke it with `bash`, passing all remaining arguments through verbatim + +**`help`**: Print help (`./run`, `./run help`) and the command list (`./run commands`, `./run --list`) + +**Error path**: Print an error plus the command list when the first argument does not resolve to a script + +The command list is discovered by scanning this directory for `*.sh` files (excluding `lib.sh`); +the one-line description for each command is pulled from its banner comment with `banner_title`. +Anything beyond the above belongs in a run script, not in `run`. + +#### `lib.sh` + +Shared helpers live in [lib.sh](./lib.sh); these are intended to be used for boilerplate and common operations across sub-commands: +- `info`: print informational messages onto stderr for the user +- `error`: print error messages prefixed with "ERROR: " +- `log_and_run`: prints the command being run in bold on stderr; use for all commands that actually get run for a scripts work +- `require`: automatically handle the env var for tool overrides and fail with info if the tool can't be found +- `dispatch`: provide a way to take a default function with args (see usage in commands for examples) +- `banner_title`: print the first one-line summary of a comment banner at the top of a script file (for command lists) +- `banner_help`: print the whole comment banner at the top of a script file (for help and usage) +- `command_help`: print doc comments for each function in a script (for help output) + +If you are writing or editing a run script, you should use these helpers. +If there's some new generic requirement, that's a decent candidate to be added to `lib.sh`. + +#### Anatomy of a run script + +All run scripts must: +- be named `.sh` in this directory; they are automatically discovered based on that naming convention. +- have a `#!/usr/bin/env bash` shebang, and are marked executable with `chmod +x`. +- have a comment banner with a one-line summary, help text, and usage examples +- contain the header below +- use shell functions for all functionality +- contain a standard `help` function +- use the `dispatch` pattern + +##### Banner comment + +A banner comment is of the form below. +This becomes the help text for this script. + +```shell +###################################### (at least 10 octothorpes starting at character 0) +# One line command summary +# +# all banner comment lines must start with an octothorpe. +# provide descriptive help text +# explain what this run script does +# give any relevant details for a user running this command +# not information about the development of the command +# +# Usage: +# ./run command # what this invocation does +# ./run command arg # what *this* one does +###################################### (at least 10 octothorpes starting at character 0) +``` + +##### Script header + +This: +1. sets up some safety defaults for Bash +2. makes sure that all scripts are executing in the same working directory context +3. sources `lib.sh` so we have the shared helpers + +```shell +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." +``` + +##### Implementation functions + +Anything that looks like an argument or a sub-subcommand is implemented as a Bash function. +The very first thing in the function definition must be comments that become the help text for this function. + +Example: + +```shell +fn () { + # the help text for this function + # can span many lines + require ... + +``` + +These functions must declare their required tools upfront with `require`, e.g., `require uvx`; +do not use a path here, but only the exact binary name. +The sole exception is Python, which must always be written as "python" verbatim (`require` handles platform specifics for this). + +Examples: +- `require uvx` +- `require python` +- `require python shellcheck uvx` + +Using `require` means we get automatic environment variable handling for these things, and helpful error messages when the binaries are not available. + +##### Using the `RUN_` variables that `require` sets up + +`require` does two jobs. +1. It checks that each named tool is available, + honoring the user's `RUN_` override as described in [Dependency overrides](#dependency-overrides), +2. It exports a `RUN_` environment variable holding the winner. + The variable name is the tool name uppercased, with dashes turned into underscores: + `require uvx` exports `RUN_UVX`, `require shellcheck` exports `RUN_SHELLCHECK`, `require python` exports `RUN_PYTHON`. + +Always invoke a required tool through its variable, never by its bare name: + +```shell +python () { + # ruff on the Python tooling + require uvx + log_and_run "$RUN_UVX" ruff check --quiet "$@" +} +``` + +Invoking `"$RUN_UVX"` (quotes included; the path may contain spaces) is what makes a user's override actually take effect. +A bare `uvx` on that line would search the user's `PATH`, but if they provided `RUN_UVX=/other/path/to/uvx ./run lint`, +then they are explicitly telling us, "Use the `uvx` at the provided path, not from `PATH`," +so we honor that. +These exports also flow into any script the current one delegates to (e.g., `watch.sh` runs `build.sh fast`, which sees the same `RUN_PYTHON`), +so an override applies consistently across a whole command. + +###### How `require` failures behave + +On failure, `require` prints what is missing and a pointer to this README. +When there are multiple arguments to `require`, e.g., `require python uvx`, it resolves and sets all variables. +If one or more tools are not found, then the error printed to the terminal includes details about all missing tools. +When it doesn't find all required tools then it **returns** nonzero; it never exits the script itself. +In the normal placement, at the top of a function that the user invoked directly, that is all you need: +the script header's `set -euo pipefail` turns the non-0 `return` into a clean stop. + +The exception is a function that gets called inside a conditional (`if`, `!`, `&&`, `||`). +Bash suspends `set -e` for everything running inside a condition, including called functions, so a failing `require` there does not stop anything; +execution falls through to the next line as if nothing happened. +In that placement, you must propagate the failure explicitly: + +```shell +require dotnet || return 1 +``` + +`setup.sh` is the resident example: its `check` function probes helpers inside `if !` conditionals, so those helpers use the explicit form. + +##### Standard `help` function + +This uses `lib.sh` helpers to give a standardized help text. +These pull help text out of the banner comment and all leading comments in functions. +They are automatically formatted as you see when you run `./run help` or `./run subcommand help`. + +```shell +help () { + # show this help + banner_help "$0" + command_help "$0" +} +``` + +##### `dispatch` pattern + +The last line of every run script is a call to `dispatch`: + +```shell +dispatch "$@" +``` + +Read it as: "look at what the user typed after `./run `, and call the function with that name". + +Take `build.sh`, which ends with `dispatch full "$@"`, as a worked example. +`"$@"` stands for "everything the user typed after `./run build`", so: + +- When a user submits `./run build`, there is nothing after the subcommand, so we end up with `dispatch full` and nothing in `"$@"`. + With no user input to look at, `dispatch` falls back to the default it was given and calls the `full` function. +- When a user submits `./run build fast`, we end up with `dispatch full fast`. + `dispatch` sees the user's `fast`, finds a function with that name in the script, and calls `fast`; the `full` default is ignored. +- When a user submits `./run build nonsense`, we end up with `dispatch full nonsense`. + There is no `nonsense` function in the script, so dispatch prints an error, then the script's `help`, and exits with a usage error. +- Anything after the function name is handed to that function as its arguments. + In `lint.sh` (default `all`), `./run lint python a.py b.py` ends up as `dispatch all python a.py b.py`; + dispatch calls the `python` function with `a.py b.py` as its arguments. + +Name the default function after what it does (`full`, `all`, `check`), not a generic `main`; +function names are user interface here, since they are exactly what a user types after `./run `. + +There are exactly two correct ways to write the dispatch line: + +1. The standard form shown above: `dispatch "$@"`. + The default function name is required; `dispatch` always needs to know what to run when the user types nothing. +2. A wrapper around form 1, for a script that wants to handle unrecognized arguments itself instead of treating them as an error. + For example, `lint.sh` treats an unrecognized first argument as a list of files to lint: + +```shell +if declare -F "${1:-all}" >/dev/null; then + dispatch all "$@" +else + all "$@" +fi +``` + +Two mistakes to avoid: + +- `dispatch "$@"` (leaving out the default) is broken, not just incomplete: + `dispatch` would mistake the user's second argument for the name of the function to call, or crash the script when there are no arguments at all. + If a script has no obvious default action, make `help` its default. +- `"$@"` must always be written with the quotes. + Without them, an argument containing a space (such as a file path) gets split into pieces before `dispatch` ever sees it. diff --git a/build_scripts/run_scripts/build.sh b/build_scripts/run_scripts/build.sh new file mode 100755 index 00000000..50ec82fc --- /dev/null +++ b/build_scripts/run_scripts/build.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +################################################################################ +# Build the English docs for local iteration +# +# Wraps the canonical build-docs.py orchestration; no parallel build logic. +# Never touches the localized es/zh content. +# +# full (default) regenerate the API reference; reuse the generated +# docfx.json configs; fail on DocFX warnings +# fast also reuse existing API metadata and tolerate DocFX warnings; +# the same build that `watch` runs +# +# Usage: +# ./run build [full|fast] +# ./run build help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +common_flags () { + # Set FLAGS to the markdown-iteration base: English only, and reuse the + # generated docfx.json configs (--skip-gen) when they exist. After a + # `clean` they do not, so the generation step runs on the next build. + FLAGS=(--lang en) + if [ -f localizedContent/en/docfx.json ] && [ -f metadata/languages.json ]; then + FLAGS+=(--skip-gen) + fi +} + +full () { + # regenerate the API reference; fail on DocFX warnings + require python + common_flags + log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" +} + +fast () { + # reuse existing API metadata (--skip-api); tolerate DocFX warnings + # (--permissive); the build that watch runs after each save + require python + common_flags + log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" --skip-api --permissive +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch full "$@" diff --git a/build_scripts/run_scripts/clean.sh b/build_scripts/run_scripts/clean.sh new file mode 100755 index 00000000..e0e1556b --- /dev/null +++ b/build_scripts/run_scripts/clean.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +################################################################################ +# Remove generated build artifacts +# +# Deletes the DocFX build outputs and generated configs: _site, obj, the +# generated docfx.json files, generated API reference yml, language metadata, +# and the localized-content staging outputs. Everything removed is gitignored; +# tracked source is never touched. +# +# Usage: +# ./run clean +# ./run clean help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +clean () { + # remove all generated build artifacts (gitignored); tracked source is + # left intact + log_and_run rm -rf _site obj docfx.json + log_and_run rm -f content/api/*.yml content/api/.manifest + log_and_run rm -f metadata/languages.json + log_and_run rm -rf localizedContent/en + log_and_run find localizedContent -maxdepth 2 -name docfx.json -delete || true + # Generated files only: each localized content/api keeps a tracked + # index.md, so the whole directory must not be removed. + log_and_run rm -f localizedContent/*/content/api/*.yml localizedContent/*/content/api/.manifest + log_and_run rm -rf localizedContent/*/content/assets + info "Clean complete." +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch clean "$@" diff --git a/build_scripts/run_scripts/lib.sh b/build_scripts/run_scripts/lib.sh new file mode 100755 index 00000000..7b1054f7 --- /dev/null +++ b/build_scripts/run_scripts/lib.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Shared helpers for the ./run command scripts. +# +# Sourced as a library by the command scripts in this directory, or executed +# directly to exercise a single helper: +# bash build_scripts/run_scripts/lib.sh require git python3 + +# Status/progress chatter, on stderr so stdout stays clean for command output. +info () { printf '%s\n' "$*" >&2; } + +# Error message with a uniform prefix, on stderr. +error () { printf 'ERROR: %s\n' "$*" >&2; } + +# Print a command then execute it, bold (like just's recipe echo) with a +# "$ " prefix marking it as a command line. The echo goes to stderr so a +# command's stdout stays clean for its own output; escape codes only when +# stderr is a terminal and NO_COLOR is unset, so redirected output stays +# plain. +log_and_run () { + if [ -t 2 ] && [ -z "${NO_COLOR:-}" ]; then + printf '\033[1m$ %s\033[0m\n' "$*" >&2 + else + printf '$ %s\n' "$*" >&2 + fi + "$@" +} + +require () { + # Check the given tools and resolve which executable to use for each. + # Takes bare tool names, not paths. A pre-set RUN_ env var (name + # uppercased, dashes to underscores: RUN_UVX for uvx, RUN_SHELLCHECK + # for shellcheck) is a user override and is the only candidate checked + # otherwise the tool must be on PATH. The winner is exported as + # RUN_ for callers to invoke and for delegated scripts to + # inherit. Failures are reported all at once. + # The name `python` is special: it delegates to _require_python, which + # version-gates the interpreter and reports its own detail. + local missing=() cmd var val failed=0 + for cmd in "$@"; do + if [ "$cmd" = "python" ]; then + _require_python || failed=1 + continue + fi + # LC_ALL=C pins ASCII range semantics regardless of locale. + var="RUN_$(printf '%s' "$cmd" | LC_ALL=C tr 'a-z-' 'A-Z_')" + val="${!var:-$cmd}" + if command -v "$val" >/dev/null 2>&1; then + export "$var=$val" + elif [ "$val" = "$cmd" ]; then + missing+=("$cmd") + else + missing+=("$cmd ($var=$val not found)") + fi + done + if [ "${#missing[@]}" -gt 0 ]; then + error "missing required command(s): ${missing[*]}" + info 'See build_scripts/run_scripts/README.md for installation instructions.' + return 1 + fi + return "$failed" +} + +_python_ok () { + # True if the given interpreter exists and is Python >= 3.11 (the + # version floor for the repo's build tooling, matching CI). + command -v "$1" >/dev/null 2>&1 && + "$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' >/dev/null 2>&1 +} + +_require_python () { + # Locate a Python >= 3.11 and export it as RUN_PYTHON for commands to + # invoke. When RUN_PYTHON is already set (contributor override) it is + # the ONLY candidate checked, so a broken override fails loudly instead + # of silently falling back. Otherwise probe python3, python, then the + # Windows launcher (py -3, resolved to its underlying executable so + # RUN_PYTHON stays a single quotable word). On failure, reports the + # version of every candidate that was present. + local cand tried="" + if [ -n "${RUN_PYTHON:-}" ]; then + if _python_ok "$RUN_PYTHON"; then + export RUN_PYTHON + return 0 + fi + error "RUN_PYTHON=$RUN_PYTHON is not a Python >= 3.11." + if command -v "$RUN_PYTHON" >/dev/null 2>&1; then + info " found: $("$RUN_PYTHON" --version 2>&1 || true)" + fi + info 'See build_scripts/run_scripts/README.md for installation instructions.' + return 1 + fi + + for cand in python3 python; do + if _python_ok "$cand"; then + RUN_PYTHON="$cand" + export RUN_PYTHON + return 0 + fi + if command -v "$cand" >/dev/null 2>&1; then + tried="$tried + $cand: $("$cand" --version 2>&1 || true)" + fi + done + + if command -v py >/dev/null 2>&1; then + cand="$(py -3 -c 'import sys; print(sys.executable)' 2>/dev/null || true)" + if [ -n "$cand" ] && _python_ok "$cand"; then + RUN_PYTHON="$cand" + export RUN_PYTHON + return 0 + fi + tried="$tried + py -3: $(py -3 --version 2>&1 || true)" + fi + + error "Python >= 3.11 is required and was not found." + if [ -n "$tried" ]; then + info "Candidates checked:$tried" + fi + info 'Set RUN_PYTHON to a suitable interpreter, or see build_scripts/run_scripts/README.md for installation instructions.' + return 1 +} + +dispatch () { + # dispatch [args...]: call the function named by the first + # remaining arg, or default_fn when none is given. Unknown names print + # the calling script's help and fail with a usage error. + local fn="${2:-$1}" + shift + if [ "$#" -gt 0 ]; then + shift + fi + if declare -F "$fn" >/dev/null; then + "$fn" "$@" + else + error "unknown command: $fn" + if declare -F help >/dev/null; then + help + fi + return 2 + fi +} + +banner_title () { + # Print only the first line of the #### banner block of the given script + # (used by ./run for its one-line-per-command index). + awk ' + /^#{10,}/ { if (inIntro) exit; inIntro=1; next } + inIntro && /^#/ { gsub(/^# ?/, ""); print; exit } + ' "$1" +} + +banner_help () { + # Print the #### banner block at the top of the given script, unprefixed. + awk ' + BEGIN { inIntro=0; introDone=0 } + /^#{10,}/ && introDone==0 && inIntro==0 { inIntro=1; next } + /^#{10,}/ && inIntro==1 { inIntro=0; introDone=1; print ""; next } + inIntro==1 { gsub(/^# ?/, ""); print } + ' "$1" +} + +command_help () { + # Print each function in the given script with its leading comment lines. + # Underscore-prefixed functions are internal and excluded. + printf 'Commands:\n' + awk ' + /^[a-z][a-z_]* *\(\) *\{/ { fn=$1; inCmd=1; next } + inCmd==1 && /^[ \t]+#/ { + if (fn != "") { print " " fn; fn="" } + gsub(/^[ \t]+# ?/, " ") + print + next + } + inCmd==1 { inCmd=0 } + ' "$1" +} + +# Dispatch only when executed directly; stay silent when sourced. +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -euo pipefail + "$@" +fi diff --git a/build_scripts/run_scripts/lint.sh b/build_scripts/run_scripts/lint.sh new file mode 100755 index 00000000..04558acf --- /dev/null +++ b/build_scripts/run_scripts/lint.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +################################################################################ +# Lint the repo's tooling +# +# Python (*.py) via uvx: ruff (F,B,SIM,I,UP) + mypy --strict. +# Shell (*.sh, and the extensionless ./run) via shellcheck. +# +# Each command takes an optional list of files; with no files it lints the +# default source lists defined at the top of this script. Runs from the repo +# root, so file arguments are repo-root-relative. +# +# Tool overrides (env vars): RUN_UVX and RUN_SHELLCHECK name the executables +# to use for uvx and shellcheck; when set, only that path is checked. +# +# Usage: +# ./run lint # python + shell on the default lists +# ./run lint file1.sh file2.py ... # route a mixed file list by extension +# ./run lint python [file.py ...] +# ./run lint shell [file.sh ...] +# ./run lint help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +# Default source lists: the tooling that meets the lint bar. Add files as +# they qualify. +PYTHON_SOURCES=( + build_scripts/check_links.py + build_scripts/te_script_runner.py + build_scripts/csharp_doctest.py +) +SHELL_SOURCES=(run build_scripts/run_scripts/*.sh) + +python () { + # ruff + mypy --strict on the Python tooling (default: PYTHON_SOURCES) + require uvx + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${PYTHON_SOURCES[@]}") + fi + log_and_run "$RUN_UVX" ruff check --quiet --select F,B,SIM,I,UP "${files[@]}" + # --no-error-summary drops only the trailing tally line ("Found N errors + # ..."/"Success: ..."); per-error diagnostics and the exit code are + # unaffected, so success is silent like shellcheck. + log_and_run "$RUN_UVX" mypy --strict --no-error-summary "${files[@]}" +} + +shell () { + # check the shell tooling with shellcheck (default: SHELL_SOURCES) + require shellcheck + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${SHELL_SOURCES[@]}") + fi + log_and_run "$RUN_SHELLCHECK" --external-sources "${files[@]}" +} + +all () { + # python + shell; with file args, routes each by extension (run -> shell) + if [ "$#" -eq 0 ]; then + python + shell + return + fi + local py=() sh=() path + for path in "$@"; do + case "$path" in + *.py) py+=("$path") ;; + *.sh|run|*/run) sh+=("$path") ;; + *) info "lint: ignoring path with unrecognized extension: $path" ;; + esac + done + if [ "${#py[@]}" -eq 0 ] && [ "${#sh[@]}" -eq 0 ]; then + error "no lintable files among: $*" + return 2 + fi + if [ "${#py[@]}" -gt 0 ]; then + python "${py[@]}" + fi + if [ "${#sh[@]}" -gt 0 ]; then + shell "${sh[@]}" + fi +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else is treated as +# a file list for `all`, so `./run lint file1.sh file2.py` just works. +if declare -F "${1:-all}" >/dev/null; then + dispatch all "$@" +else + all "$@" +fi diff --git a/build_scripts/run_scripts/serve.sh b/build_scripts/run_scripts/serve.sh new file mode 100755 index 00000000..a92b7cf3 --- /dev/null +++ b/build_scripts/run_scripts/serve.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +################################################################################ +# Serve a local English preview at http://localhost:8080 +# +# Does a full English build first (including the API reference), so it works +# from a clean tree with no prior build, then serves the result. Permissive: +# transient DocFX warnings do not block the preview. Ctrl-C stops the server. +# +# Run `./run watch` in another terminal for live rebuilds as you edit, then +# refresh the browser after each rebuild. +# +# Usage: +# ./run serve +# ./run serve help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +serve () { + # full English build, then serve at http://localhost:8080 (Ctrl-C stops) + require python + log_and_run "$RUN_PYTHON" build-docs.py --serve --permissive +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch serve "$@" diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh new file mode 100755 index 00000000..606788f4 --- /dev/null +++ b/build_scripts/run_scripts/setup.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +################################################################################ +# Check that all required tooling is installed +# +# Verifies every dependency the run commands need: Python >= 3.11, uvx, +# and shellcheck, plus docfx (either the repo-local dotnet tool or a global +# install). Reports everything that is missing in one pass; installs nothing. +# +# Usage: +# ./run setup +# ./run setup help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +_docfx_ok () { + # True if docfx is invocable in either form build-docs.py resolves: + # the repo-local dotnet tool, or a docfx on PATH. + if docfx --version >/dev/null 2>&1 + then return 0 + fi + + # This function runs inside a conditional (check's `if ! _docfx_ok`), + # which suspends set -e; a failing require does not stop execution + # here, so its failure must be propagated explicitly. + require dotnet || return 1 + "$RUN_DOTNET" docfx --version >/dev/null 2>&1 +} + +check () { + # verify all tooling the run commands need; report problems at once + local failed=0 + # dotnet is not in this list: _docfx_ok requires it only when no global + # docfx exists, so it is reported exactly once and only when relevant. + require python shellcheck uvx || failed=1 + if ! _docfx_ok; then + error "docfx not found: neither 'dotnet docfx --version' nor 'docfx --version' succeeded." + info 'See build_scripts/run_scripts/README.md for installation instructions.' + failed=1 + fi + if [ "$failed" -eq 0 ]; then + info 'All dependencies found.' + fi + return "$failed" +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch check "$@" diff --git a/build_scripts/run_scripts/watch.sh b/build_scripts/run_scripts/watch.sh new file mode 100755 index 00000000..9ed3e52f --- /dev/null +++ b/build_scripts/run_scripts/watch.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +################################################################################ +# Rebuild the English docs on every save +# +# Polling loop: watches content/ and templates/ and runs the fast build +# (build.sh fast) whenever a file changes. Zero dependencies beyond the build +# itself; the ~1s poll latency is nothing next to the build time, and rapid +# saves coalesce into one rebuild. Ctrl-C to stop. +# +# Run alongside `./run serve` in another terminal, then refresh the browser +# after each rebuild. +# +# Usage: +# ./run watch +# ./run watch help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +watch () { + # poll content/ and templates/; run the fast build on each change + require python + info "Watching content/ and templates/ for changes (Ctrl-C to stop)..." + # stamp is intentionally a script global (not local) so the EXIT trap + # can still reference it when the loop is interrupted. + stamp="$(mktemp)" + trap 'rm -f "$stamp"' EXIT + while true; do + if find content templates -type f -newer "$stamp" -print -quit | grep -q .; then + touch "$stamp" + bash "$SCRIPT_DIR/build.sh" fast || true + info '' + info "--- rebuilt $(date +%T) - refresh your browser ---" + fi + sleep 1 + done +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch watch "$@" diff --git a/run b/run new file mode 100755 index 00000000..6e48ffd0 --- /dev/null +++ b/run @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +################################################################################ +# Task runner for the Tabular Editor docs repo +# +# Dispatches to the command scripts in build_scripts/run_scripts/. Always runs +# from the repo root, regardless of the caller's working directory, so file +# arguments are repo-root-relative. +# +# Usage: +# ./run # this documentation plus the command list (alias: help) +# ./run [args] # run a command +# ./run help # detailed help for one command +# ./run commands # just the command list (alias: --list) +# +# On Windows, invoke from Git Bash (`./run `, or `bash run `). +################################################################################ + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS="$ROOT/build_scripts/run_scripts" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPTS/lib.sh" +cd "$ROOT" + +commands () { + # List every command script with the first line of its banner. + printf 'Commands:\n' + local script name + for script in "$SCRIPTS"/*.sh; do + name="$(basename "$script" .sh)" + if [ "$name" = "lib" ]; then + continue + fi + printf ' %-8s %s\n' "$name" "$(banner_title "$script")" + done + printf '\nRun ./run help for details on one command.\n' + printf 'Run ./run help for full runner documentation.\n' +} + +help () { + # Full runner documentation (the banner above) plus the command index. + banner_help "$0" + commands +} + +cmd="${1:-help}" +if [ "$#" -gt 0 ]; then + shift +fi + +case "$cmd" in + commands|--list) + commands + exit 0 + ;; + help|--help|-h) + help + exit 0 + ;; +esac + +script="$SCRIPTS/$cmd.sh" +if [ ! -f "$script" ]; then + error "unknown command: $cmd" + commands >&2 + exit 2 +fi + +exec bash "$script" "$@" From 474ababc0426b3f05aa80fa0d5f3126a343692de Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Thu, 16 Jul 2026 11:33:26 +0200 Subject: [PATCH 2/6] add check-links run script - fix latent bug in check_links.py: empty or missing _site/ would report success with no errors; now aborts - check-links.sh: wraps check_links.py in run script format - `./run commands` (and other command printers): wider output for command list to accomodate "check-links" --- build_scripts/check_links.py | 10 +++++ build_scripts/run_scripts/README.md | 19 +++++---- build_scripts/run_scripts/check-links.sh | 54 ++++++++++++++++++++++++ run | 2 +- 4 files changed, 76 insertions(+), 9 deletions(-) create mode 100755 build_scripts/run_scripts/check-links.sh diff --git a/build_scripts/check_links.py b/build_scripts/check_links.py index 78ac8c71..339a0baa 100644 --- a/build_scripts/check_links.py +++ b/build_scripts/check_links.py @@ -851,10 +851,20 @@ def cmd_validate(args: list[str], progress: _Progress, base_url: "Callable[[], s under = next((arg[len("under=") :] for arg in args if arg.startswith("under=")), None) positional = [arg for arg in args if arg not in flags and not arg.startswith("under=")] root = Path(positional[0]) if positional else Path("_site") + if not root.is_dir(): + # A missing root would otherwise scan zero pages and "pass"; that + # silence would hide a forgotten or failed build. + print(f"error: site root not found: {root} (build the site first)", file=sys.stderr) + return 2 source_prefix = os.path.normpath(root / under) if under else None progress.phase = "enumerating" refs, anchors_by_file = enumerate_site(root, source_prefix, progress) + if progress.pages == 0: + # Same silence risk as a missing root: a site with no HTML means + # nothing was validated, not that nothing is broken. + print(f"error: no HTML pages found under {root} (build the site first)", file=sys.stderr) + return 2 existing = _existing_files(root) progress.phase = "resolving" # Resolve the base URL only when needed: local mode never checks live, so it stays offline and config-free. diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index 779e26d0..f1ac88b5 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -19,14 +19,14 @@ problems in one pass. It installs nothing. ## Requirements -| Tool | Needed by | Notes | -|------------|---------------------------|-----------------------------------------------------------------------------| -| bash | everything | 3.2+; Git Bash on Windows, but available by default on most other platforms | -| Python | `build`, `serve`, `watch` | 3.11 or newer | -| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | -| docfx | `build`, `serve`, `watch` | local dotnet tool or global | -| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | -| shellcheck | `lint` (shell sources) | | +| Tool | Needed by | Notes | +|------------|------------------------------------------|-----------------------------------------------------------------------------| +| bash | everything | 3.2+; Git Bash on Windows, but available by default on most other platforms | +| Python | `build`, `serve`, `watch`, `check-links` | 3.11 or newer | +| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | +| docfx | `build`, `serve`, `watch` | local dotnet tool or global | +| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | +| shellcheck | `lint` (shell sources) | | The first `./run lint` needs network access: uvx downloads ruff and mypy into its cache on first use. @@ -102,6 +102,9 @@ All sub-commands provide help text at the command line. - Lint build scripts and run scripts: - `./run lint`: lint everything that is known to already conform to linting standards (all shell scripts and a subset of build scripts) - `./run lint file1.sh file2.py ...`: lint specific shell scripts or python scripts (pass as many filenames of either or both types as you like) +- Check the built site for broken links (needs a built site: run `./run build` first) + - `./run check-links`: full check; fetches every unique external URL once, so it needs network access and can take a while + - `./run check-links local`: offline; on-disk files and anchors only - Clean build artifacts: `./run clean` - Get help: - `./run help`: print top-level help for the `run` dispatcher and list all subcommands diff --git a/build_scripts/run_scripts/check-links.sh b/build_scripts/run_scripts/check-links.sh new file mode 100755 index 00000000..86a2e638 --- /dev/null +++ b/build_scripts/run_scripts/check-links.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +################################################################################ +# Check the built site for broken links +# +# Wraps build_scripts/check_links.py: walks the built HTML in _site, resolves +# every href/src to a local file or external URL, and reports references to +# broken targets. Local failures (missing files or anchors) are errors and +# fail the run; external failures are warnings, listed for verification by +# hand. Needs a built site: run `./run build` first. +# +# The full check fetches every unique external URL once, so it needs network +# access and can take a while; `local` stays entirely on disk. +# +# Arguments pass through to check_links.py validate: +# local skip external fetching (on-disk checks only; offline) +# all include generated API and localized pages +# (default: authored content only) +# stats print per-host fetch diagnostics +# under= only check links from pages under _site/ +# +# Usage: +# ./run check-links # full check (fetches external URLs) +# ./run check-links local # offline: on-disk checks only +# ./run check-links local all # offline, including generated pages +# ./run check-links help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +check () { + # validate the built site; args pass through to check_links.py validate + require python + log_and_run "$RUN_PYTHON" build_scripts/check_links.py validate "$@" +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else passes through +# to `check` as validate arguments, so `./run check-links local` just works. +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" +else + check "$@" +fi diff --git a/run b/run index 6e48ffd0..b6c5b3ea 100755 --- a/run +++ b/run @@ -33,7 +33,7 @@ commands () { if [ "$name" = "lib" ]; then continue fi - printf ' %-8s %s\n' "$name" "$(banner_title "$script")" + printf ' %-12s %s\n' "$name" "$(banner_title "$script")" done printf '\nRun ./run help for details on one command.\n' printf 'Run ./run help for full runner documentation.\n' From b0565e707c9b1efe9371dc1d0fba0a978a1102a5 Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Thu, 16 Jul 2026 12:14:42 +0200 Subject: [PATCH 3/6] Add doctest run script - update te_script_runner.py to be aware of `RUN_TE` environment variable - doctest.sh wraps `csharp_doctest.py` --- build_scripts/run_scripts/README.md | 34 ++++++--- build_scripts/run_scripts/doctest.sh | 107 +++++++++++++++++++++++++++ build_scripts/run_scripts/setup.sh | 7 +- build_scripts/te_script_runner.py | 6 +- 4 files changed, 141 insertions(+), 13 deletions(-) create mode 100755 build_scripts/run_scripts/doctest.sh diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index f1ac88b5..2ff6e801 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -19,14 +19,15 @@ problems in one pass. It installs nothing. ## Requirements -| Tool | Needed by | Notes | -|------------|------------------------------------------|-----------------------------------------------------------------------------| -| bash | everything | 3.2+; Git Bash on Windows, but available by default on most other platforms | -| Python | `build`, `serve`, `watch`, `check-links` | 3.11 or newer | -| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | -| docfx | `build`, `serve`, `watch` | local dotnet tool or global | -| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | -| shellcheck | `lint` (shell sources) | | +| Tool | Needed by | Notes | +|------------|-----------------------------------------------------|-------------------------------------------------------------| +| bash | everything | 3.2+; Git Bash on Windows; included on most other platforms | +| Python | `build`, `serve`, `watch`, `check-links`, `doctest` | 3.11 or newer | +| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | +| docfx | `build`, `serve`, `watch` | local dotnet tool or global | +| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | +| shellcheck | `lint` (shell sources) | | +| te | `doctest` (all but validate) | Tabular Editor CLI | The first `./run lint` needs network access: uvx downloads ruff and mypy into its cache on first use. @@ -42,6 +43,7 @@ If you prefer, though, you can follow these links to each tool's official instal - docfx: https://dotnet.github.io/docfx/ - uv: https://docs.astral.sh/uv/#installation - shellcheck: https://github.com/koalaman/shellcheck#user-content-installing +- te (Tabular Editor CLI): https://tabulareditor.com/download-tabular-editor-cli ### Linux @@ -85,6 +87,15 @@ dotnet tool restore If you are running a `dotnet` older than 10.0, run `dotnet new tool-manifest` first. +### te CLI (all platforms) + +The te CLI is installed the same way on every platform: sign in at +[tabulareditor.com](https://tabulareditor.com/download-tabular-editor-cli), +download the archive for your platform and architecture, extract it, and put +`te` on your PATH (or point `RUN_TE` at it). Full instructions, including +verification, live in this repo's own docs: +[te CLI install guide](../../content/features/te-cli/te-cli-install.md). + ## Usage The most common commands are provided below for reference. @@ -105,6 +116,11 @@ All sub-commands provide help text at the command line. - Check the built site for broken links (needs a built site: run `./run build` first) - `./run check-links`: full check; fetches every unique external URL once, so it needs network access and can take a while - `./run check-links local`: offline; on-disk files and anchors only +- Test the annotated C# code blocks in the docs (needs the te CLI); without file args, automatically discovers all markdown files with annotated C# code blocks. + - `./run doctest`: compile and run all annotated code blocks; compare results to `**Output**` blocks, but do not update files + - `./run doctest file1.md file2.md`: same, for exactly the named files + - `./run doctest update`: compile, run, and replace `**Output**` blocks in all markdown files with annotated code blocks + - `./run doctest update file1.md path/to/file2.md`: same, but only for the named files. - Clean build artifacts: `./run clean` - Get help: - `./run help`: print top-level help for the `run` dispatcher and list all subcommands @@ -113,7 +129,7 @@ All sub-commands provide help text at the command line. ### Dependency overrides Every external tool (e.g., python, uvx, shellcheck) can be pinned to a specific executable with a `RUN_` environment variable: -`RUN_PYTHON`, `RUN_UVX`, `RUN_SHELLCHECK`. +`RUN_PYTHON`, `RUN_UVX`, `RUN_SHELLCHECK`, `RUN_TE`. When set, that path is the only candidate checked; there is no fallback to PATH. Example: ```bash diff --git a/build_scripts/run_scripts/doctest.sh b/build_scripts/run_scripts/doctest.sh new file mode 100755 index 00000000..7e8add7c --- /dev/null +++ b/build_scripts/run_scripts/doctest.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +################################################################################ +# Test the annotated C# code blocks in the docs against the te CLI +# +# Wraps build_scripts/csharp_doctest.py. Code blocks in content markdown opt +# in with a fenced-code annotation (```csharp {compile} or +# ```csharp {run ...}); see the header of csharp_doctest.py for the +# annotation reference. With no file arguments, every annotated markdown +# file under content/ is processed; having none is fine (reported, exit 0). +# +# Success is silent beyond the echoed commands; a failing file's full report +# is printed, and the run exits nonzero after all files have been processed. +# +# Needs the Tabular Editor CLI for everything except validate: `te` on PATH, +# or RUN_TE=/path/to/te. update REWRITES the documented outputs inside the +# markdown files. +# +# Usage: +# ./run doctest [file.md ...] # compare (the default) +# ./run doctest validate [file.md ...] # annotation coverage; no te needed +# ./run doctest compile [file.md ...] # compile-only (te --dry-run) +# ./run doctest update [file.md ...] # execute and rewrite doc outputs +# ./run doctest help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +_annotated_files () { + # print the content markdown files containing annotated csharp fences + grep --recursive --files-with-matches --include='*.md' -- '```csharp {' content || true +} + +_run_mode () { + # run csharp_doctest.py once per file (it takes exactly one file + # per invocation), continuing past failures so every file is reported. + # Explicit file arguments always win; discovery of annotated content + # files fills in only when no files are given. + # Success is silent: each file's report is captured and printed only + # when that file fails. + local mode="$1" + shift + local files=("$@") f + if [ "${#files[@]}" -eq 0 ]; then + while IFS= read -r f; do + files+=("$f") + done < <(_annotated_files) + fi + if [ "${#files[@]}" -eq 0 ]; then + info 'doctest: no annotated C# blocks found under content/; nothing to do' + return 0 + fi + local out status=0 + for f in "${files[@]}"; do + if ! out="$(log_and_run "$RUN_PYTHON" build_scripts/csharp_doctest.py "$mode" "$f")"; then + printf '%s\n' "$out" + status=1 + fi + done + return "$status" +} + +validate () { + # check annotation correctness: report on malformed annotations. + # no te requirement + require python + _run_mode validate "$@" +} + +compile () { + # compile every annotated block via te --dry-run; catches API drift + # without executing anything + require python te + _run_mode compile "$@" +} + +compare () { + # execute annotated blocks and compare their documented outputs + require python te + _run_mode compare "$@" +} + +update () { + # execute annotated blocks and REWRITE the documented outputs in the + # markdown files in place + require python te + _run_mode update "$@" +} + +help () { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else passes through +# to `compare` as file paths, so `./run doctest some-doc.md` just works. +if declare -F "${1:-compare}" >/dev/null; then + dispatch compare "$@" +else + compare "$@" +fi diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh index 606788f4..ae7a34b0 100755 --- a/build_scripts/run_scripts/setup.sh +++ b/build_scripts/run_scripts/setup.sh @@ -4,8 +4,9 @@ # Check that all required tooling is installed # # Verifies every dependency the run commands need: Python >= 3.11, uvx, -# and shellcheck, plus docfx (either the repo-local dotnet tool or a global -# install). Reports everything that is missing in one pass; installs nothing. +# and shellcheck, the te CLI, plus docfx (either the repo-local dotnet tool +# or a global install). Reports everything that is missing in one pass; +# installs nothing. # # Usage: # ./run setup @@ -38,7 +39,7 @@ check () { local failed=0 # dotnet is not in this list: _docfx_ok requires it only when no global # docfx exists, so it is reported exactly once and only when relevant. - require python shellcheck uvx || failed=1 + require python shellcheck te uvx || failed=1 if ! _docfx_ok; then error "docfx not found: neither 'dotnet docfx --version' nor 'docfx --version' succeeded." info 'See build_scripts/run_scripts/README.md for installation instructions.' diff --git a/build_scripts/te_script_runner.py b/build_scripts/te_script_runner.py index 98f7f2f2..5c8d1c35 100755 --- a/build_scripts/te_script_runner.py +++ b/build_scripts/te_script_runner.py @@ -28,6 +28,7 @@ """ import json +import os import shutil import subprocess import sys @@ -36,7 +37,10 @@ from pathlib import Path from typing import Any, NamedTuple -DEFAULT_TE_BIN = "te" +# RUN_TE is the ./run tooling's override convention (see +# build_scripts/run_scripts/README.md); honoring it here means a pinned te +# applies to everything built on this module. +DEFAULT_TE_BIN = os.environ.get("RUN_TE", "te") _WORKDIR_PREFIX = "te-script-run." # Emitted between the next-to-last and last script when last_only is set, so the runner # can report only the final script's output. Chosen to be absent from any real output. From 6d5c220644f8319be17778a38929c3feea3a427a Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Thu, 16 Jul 2026 13:23:03 +0200 Subject: [PATCH 4/6] clean up setup run script output --- build_scripts/run_scripts/README.md | 1 + build_scripts/run_scripts/lib.sh | 15 ++++++++++++--- build_scripts/run_scripts/setup.sh | 20 +++++++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index 2ff6e801..ca5f8f17 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -184,6 +184,7 @@ Shared helpers live in [lib.sh](./lib.sh); these are intended to be used for boi - `error`: print error messages prefixed with "ERROR: " - `log_and_run`: prints the command being run in bold on stderr; use for all commands that actually get run for a scripts work - `require`: automatically handle the env var for tool overrides and fail with info if the tool can't be found +- `run_var_name`: print the `RUN_` env var name for a tool name (the naming convention, defined in one place) - `dispatch`: provide a way to take a default function with args (see usage in commands for examples) - `banner_title`: print the first one-line summary of a comment banner at the top of a script file (for command lists) - `banner_help`: print the whole comment banner at the top of a script file (for help and usage) diff --git a/build_scripts/run_scripts/lib.sh b/build_scripts/run_scripts/lib.sh index 7b1054f7..42d9d1ec 100755 --- a/build_scripts/run_scripts/lib.sh +++ b/build_scripts/run_scripts/lib.sh @@ -25,6 +25,13 @@ log_and_run () { "$@" } +run_var_name () { + # Print the RUN_ variable name for a tool (uppercased, dashes to + # underscores: uvx -> RUN_UVX). The single source of the require naming + # convention. LC_ALL=C pins ASCII range semantics regardless of locale. + printf 'RUN_%s' "$(printf '%s' "$1" | LC_ALL=C tr 'a-z-' 'A-Z_')" +} + require () { # Check the given tools and resolve which executable to use for each. # Takes bare tool names, not paths. A pre-set RUN_ env var (name @@ -41,8 +48,7 @@ require () { _require_python || failed=1 continue fi - # LC_ALL=C pins ASCII range semantics regardless of locale. - var="RUN_$(printf '%s' "$cmd" | LC_ALL=C tr 'a-z-' 'A-Z_')" + var="$(run_var_name "$cmd")" val="${!var:-$cmd}" if command -v "$val" >/dev/null 2>&1; then export "$var=$val" @@ -53,7 +59,10 @@ require () { fi done if [ "${#missing[@]}" -gt 0 ]; then - error "missing required command(s): ${missing[*]}" + error 'missing required command(s):' + for cmd in "${missing[@]}"; do + info " $cmd" + done info 'See build_scripts/run_scripts/README.md for installation instructions.' return 1 fi diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh index ae7a34b0..34afb459 100755 --- a/build_scripts/run_scripts/setup.sh +++ b/build_scripts/run_scripts/setup.sh @@ -34,8 +34,20 @@ _docfx_ok () { "$RUN_DOTNET" docfx --version >/dev/null 2>&1 } +_docfx_desc () { + # Describe which docfx form resolved, mirroring _docfx_ok's probe order: + # a docfx on PATH, else the repo-local dotnet tool (RUN_DOTNET is set in + # that case, by _docfx_ok's require). + if command -v docfx >/dev/null 2>&1; then + command -v docfx + else + printf '%s docfx (repo-local dotnet tool)\n' "$(command -v "$RUN_DOTNET")" + fi +} + check () { - # verify all tooling the run commands need; report problems at once + # verify all tooling the run commands need and print what each tool + # resolved to; report problems all at once local failed=0 # dotnet is not in this list: _docfx_ok requires it only when no global # docfx exists, so it is reported exactly once and only when relevant. @@ -46,6 +58,12 @@ check () { failed=1 fi if [ "$failed" -eq 0 ]; then + local tool var + for tool in python shellcheck te uvx; do + var="$(run_var_name "$tool")" + printf ' %-11s %s\n' "$tool" "$(command -v "${!var}")" + done + printf ' %-11s %s\n' docfx "$(_docfx_desc)" info 'All dependencies found.' fi return "$failed" From d30b02b848f4612ccad69778668412667373e23b Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Thu, 16 Jul 2026 13:26:25 +0200 Subject: [PATCH 5/6] adopt `shfmt -w -s` standard formatting for scripts --- build_scripts/run_scripts/README.md | 13 ++++++------ build_scripts/run_scripts/build.sh | 8 ++++---- build_scripts/run_scripts/check-links.sh | 4 ++-- build_scripts/run_scripts/clean.sh | 4 ++-- build_scripts/run_scripts/doctest.sh | 14 ++++++------- build_scripts/run_scripts/lib.sh | 26 ++++++++++++------------ build_scripts/run_scripts/lint.sh | 14 ++++++------- build_scripts/run_scripts/serve.sh | 4 ++-- build_scripts/run_scripts/setup.sh | 12 +++++------ build_scripts/run_scripts/watch.sh | 4 ++-- run | 20 +++++++++--------- 11 files changed, 62 insertions(+), 61 deletions(-) diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index ca5f8f17..d8c707ca 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -147,6 +147,7 @@ a `DOCFX` environment variable, then a repo-local dotnet tool manifest, then a g ### Conventions for these scripts - bash 3.2 compatible (macOS default bash), ASCII only, long-form CLI flags. +- Formatted with shfmt defaults: `shfmt -w run build_scripts/run_scripts` (tab indentation, `name() {` function style). - Each script: banner help block, per-function comment docs, awk-extracted help, `dispatch "$@"` at the bottom. - Shared helpers live in `lib.sh`; command-specific state lives in the command's script. - All of this tooling is linted by `./run lint` (shellcheck for the shell scripts, ruff + mypy for the Python sources). @@ -249,11 +250,11 @@ The very first thing in the function definition must be comments that become the Example: ```shell -fn () { - # the help text for this function - # can span many lines - require ... - +fn() { + # the help text for this function + # can span many lines + require ... + ``` These functions must declare their required tools upfront with `require`, e.g., `require uvx`; @@ -320,7 +321,7 @@ These pull help text out of the banner comment and all leading comments in funct They are automatically formatted as you see when you run `./run help` or `./run subcommand help`. ```shell -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/build.sh b/build_scripts/run_scripts/build.sh index 50ec82fc..ca37e3c0 100755 --- a/build_scripts/run_scripts/build.sh +++ b/build_scripts/run_scripts/build.sh @@ -23,7 +23,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -common_flags () { +common_flags() { # Set FLAGS to the markdown-iteration base: English only, and reuse the # generated docfx.json configs (--skip-gen) when they exist. After a # `clean` they do not, so the generation step runs on the next build. @@ -33,14 +33,14 @@ common_flags () { fi } -full () { +full() { # regenerate the API reference; fail on DocFX warnings require python common_flags log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" } -fast () { +fast() { # reuse existing API metadata (--skip-api); tolerate DocFX warnings # (--permissive); the build that watch runs after each save require python @@ -48,7 +48,7 @@ fast () { log_and_run "$RUN_PYTHON" build-docs.py "${FLAGS[@]}" --skip-api --permissive } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/check-links.sh b/build_scripts/run_scripts/check-links.sh index 86a2e638..65610d2d 100755 --- a/build_scripts/run_scripts/check-links.sh +++ b/build_scripts/run_scripts/check-links.sh @@ -33,13 +33,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -check () { +check() { # validate the built site; args pass through to check_links.py validate require python log_and_run "$RUN_PYTHON" build_scripts/check_links.py validate "$@" } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/clean.sh b/build_scripts/run_scripts/clean.sh index e0e1556b..7b299cb9 100755 --- a/build_scripts/run_scripts/clean.sh +++ b/build_scripts/run_scripts/clean.sh @@ -20,7 +20,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -clean () { +clean() { # remove all generated build artifacts (gitignored); tracked source is # left intact log_and_run rm -rf _site obj docfx.json @@ -35,7 +35,7 @@ clean () { info "Clean complete." } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/doctest.sh b/build_scripts/run_scripts/doctest.sh index 7e8add7c..01715dc0 100755 --- a/build_scripts/run_scripts/doctest.sh +++ b/build_scripts/run_scripts/doctest.sh @@ -31,12 +31,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -_annotated_files () { +_annotated_files() { # print the content markdown files containing annotated csharp fences grep --recursive --files-with-matches --include='*.md' -- '```csharp {' content || true } -_run_mode () { +_run_mode() { # run csharp_doctest.py once per file (it takes exactly one file # per invocation), continuing past failures so every file is reported. # Explicit file arguments always win; discovery of annotated content @@ -65,34 +65,34 @@ _run_mode () { return "$status" } -validate () { +validate() { # check annotation correctness: report on malformed annotations. # no te requirement require python _run_mode validate "$@" } -compile () { +compile() { # compile every annotated block via te --dry-run; catches API drift # without executing anything require python te _run_mode compile "$@" } -compare () { +compare() { # execute annotated blocks and compare their documented outputs require python te _run_mode compare "$@" } -update () { +update() { # execute annotated blocks and REWRITE the documented outputs in the # markdown files in place require python te _run_mode update "$@" } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/lib.sh b/build_scripts/run_scripts/lib.sh index 42d9d1ec..f83566fd 100755 --- a/build_scripts/run_scripts/lib.sh +++ b/build_scripts/run_scripts/lib.sh @@ -6,17 +6,17 @@ # bash build_scripts/run_scripts/lib.sh require git python3 # Status/progress chatter, on stderr so stdout stays clean for command output. -info () { printf '%s\n' "$*" >&2; } +info() { printf '%s\n' "$*" >&2; } # Error message with a uniform prefix, on stderr. -error () { printf 'ERROR: %s\n' "$*" >&2; } +error() { printf 'ERROR: %s\n' "$*" >&2; } # Print a command then execute it, bold (like just's recipe echo) with a # "$ " prefix marking it as a command line. The echo goes to stderr so a # command's stdout stays clean for its own output; escape codes only when # stderr is a terminal and NO_COLOR is unset, so redirected output stays # plain. -log_and_run () { +log_and_run() { if [ -t 2 ] && [ -z "${NO_COLOR:-}" ]; then printf '\033[1m$ %s\033[0m\n' "$*" >&2 else @@ -25,14 +25,14 @@ log_and_run () { "$@" } -run_var_name () { +run_var_name() { # Print the RUN_ variable name for a tool (uppercased, dashes to # underscores: uvx -> RUN_UVX). The single source of the require naming # convention. LC_ALL=C pins ASCII range semantics regardless of locale. printf 'RUN_%s' "$(printf '%s' "$1" | LC_ALL=C tr 'a-z-' 'A-Z_')" } -require () { +require() { # Check the given tools and resolve which executable to use for each. # Takes bare tool names, not paths. A pre-set RUN_ env var (name # uppercased, dashes to underscores: RUN_UVX for uvx, RUN_SHELLCHECK @@ -69,14 +69,14 @@ require () { return "$failed" } -_python_ok () { +_python_ok() { # True if the given interpreter exists and is Python >= 3.11 (the # version floor for the repo's build tooling, matching CI). command -v "$1" >/dev/null 2>&1 && "$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' >/dev/null 2>&1 } -_require_python () { +_require_python() { # Locate a Python >= 3.11 and export it as RUN_PYTHON for commands to # invoke. When RUN_PYTHON is already set (contributor override) it is # the ONLY candidate checked, so a broken override fails loudly instead @@ -129,7 +129,7 @@ _require_python () { return 1 } -dispatch () { +dispatch() { # dispatch [args...]: call the function named by the first # remaining arg, or default_fn when none is given. Unknown names print # the calling script's help and fail with a usage error. @@ -149,7 +149,7 @@ dispatch () { fi } -banner_title () { +banner_title() { # Print only the first line of the #### banner block of the given script # (used by ./run for its one-line-per-command index). awk ' @@ -158,7 +158,7 @@ banner_title () { ' "$1" } -banner_help () { +banner_help() { # Print the #### banner block at the top of the given script, unprefixed. awk ' BEGIN { inIntro=0; introDone=0 } @@ -168,12 +168,12 @@ banner_help () { ' "$1" } -command_help () { +command_help() { # Print each function in the given script with its leading comment lines. # Underscore-prefixed functions are internal and excluded. printf 'Commands:\n' awk ' - /^[a-z][a-z_]* *\(\) *\{/ { fn=$1; inCmd=1; next } + /^[a-z][a-z_]* *\(\) *\{/ { fn=$1; sub(/\(\)$/, "", fn); inCmd=1; next } inCmd==1 && /^[ \t]+#/ { if (fn != "") { print " " fn; fn="" } gsub(/^[ \t]+# ?/, " ") @@ -185,7 +185,7 @@ command_help () { } # Dispatch only when executed directly; stay silent when sourced. -if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then set -euo pipefail "$@" fi diff --git a/build_scripts/run_scripts/lint.sh b/build_scripts/run_scripts/lint.sh index 04558acf..17ad9ffc 100755 --- a/build_scripts/run_scripts/lint.sh +++ b/build_scripts/run_scripts/lint.sh @@ -37,7 +37,7 @@ PYTHON_SOURCES=( ) SHELL_SOURCES=(run build_scripts/run_scripts/*.sh) -python () { +python() { # ruff + mypy --strict on the Python tooling (default: PYTHON_SOURCES) require uvx local files=("$@") @@ -51,7 +51,7 @@ python () { log_and_run "$RUN_UVX" mypy --strict --no-error-summary "${files[@]}" } -shell () { +shell() { # check the shell tooling with shellcheck (default: SHELL_SOURCES) require shellcheck local files=("$@") @@ -61,7 +61,7 @@ shell () { log_and_run "$RUN_SHELLCHECK" --external-sources "${files[@]}" } -all () { +all() { # python + shell; with file args, routes each by extension (run -> shell) if [ "$#" -eq 0 ]; then python @@ -71,9 +71,9 @@ all () { local py=() sh=() path for path in "$@"; do case "$path" in - *.py) py+=("$path") ;; - *.sh|run|*/run) sh+=("$path") ;; - *) info "lint: ignoring path with unrecognized extension: $path" ;; + *.py) py+=("$path") ;; + *.sh | run | */run) sh+=("$path") ;; + *) info "lint: ignoring path with unrecognized extension: $path" ;; esac done if [ "${#py[@]}" -eq 0 ] && [ "${#sh[@]}" -eq 0 ]; then @@ -88,7 +88,7 @@ all () { fi } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/serve.sh b/build_scripts/run_scripts/serve.sh index a92b7cf3..28dad7f7 100755 --- a/build_scripts/run_scripts/serve.sh +++ b/build_scripts/run_scripts/serve.sh @@ -22,13 +22,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -serve () { +serve() { # full English build, then serve at http://localhost:8080 (Ctrl-C stops) require python log_and_run "$RUN_PYTHON" build-docs.py --serve --permissive } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh index 34afb459..e1b808bc 100755 --- a/build_scripts/run_scripts/setup.sh +++ b/build_scripts/run_scripts/setup.sh @@ -20,11 +20,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -_docfx_ok () { +_docfx_ok() { # True if docfx is invocable in either form build-docs.py resolves: # the repo-local dotnet tool, or a docfx on PATH. - if docfx --version >/dev/null 2>&1 - then return 0 + if docfx --version >/dev/null 2>&1; then + return 0 fi # This function runs inside a conditional (check's `if ! _docfx_ok`), @@ -34,7 +34,7 @@ _docfx_ok () { "$RUN_DOTNET" docfx --version >/dev/null 2>&1 } -_docfx_desc () { +_docfx_desc() { # Describe which docfx form resolved, mirroring _docfx_ok's probe order: # a docfx on PATH, else the repo-local dotnet tool (RUN_DOTNET is set in # that case, by _docfx_ok's require). @@ -45,7 +45,7 @@ _docfx_desc () { fi } -check () { +check() { # verify all tooling the run commands need and print what each tool # resolved to; report problems all at once local failed=0 @@ -69,7 +69,7 @@ check () { return "$failed" } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/build_scripts/run_scripts/watch.sh b/build_scripts/run_scripts/watch.sh index 9ed3e52f..ae438cc7 100755 --- a/build_scripts/run_scripts/watch.sh +++ b/build_scripts/run_scripts/watch.sh @@ -23,7 +23,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" cd "$SCRIPT_DIR/../.." -watch () { +watch() { # poll content/ and templates/; run the fast build on each change require python info "Watching content/ and templates/ for changes (Ctrl-C to stop)..." @@ -42,7 +42,7 @@ watch () { done } -help () { +help() { # show this help banner_help "$0" command_help "$0" diff --git a/run b/run index b6c5b3ea..d0d5133c 100755 --- a/run +++ b/run @@ -24,7 +24,7 @@ SCRIPTS="$ROOT/build_scripts/run_scripts" source "$SCRIPTS/lib.sh" cd "$ROOT" -commands () { +commands() { # List every command script with the first line of its banner. printf 'Commands:\n' local script name @@ -39,7 +39,7 @@ commands () { printf 'Run ./run help for full runner documentation.\n' } -help () { +help() { # Full runner documentation (the banner above) plus the command index. banner_help "$0" commands @@ -51,14 +51,14 @@ if [ "$#" -gt 0 ]; then fi case "$cmd" in - commands|--list) - commands - exit 0 - ;; - help|--help|-h) - help - exit 0 - ;; +commands | --list) + commands + exit 0 + ;; +help | --help | -h) + help + exit 0 + ;; esac script="$SCRIPTS/$cmd.sh" From 0b868146cb0abc5bc6a478319f57dba8ddc8646d Mon Sep 17 00:00:00 2001 From: Greg Baldini Date: Thu, 16 Jul 2026 15:34:23 +0200 Subject: [PATCH 6/6] Run script reorganization and cleanup - introduce pyproject.toml for python tool and file list centralization; also supports other tools (e.g. editors) - adopt ruff formatting standards for python files under run script purview - standardize banner comment contents in run scripts - introducs `scripts.sh`: single entry point for run scripts operating on run scripts (linting, formatting, scaffolding new scripts); this allows `./run setup` to focus on doc contributors and reduces dependencies for anyone *not* developing scripts - enhanced and reorganized README to split doc contributor vs script contributor sections; added code style guidance --- build_scripts/check_links.py | 20 +- build_scripts/csharp_doctest.py | 3 +- build_scripts/run_scripts/README.md | 217 +++++++++++++----- build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE | 36 +++ build_scripts/run_scripts/build.sh | 2 + build_scripts/run_scripts/check-links.sh | 2 + build_scripts/run_scripts/doctest.sh | 58 +++-- build_scripts/run_scripts/lib.sh | 27 ++- build_scripts/run_scripts/lint.sh | 103 --------- build_scripts/run_scripts/scripts.sh | 169 ++++++++++++++ build_scripts/run_scripts/serve.sh | 2 + build_scripts/run_scripts/setup.sh | 19 +- build_scripts/run_scripts/watch.sh | 2 + build_scripts/te_script_runner.py | 12 +- pyproject.toml | 23 ++ 15 files changed, 484 insertions(+), 211 deletions(-) create mode 100644 build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE delete mode 100755 build_scripts/run_scripts/lint.sh create mode 100755 build_scripts/run_scripts/scripts.sh create mode 100644 pyproject.toml diff --git a/build_scripts/check_links.py b/build_scripts/check_links.py index 339a0baa..a48b8c64 100644 --- a/build_scripts/check_links.py +++ b/build_scripts/check_links.py @@ -348,7 +348,7 @@ def fetch_external(url: str, need_body: bool) -> FetchResult: `_GET_ONLY_HOSTS`). GET downloads the body only when a fragment must be verified.""" target = "https:" + url if url.startswith("//") else url # protocol-relative //host needs a scheme get_only = need_body or _host_of(url) in _GET_ONLY_HOSTS - for method in (("GET",) if get_only else ("HEAD", "GET")): + for method in ("GET",) if get_only else ("HEAD", "GET"): request = urllib.request.Request(target, method=method, headers={"User-Agent": _USER_AGENT}) try: with urllib.request.urlopen(request, timeout=20) as response: @@ -642,7 +642,7 @@ def map_to_source(built: str, root: Path) -> str | None: def _anchor_member(type_stem: str, fragment: str) -> str: """Best-effort member name from a DocFX member anchor; empty when the anchor scheme does not match.""" prefix = type_stem.replace(".", "_") + "_" - return fragment[len(prefix):].split("_", 1)[0] if fragment.startswith(prefix) else "" + return fragment[len(prefix) :].split("_", 1)[0] if fragment.startswith(prefix) else "" def _fragment_suffix(target: Target) -> str: @@ -826,16 +826,20 @@ def rank(item: tuple[str, HostStats]) -> tuple[int, int, int, float]: return outcome.broken, outcome.partial, stat.rate_limited, stat.wait_seconds print(f"\n--- stats ({elapsed:.1f}s, {len(active)} hosts) ---") - print(f"{'total':>5} {'ok':>5} {'frag':>5} {'bad':>5} {'reqs':>5} {'429':>4} {'401':>4} {'403':>4} {'404':>4} " - f"{'oth':>4} {'net':>4} {'wait(s)':>8} {'fb':>4} {'cap':>4} host [retry-after seen]") + print( + f"{'total':>5} {'ok':>5} {'frag':>5} {'bad':>5} {'reqs':>5} {'429':>4} {'401':>4} {'403':>4} {'404':>4} " + f"{'oth':>4} {'net':>4} {'wait(s)':>8} {'fb':>4} {'cap':>4} host [retry-after seen]" + ) for host, stat in sorted(active.items(), key=rank, reverse=True): oc = outcomes.get(host, empty) seen = sorted(stat.retry_after_seen) tail = f" {seen}" if stat.rate_limited else "" - print(f"{oc.total:>5} {oc.ok:>5} {oc.partial:>5} {oc.broken:>5} {stat.requests:>5} {stat.rate_limited:>4} " - f"{oc.status.get(401, 0):>4} {oc.status.get(403, 0):>4} {oc.status.get(404, 0):>4} " - f"{oc.other_http:>4} {oc.transport:>4} {stat.wait_seconds:>8.1f} " - f"{stat.head_fallbacks:>4} {stat.final_cap:>4} {host}{tail}") + print( + f"{oc.total:>5} {oc.ok:>5} {oc.partial:>5} {oc.broken:>5} {stat.requests:>5} {stat.rate_limited:>4} " + f"{oc.status.get(401, 0):>4} {oc.status.get(403, 0):>4} {oc.status.get(404, 0):>4} " + f"{oc.other_http:>4} {oc.transport:>4} {stat.wait_seconds:>8.1f} " + f"{stat.head_fallbacks:>4} {stat.final_cap:>4} {host}{tail}" + ) def cmd_validate(args: list[str], progress: _Progress, base_url: "Callable[[], str]") -> int: diff --git a/build_scripts/csharp_doctest.py b/build_scripts/csharp_doctest.py index 220c1be5..3b348790 100644 --- a/build_scripts/csharp_doctest.py +++ b/build_scripts/csharp_doctest.py @@ -352,8 +352,7 @@ def _classify_fence(fence: Fence, following: Fence | None) -> Block | None: first = fence.annotation.split(None, 1)[0] if fence.annotation else "" if first in ("compile", "run"): raise ValueError( - f"{first} annotation is only valid on a csharp block " - f"(line {fence.line}, lang={fence.lang or 'plain'})" + f"{first} annotation is only valid on a csharp block (line {fence.line}, lang={fence.lang or 'plain'})" ) return None annotation = fence.annotation diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index d8c707ca..6926f5a1 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -19,20 +19,18 @@ problems in one pass. It installs nothing. ## Requirements -| Tool | Needed by | Notes | -|------------|-----------------------------------------------------|-------------------------------------------------------------| -| bash | everything | 3.2+; Git Bash on Windows; included on most other platforms | -| Python | `build`, `serve`, `watch`, `check-links`, `doctest` | 3.11 or newer | -| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | -| docfx | `build`, `serve`, `watch` | local dotnet tool or global | -| uvx | `lint` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | -| shellcheck | `lint` (shell sources) | | -| te | `doctest` (all but validate) | Tabular Editor CLI | +| Tool | Needed by | Notes | +|--------|-----------------------------------------------------|-------------------------------------------------------------| +| bash | everything | 3.2+; Git Bash on Windows; included on most other platforms | +| Python | `build`, `serve`, `watch`, `check-links`, `doctest` | 3.11 or newer | +| dotnet | docfx install and hosting | .NET SDK 8.0 or newer | +| docfx | `build`, `serve`, `watch` | local dotnet tool or global | +| te | `doctest` (all but validate) | Tabular Editor CLI | -The first `./run lint` needs network access: uvx downloads ruff and mypy into -its cache on first use. +Working on these scripts (python and shell scripts) needs a few more tools; +see [Run script development](#run-script-development) under Contributing below. -## Installation +### Installation We provide the most common and script-friendly installation methods per platform in each section below. If you prefer, though, you can follow these links to each tool's official installation docs @@ -41,41 +39,35 @@ If you prefer, though, you can follow these links to each tool's official instal - Bash (non-windows): https://www.gnu.org/software/bash/ - Python: https://www.python.org/ - docfx: https://dotnet.github.io/docfx/ -- uv: https://docs.astral.sh/uv/#installation -- shellcheck: https://github.com/koalaman/shellcheck#user-content-installing - te (Tabular Editor CLI): https://tabulareditor.com/download-tabular-editor-cli -### Linux +#### Linux ```bash -# Python and shellcheck from your distro, e.g. Debian/Ubuntu: -sudo apt-get install python3 shellcheck -# uv (provides uvx): -curl -LsSf https://astral.sh/uv/install.sh | sh +# Python from your distro, e.g. Debian/Ubuntu: +sudo apt-get install python3 # .NET SDK (for docfx): see https://learn.microsoft.com/dotnet/core/install/linux ``` -### macOS +#### macOS ```bash -brew install python3 uv shellcheck +brew install python3 brew install --cask dotnet-sdk # .NET SDK (for docfx) ``` -### Windows +#### Windows Install [Git for Windows](https://gitforwindows.org/) for Git Bash, then: ```powershell winget install Python.PythonInstallManager -winget install astral-sh.uv -winget install koalaman.shellcheck winget install Microsoft.DotNet.SDK.10 ``` Note: winget has no unversioned .NET SDK id; docfx needs SDK 8.0 or newer. -### docfx (all platforms) +#### docfx (all platforms) With the .NET SDK installed (see your platform above), install docfx as a repo-local dotnet tool. From the repo root: @@ -87,13 +79,13 @@ dotnet tool restore If you are running a `dotnet` older than 10.0, run `dotnet new tool-manifest` first. -### te CLI (all platforms) +#### te CLI (all platforms) -The te CLI is installed the same way on every platform: sign in at -[tabulareditor.com](https://tabulareditor.com/download-tabular-editor-cli), -download the archive for your platform and architecture, extract it, and put -`te` on your PATH (or point `RUN_TE` at it). Full instructions, including -verification, live in this repo's own docs: +The te CLI is installed the same way on every platform: +sign in at [tabulareditor.com](https://tabulareditor.com/download-tabular-editor-cli), +download the archive for your platform and architecture, extract it, +and put `te` on your PATH (or point `RUN_TE` at it). +Full instructions, including verification, live in this repo's own docs: [te CLI install guide](../../content/features/te-cli/te-cli-install.md). ## Usage @@ -102,25 +94,22 @@ The most common commands are provided below for reference. All sub-commands provide help text at the command line. - Check all dependencies are installed: - - `./run setup` - - `RUN_UVX=/path/to/uvx RUN_PYTHON=/path/to/python ./run setup`: confirm that tools at custom paths are resolved correctly -- Hot reloading iteration on markdown content (run in two separate terminals) + - `./run setup`: run only to start and ensure you have the correct tools + - `RUN_TE=/path/to/te RUN_PYTHON=/path/to/python ./run setup`: (advanced usage) confirm that tools at custom paths are resolved correctly +- Hot reloading iteration on markdown content (run in two separate terminals); likely all you need for most content contribution: - `./run serve`: builds docs site and launches a localhost server at http://localhost:8080 where you can browse rendered docs - `./run watch`: watches for changes in template files and markdown files, runs a fast rebuild upon detecting changes -- One-off build (pick one) +- One-off build (pick one): - `./run build`: required after a `./run clean` or a fresh clone of the repo - `./run build fast`: skip re-generating the API docs; what you probably want most of the time and saves 1/3-1/2 build time -- Lint build scripts and run scripts: - - `./run lint`: lint everything that is known to already conform to linting standards (all shell scripts and a subset of build scripts) - - `./run lint file1.sh file2.py ...`: lint specific shell scripts or python scripts (pass as many filenames of either or both types as you like) -- Check the built site for broken links (needs a built site: run `./run build` first) +- Check the built site for broken links (needs a built site: run `./run build` first): - `./run check-links`: full check; fetches every unique external URL once, so it needs network access and can take a while - `./run check-links local`: offline; on-disk files and anchors only -- Test the annotated C# code blocks in the docs (needs the te CLI); without file args, automatically discovers all markdown files with annotated C# code blocks. +- Test the annotated C# code blocks in the docs (needs the te CLI); without file args, automatically discovers all markdown files with annotated C# code blocks: - `./run doctest`: compile and run all annotated code blocks; compare results to `**Output**` blocks, but do not update files - `./run doctest file1.md file2.md`: same, for exactly the named files - `./run doctest update`: compile, run, and replace `**Output**` blocks in all markdown files with annotated code blocks - - `./run doctest update file1.md path/to/file2.md`: same, but only for the named files. + - `./run doctest update file1.md path/to/file2.md`: same, but only for the named files - Clean build artifacts: `./run clean` - Get help: - `./run help`: print top-level help for the `run` dispatcher and list all subcommands @@ -128,9 +117,10 @@ All sub-commands provide help text at the command line. ### Dependency overrides -Every external tool (e.g., python, uvx, shellcheck) can be pinned to a specific executable with a `RUN_` environment variable: -`RUN_PYTHON`, `RUN_UVX`, `RUN_SHELLCHECK`, `RUN_TE`. -When set, that path is the only candidate checked; there is no fallback to PATH. Example: +Every external tool can be pinned to a specific executable with a `RUN_` environment variable: +`RUN_PYTHON`, `RUN_TE` (and, for the tooling developers' tools, `RUN_UVX`, `RUN_SHELLCHECK`, `RUN_SHFMT`). +When set, that path is the only candidate checked; there is no fallback to your `PATH` environment variable. +Example: ```bash RUN_PYTHON=/opt/python3.12/bin/python3 ./run build @@ -144,18 +134,67 @@ a `DOCFX` environment variable, then a repo-local dotnet tool manifest, then a g ## Contributing +This section is specifically about contributing to the run scripts themselves. +For guidance on contributing documentation content, [see the project README](../../README.md#bookmark-links-and-translations) +and the authoring guidance below that section. + +### Run script development + +`./run scripts` and its nested commands are the gateway to run scripts about run scripts. + +- Check the script-development tools are installed: `./run scripts setup` +- Lint and verify formatting, without modifying anything (the check a PR should pass): + - `./run scripts check`: everything (all shell tooling, the qualified Python sources) + - `./run scripts check file1.sh file2.py ...`: specific files, routed by kind +- Apply the formatters (writes files): `./run scripts format [file ...]` +- Scaffold a new run script: `./run scripts new ` (creates `build_scripts/run_scripts/.sh` from the anatomy below, ready to edit) + +Additional requirements beyond the doc-contributor table above: + +| Tool | Needed by | Notes | +|------------|------------------------------|------------------------------------------| +| uvx | `scripts` (Python sources) | part of [uv](https://docs.astral.sh/uv/) | +| shellcheck | `scripts` (shell sources) | | +| shfmt | `scripts` (shell formatting) | [mvdan/sh](https://github.com/mvdan/sh) | + +The first `./run scripts check` needs network access: uvx downloads ruff and mypy into its cache on first use. + +**Installation:** + +```bash +# Linux (Debian/Ubuntu): +sudo apt-get install shellcheck shfmt +curl -LsSf https://astral.sh/uv/install.sh | sh # uv provides uvx + +# macOS: +brew install uv shellcheck shfmt +``` + +```powershell +# Windows: +winget install astral-sh.uv +winget install koalaman.shellcheck +winget install mvdan.shfmt +``` + ### Conventions for these scripts -- bash 3.2 compatible (macOS default bash), ASCII only, long-form CLI flags. -- Formatted with shfmt defaults: `shfmt -w run build_scripts/run_scripts` (tab indentation, `name() {` function style). -- Each script: banner help block, per-function comment docs, awk-extracted help, `dispatch "$@"` at the bottom. -- Shared helpers live in `lib.sh`; command-specific state lives in the command's script. -- All of this tooling is linted by `./run lint` (shellcheck for the shell scripts, ruff + mypy for the Python sources). +- bash 3.2 compatible (macOS default bash), ASCII only, long-form CLI flags where a long form exists +- Shell formatted by `./run scripts format` + (shfmt -s: tab indentation, `name() {` function style, simplifications applied) +- Python tooling formatted with ruff format (also via `./run scripts format`); + lint rules, line length, strict mode, and the qualified-file lists live in pyproject.toml at the repo root + (the ruff `include` and mypy `files` lists sit adjacent there and must stay in sync) +- Each script: banner help block, per-function comment docs, awk-extracted help, `dispatch "$@"` at the bottom +- Shared helpers live in `lib.sh`; command-specific state lives in the command's script +- All of this tooling must pass `./run scripts check` + (shellcheck + shfmt diff for the shell sources; ruff, ruff format diff, and mypy for the Python sources) - Any internal function not intended for re-use is prefixed with an underscore ### General architecture -Everything flows through the `run` command, which is a dispatcher to various special-purpose scripts, exposed as subcommands to `run`. +Everything flows through the `run` command, +which is a dispatcher to various special-purpose scripts, exposed as subcommands to `run`. Every subcommand is a script in this directory (run_scripts/). The script must be named `name.sh` and `name` becomes the name of the sub-command; there is no other registration for sub-commands, if the script exists, it's a subcommand. @@ -184,8 +223,11 @@ Shared helpers live in [lib.sh](./lib.sh); these are intended to be used for boi - `info`: print informational messages onto stderr for the user - `error`: print error messages prefixed with "ERROR: " - `log_and_run`: prints the command being run in bold on stderr; use for all commands that actually get run for a scripts work +- `log_cmd`: the printing half of `log_and_run` on its own; + use when the command's execution is indirected (e.g. parallel workers whose output is captured, as in `doctest.sh`) - `require`: automatically handle the env var for tool overrides and fail with info if the tool can't be found - `run_var_name`: print the `RUN_` env var name for a tool name (the naming convention, defined in one place) +- `print_resolved`: print each given tool with the executable path its `RUN_` resolved to (for setup-style reports; call `require` first) - `dispatch`: provide a way to take a default function with args (see usage in commands for examples) - `banner_title`: print the first one-line summary of a comment banner at the top of a script file (for command lists) - `banner_help`: print the whole comment banner at the top of a script file (for help and usage) @@ -288,7 +330,7 @@ python () { ``` Invoking `"$RUN_UVX"` (quotes included; the path may contain spaces) is what makes a user's override actually take effect. -A bare `uvx` on that line would search the user's `PATH`, but if they provided `RUN_UVX=/other/path/to/uvx ./run lint`, +A bare `uvx` on that line would search the user's `PATH`, but if they provided `RUN_UVX=/other/path/to/uvx ./run scripts check`, then they are explicitly telling us, "Use the `uvx` at the provided path, not from `PATH`," so we honor that. These exports also flow into any script the current one delegates to (e.g., `watch.sh` runs `build.sh fast`, which sees the same `RUN_PYTHON`), @@ -348,8 +390,8 @@ Take `build.sh`, which ends with `dispatch full "$@"`, as a worked example. - When a user submits `./run build nonsense`, we end up with `dispatch full nonsense`. There is no `nonsense` function in the script, so dispatch prints an error, then the script's `help`, and exits with a usage error. - Anything after the function name is handed to that function as its arguments. - In `lint.sh` (default `all`), `./run lint python a.py b.py` ends up as `dispatch all python a.py b.py`; - dispatch calls the `python` function with `a.py b.py` as its arguments. + In `scripts.sh` (default `check`), `./run scripts format a.py b.sh` ends up as `dispatch check format a.py b.sh`; + dispatch calls the `format` function with `a.py b.sh` as its arguments. Name the default function after what it does (`full`, `all`, `check`), not a generic `main`; function names are user interface here, since they are exactly what a user types after `./run `. @@ -359,13 +401,13 @@ There are exactly two correct ways to write the dispatch line: 1. The standard form shown above: `dispatch "$@"`. The default function name is required; `dispatch` always needs to know what to run when the user types nothing. 2. A wrapper around form 1, for a script that wants to handle unrecognized arguments itself instead of treating them as an error. - For example, `lint.sh` treats an unrecognized first argument as a list of files to lint: + For example, `scripts.sh` treats an unrecognized first argument as a list of files to check: ```shell -if declare -F "${1:-all}" >/dev/null; then - dispatch all "$@" +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" else - all "$@" + check "$@" fi ``` @@ -376,3 +418,64 @@ Two mistakes to avoid: If a script has no obvious default action, make `help` its default. - `"$@"` must always be written with the quotes. Without them, an argument containing a space (such as a file path) gets split into pieces before `dispatch` ever sees it. + + +### Shell script or python? + +We use shell scripts primarily for orchestration of other tools. +If you just want to provide a friendly wrapper for some more complex command, +or a bunch of reasonable defaults, then a shell script in this directory is the right choice. +If there is significant logic, data processing, or any sort of parsing, then a python script is probably the right choice. +If you build a python script that is a reasonable tool for most contributors to use, +then you should also give it a friendly run script wrapper here. +A good example of a run script wrapper improving contributor convenience is [doctest.sh](./doctest.sh); +this wraps a python tool that can only operate on one file, +and gives a run script interface that lets you pass many files, +calling the python script once for each. + +### Style guidelines + +Write everything as a series of small functions, +using dispatch patterns as in these run scripts and demonstrated in [`check_links.py`](../check_links.py), +[`te_script_runner.py`](../te_script_runner.py), and [`csharp_doctest.py`](../csharp_doctest.py). +This allows incremental testing and validation: +small pieces are built that do a part of the work and their logic is demonstrated to be sound; +then downstream components compose and orchestrate this functionality. +This style supports incremental validation, as well as ad hoc reuse of components. +We do not follow TDD (there are not test suites for these scripts), +but following this approach still keeps the code test*able*, which is critical. + + +General conventions and guidelines. + +- Output routing: stdout carries only a command's real output; + all chatter (progress, status, errors) goes to stderr via `info`/`error`, + so output stays clean for piping and redirection. +- Success is quiet: pick tool flags that silence success output + (e.g. ruff `--quiet`, mypy `--no-error-summary`), + or capture a step's report and print it only on failure (as `doctest.sh` does). + Anything printed should mean something needs attention. +- Ensure failures have non-0 exit stati; + for anything in a pipeline or other automation, this is the sole reliable indicator of failure. +- `log_and_run` executed commands: every command that performs a script's actual work runs through `log_and_run`, + or announces itself with `log_cmd` when its execution is indirected (see the parallel workers in `doctest.sh`); + silent probes and internal plumbing do not. + When delegating to another script, let the delegate echo its own commands: see `watch.sh` and `build.sh` +- Report everything at once: collect all failures (missing tools, broken files) and report them together, + one per line, before failing; this allows a user to remediate as much as possible at once, + rather than chasing one error at a time. + If there is useful documentation to point to in an error context, do so; this README is a good choice. +- No silent false successes: a checker that found nothing to check must fail or say so loudly, + never pass quietly (see the missing-root and zero-pages guards in `check_links.py`). +- Explicit user input always overrides defaults: if a user gives us filenames or sets an env var, + we use exactly what they have provided, rather than automatic discovery or default file lists. + See `RUN_` handling and all commands that accept file args. +- Run scripts should always accept multiple files, and route these files intelligently; + see the mixed .py and .sh handling in `scripts.sh`. +- A wrapper is a chance to fix warts in what it wraps, not to reproduce them; + when porting or wrapping existing behavior surfaces a bug, fix it. +- Use constructs portable across the BSD and GNU userlands: + plain `tr` ranges under `LC_ALL=C` rather than `[:class:]`-plus-literal sets, + and streaming filters rather than `sed -i` (whose flags differ between the two). +- Keep docs, comments, and implementation in sync. +- Ensure python scripts remain cleanly importable by using setup functions guarded in `if __name__ == '__main__':`. diff --git a/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE b/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE new file mode 100644 index 00000000..0cec5ee0 --- /dev/null +++ b/build_scripts/run_scripts/RUN_SCRIPT_TEMPLATE @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +################################################################################ +# TODO one-line summary, shown by './run commands' and './run help' +# +# TODO describe what this command does for the user running it. +# +# TODO list all RUN_ environment vars applicable to this command. +# +# Usage: +# ./run TODO +# ./run TODO help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +TODO_FUNCTION_NAME() { + # TODO help text for this function + # TODO declare tools upfront: require + require TODO + error 'not implemented' + return 1 #TODO return 0 on success +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +dispatch TODO_FUNCTION_DEFAULT "$@" diff --git a/build_scripts/run_scripts/build.sh b/build_scripts/run_scripts/build.sh index ca37e3c0..d9402ee3 100755 --- a/build_scripts/run_scripts/build.sh +++ b/build_scripts/run_scripts/build.sh @@ -11,6 +11,8 @@ # fast also reuse existing API metadata and tolerate DocFX warnings; # the same build that `watch` runs # +# Tool overrides (env vars): RUN_PYTHON. +# # Usage: # ./run build [full|fast] # ./run build help diff --git a/build_scripts/run_scripts/check-links.sh b/build_scripts/run_scripts/check-links.sh index 65610d2d..b0973716 100755 --- a/build_scripts/run_scripts/check-links.sh +++ b/build_scripts/run_scripts/check-links.sh @@ -19,6 +19,8 @@ # stats print per-host fetch diagnostics # under= only check links from pages under _site/ # +# Tool overrides (env vars): RUN_PYTHON. +# # Usage: # ./run check-links # full check (fetches external URLs) # ./run check-links local # offline: on-disk checks only diff --git a/build_scripts/run_scripts/doctest.sh b/build_scripts/run_scripts/doctest.sh index 01715dc0..0bf3318b 100755 --- a/build_scripts/run_scripts/doctest.sh +++ b/build_scripts/run_scripts/doctest.sh @@ -9,18 +9,28 @@ # annotation reference. With no file arguments, every annotated markdown # file under content/ is processed; having none is fine (reported, exit 0). # -# Success is silent beyond the echoed commands; a failing file's full report -# is printed, and the run exits nonzero after all files have been processed. +# Files are processed in parallel, at most DOCTEST_JOBS at a time (default +# 4; each invocation runs against its own throwaway model, and update only +# writes its own markdown file). Results print in the order files finish: +# every file's command line, plus the full report (including te's own +# stderr) only when that file failed. The run exits nonzero if any file +# failed. # # Needs the Tabular Editor CLI for everything except validate: `te` on PATH, # or RUN_TE=/path/to/te. update REWRITES the documented outputs inside the # markdown files. # +# NOTE: The correctness of the results of doctest` depend entirely on the +# alignment of your te CLI version with the scripting surface area +# being documented. If your te version is out of date, you may see false +# error reports from this command. +# # Usage: # ./run doctest [file.md ...] # compare (the default) # ./run doctest validate [file.md ...] # annotation coverage; no te needed # ./run doctest compile [file.md ...] # compile-only (te --dry-run) # ./run doctest update [file.md ...] # execute and rewrite doc outputs +# DOCTEST_JOBS=N ./run doctest # run at most N jobs at once # ./run doctest help ################################################################################ @@ -36,13 +46,29 @@ _annotated_files() { grep --recursive --files-with-matches --include='*.md' -- '```csharp {' content || true } +_run_one() { + # xargs worker: run one file, echo its command line on completion, and + # print its captured report (stdout and stderr, so parallel te chatter + # never interleaves) only on failure, as a single write so concurrent + # reports stay intact. + local -a cmd=("$RUN_PYTHON" build_scripts/csharp_doctest.py "$1" "$2") + local out + if out="$("${cmd[@]}" 2>&1)"; then + log_cmd "${cmd[@]}" + else + log_cmd "${cmd[@]}" + printf '%s\n' "$out" + return 1 + fi +} + _run_mode() { # run csharp_doctest.py once per file (it takes exactly one file - # per invocation), continuing past failures so every file is reported. - # Explicit file arguments always win; discovery of annotated content - # files fills in only when no files are given. - # Success is silent: each file's report is captured and printed only - # when that file fails. + # per invocation), at most DOCTEST_JOBS files at a time. Each file's + # own blocks already run in parallel inside csharp_doctest.py (one te + # per block), so an unbounded file-level fan-out would oversubscribe + # the machine. Explicit file arguments always win; discovery of + # annotated content files fills in only when no files are given. local mode="$1" shift local files=("$@") f @@ -55,13 +81,17 @@ _run_mode() { info 'doctest: no annotated C# blocks found under content/; nothing to do' return 0 fi - local out status=0 - for f in "${files[@]}"; do - if ! out="$(log_and_run "$RUN_PYTHON" build_scripts/csharp_doctest.py "$mode" "$f")"; then - printf '%s\n' "$out" - status=1 - fi - done + + # The xargs-spawned worker shells receive these through the environment. + export -f _run_one log_cmd + + # xargs runs the workers and refills free slots as files finish, so + # results print in finish order. Embedding $mode is safe: dispatch + # already resolved it to one of this script's function names. xargs + # exits 123 when any invocation failed (BSD and GNU alike). + local status=0 + printf '%s\0' "${files[@]}" | + xargs -0 -n 1 -P "${DOCTEST_JOBS:-4}" bash -c "_run_one $mode \"\$1\"" _ || status=1 return "$status" } diff --git a/build_scripts/run_scripts/lib.sh b/build_scripts/run_scripts/lib.sh index f83566fd..2bfcfc49 100755 --- a/build_scripts/run_scripts/lib.sh +++ b/build_scripts/run_scripts/lib.sh @@ -11,17 +11,22 @@ info() { printf '%s\n' "$*" >&2; } # Error message with a uniform prefix, on stderr. error() { printf 'ERROR: %s\n' "$*" >&2; } -# Print a command then execute it, bold (like just's recipe echo) with a -# "$ " prefix marking it as a command line. The echo goes to stderr so a -# command's stdout stays clean for its own output; escape codes only when -# stderr is a terminal and NO_COLOR is unset, so redirected output stays -# plain. -log_and_run() { +# Print a command line without running it, bold (like just's recipe echo) +# with a "$ " prefix. Goes to stderr so a command's stdout stays clean for +# its own output; escape codes only when stderr is a terminal and NO_COLOR +# is unset, so redirected output stays plain. Use directly when the command +# itself runs indirected (e.g. backgrounded with captured output). +log_cmd() { if [ -t 2 ] && [ -z "${NO_COLOR:-}" ]; then printf '\033[1m$ %s\033[0m\n' "$*" >&2 else printf '$ %s\n' "$*" >&2 fi +} + +# Print a command then execute it. +log_and_run() { + log_cmd "$@" "$@" } @@ -129,6 +134,16 @@ _require_python() { return 1 } +print_resolved() { + # Print each tool name with the executable path its RUN_ variable + # resolved to. Call require on the tools first. + local tool var + for tool in "$@"; do + var="$(run_var_name "$tool")" + printf ' %-11s %s\n' "$tool" "$(command -v "${!var}")" + done +} + dispatch() { # dispatch [args...]: call the function named by the first # remaining arg, or default_fn when none is given. Unknown names print diff --git a/build_scripts/run_scripts/lint.sh b/build_scripts/run_scripts/lint.sh deleted file mode 100755 index 17ad9ffc..00000000 --- a/build_scripts/run_scripts/lint.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash - -################################################################################ -# Lint the repo's tooling -# -# Python (*.py) via uvx: ruff (F,B,SIM,I,UP) + mypy --strict. -# Shell (*.sh, and the extensionless ./run) via shellcheck. -# -# Each command takes an optional list of files; with no files it lints the -# default source lists defined at the top of this script. Runs from the repo -# root, so file arguments are repo-root-relative. -# -# Tool overrides (env vars): RUN_UVX and RUN_SHELLCHECK name the executables -# to use for uvx and shellcheck; when set, only that path is checked. -# -# Usage: -# ./run lint # python + shell on the default lists -# ./run lint file1.sh file2.py ... # route a mixed file list by extension -# ./run lint python [file.py ...] -# ./run lint shell [file.sh ...] -# ./run lint help -################################################################################ - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=build_scripts/run_scripts/lib.sh -source "$SCRIPT_DIR/lib.sh" -cd "$SCRIPT_DIR/../.." - -# Default source lists: the tooling that meets the lint bar. Add files as -# they qualify. -PYTHON_SOURCES=( - build_scripts/check_links.py - build_scripts/te_script_runner.py - build_scripts/csharp_doctest.py -) -SHELL_SOURCES=(run build_scripts/run_scripts/*.sh) - -python() { - # ruff + mypy --strict on the Python tooling (default: PYTHON_SOURCES) - require uvx - local files=("$@") - if [ "${#files[@]}" -eq 0 ]; then - files=("${PYTHON_SOURCES[@]}") - fi - log_and_run "$RUN_UVX" ruff check --quiet --select F,B,SIM,I,UP "${files[@]}" - # --no-error-summary drops only the trailing tally line ("Found N errors - # ..."/"Success: ..."); per-error diagnostics and the exit code are - # unaffected, so success is silent like shellcheck. - log_and_run "$RUN_UVX" mypy --strict --no-error-summary "${files[@]}" -} - -shell() { - # check the shell tooling with shellcheck (default: SHELL_SOURCES) - require shellcheck - local files=("$@") - if [ "${#files[@]}" -eq 0 ]; then - files=("${SHELL_SOURCES[@]}") - fi - log_and_run "$RUN_SHELLCHECK" --external-sources "${files[@]}" -} - -all() { - # python + shell; with file args, routes each by extension (run -> shell) - if [ "$#" -eq 0 ]; then - python - shell - return - fi - local py=() sh=() path - for path in "$@"; do - case "$path" in - *.py) py+=("$path") ;; - *.sh | run | */run) sh+=("$path") ;; - *) info "lint: ignoring path with unrecognized extension: $path" ;; - esac - done - if [ "${#py[@]}" -eq 0 ] && [ "${#sh[@]}" -eq 0 ]; then - error "no lintable files among: $*" - return 2 - fi - if [ "${#py[@]}" -gt 0 ]; then - python "${py[@]}" - fi - if [ "${#sh[@]}" -gt 0 ]; then - shell "${sh[@]}" - fi -} - -help() { - # show this help - banner_help "$0" - command_help "$0" -} - -# First arg selects a command when it names one; anything else is treated as -# a file list for `all`, so `./run lint file1.sh file2.py` just works. -if declare -F "${1:-all}" >/dev/null; then - dispatch all "$@" -else - all "$@" -fi diff --git a/build_scripts/run_scripts/scripts.sh b/build_scripts/run_scripts/scripts.sh new file mode 100755 index 00000000..65fad25b --- /dev/null +++ b/build_scripts/run_scripts/scripts.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +################################################################################ +# Develop the repo tooling: check, format, and scaffold the scripts themselves +# +# Commands for working on the repo's own tooling: the run scripts and the +# qualified Python build scripts. Contributing docs does not need any of +# this; see `./run setup` for the doc-contributor checks. +# +# Python rules, line length, strict mode, and the qualified-file lists live +# in pyproject.toml at the repo root; the shell file list is defined at the +# top of this script. File arguments are routed by kind (.py to the Python +# tools; .sh and the `run` dispatcher to the shell tools) and always trump +# the configured lists. +# +# Tool overrides (env vars): RUN_UVX, RUN_SHELLCHECK, RUN_SHFMT. +# +# Usage: +# ./run scripts # check: linters + formatter diffs +# ./run scripts check [file ...] # same, explicitly +# ./run scripts format [file ...] # apply the formatters (writes files) +# ./run scripts new # scaffold a new run script +# ./run scripts setup # check the script-development tools +# ./run scripts help +################################################################################ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=build_scripts/run_scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" +cd "$SCRIPT_DIR/../.." + +# Default shell source list: the tooling that meets the check bar. The Python +# equivalent lives in pyproject.toml (ruff include / mypy files). +SHELL_SOURCES=(run build_scripts/run_scripts/*.sh) + +_check_python() { + # non-mutating: ruff lint, ruff format diff, mypy (slowest last) + require uvx + log_and_run "$RUN_UVX" ruff check --quiet "$@" + log_and_run "$RUN_UVX" ruff format --check --quiet "$@" + # --no-error-summary drops only the trailing tally line ("Found N errors + # ..."/"Success: ..."); per-error diagnostics and the exit code are + # unaffected, so success is silent like shellcheck. + log_and_run "$RUN_UVX" mypy --no-error-summary "$@" +} + +_check_shell() { + # non-mutating: shellcheck, then shfmt as a diff + require shellcheck shfmt + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${SHELL_SOURCES[@]}") + fi + log_and_run "$RUN_SHELLCHECK" --external-sources "${files[@]}" + # -s simplifies the code where possible; -d prints a diff and fails + # when formatting is off (shfmt has no long-form flags) + log_and_run "$RUN_SHFMT" -s -d "${files[@]}" +} + +_format_python() { + require uvx + log_and_run "$RUN_UVX" ruff format "$@" +} + +_format_shell() { + require shfmt + local files=("$@") + if [ "${#files[@]}" -eq 0 ]; then + files=("${SHELL_SOURCES[@]}") + fi + log_and_run "$RUN_SHFMT" -s -w "${files[@]}" +} + +_route() { + # Split file args by kind and hand them to the given python/shell + # functions. With no files, call both with none, so each falls back to + # its configured or default list. + local python_fn="$1" shell_fn="$2" + shift 2 + if [ "$#" -eq 0 ]; then + "$python_fn" + "$shell_fn" + return + fi + local py=() sh=() path + for path in "$@"; do + case "$path" in + *.py) py+=("$path") ;; + *.sh | run | */run) sh+=("$path") ;; + *) info "scripts: ignoring path of unrecognized kind: $path" ;; + esac + done + if [ "${#py[@]}" -eq 0 ] && [ "${#sh[@]}" -eq 0 ]; then + error "no recognized files among: $*" + return 2 + fi + if [ "${#py[@]}" -gt 0 ]; then + "$python_fn" "${py[@]}" + fi + if [ "${#sh[@]}" -gt 0 ]; then + "$shell_fn" "${sh[@]}" + fi +} + +check() { + # lint and verify formatting without modifying anything: shellcheck + + # shfmt diff on the shell tooling, ruff + format diff + mypy on the + # Python tooling + _route _check_python _check_shell "$@" +} + +format() { + # apply the formatters (WRITES files): shfmt on the shell tooling, + # ruff format on the Python tooling + _route _format_python _format_shell "$@" +} + +new() { + # scaffold a new run script with the standard anatomy and mark it + # executable; the name becomes the subcommand (./run ) + local name="${1:-}" + case "$name" in + "" | *[!a-z0-9-]* | -* | *-) + error 'usage: ./run scripts new (lowercase letters, digits, and inner dashes)' + return 2 + ;; + help | commands) + # the run dispatcher answers these itself, so a script by either + # name would be unreachable + error "$name is reserved by the run dispatcher" + return 2 + ;; + esac + local target="$SCRIPT_DIR/$name.sh" + if [ -e "$target" ]; then + error "$target already exists" + return 1 + fi + local fn + fn="$(printf '%s' "$name" | LC_ALL=C tr '-' '_')" + cat "$SCRIPT_DIR"/RUN_SCRIPT_TEMPLATE >"$target" + chmod +x "$target" + info "Created $target." + info "Replace every TODO (suggested function name: $fn); ./run picks the command up automatically." +} + +setup() { + # check that the script-development tools are installed and print what + # each tool resolved to + require shellcheck shfmt uvx || return 1 + print_resolved shellcheck shfmt uvx + info 'All script-development tools found.' +} + +help() { + # show this help + banner_help "$0" + command_help "$0" +} + +# First arg selects a command when it names one; anything else is treated as +# a file list for `check`, so `./run scripts file1.sh file2.py` just works. +if declare -F "${1:-check}" >/dev/null; then + dispatch check "$@" +else + check "$@" +fi diff --git a/build_scripts/run_scripts/serve.sh b/build_scripts/run_scripts/serve.sh index 28dad7f7..4a4af4ac 100755 --- a/build_scripts/run_scripts/serve.sh +++ b/build_scripts/run_scripts/serve.sh @@ -10,6 +10,8 @@ # Run `./run watch` in another terminal for live rebuilds as you edit, then # refresh the browser after each rebuild. # +# Tool overrides (env vars): RUN_PYTHON. +# # Usage: # ./run serve # ./run serve help diff --git a/build_scripts/run_scripts/setup.sh b/build_scripts/run_scripts/setup.sh index e1b808bc..5c223b1a 100755 --- a/build_scripts/run_scripts/setup.sh +++ b/build_scripts/run_scripts/setup.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash ################################################################################ -# Check that all required tooling is installed +# Check that the tools needed to contribute docs are installed # -# Verifies every dependency the run commands need: Python >= 3.11, uvx, -# and shellcheck, the te CLI, plus docfx (either the repo-local dotnet tool -# or a global install). Reports everything that is missing in one pass; -# installs nothing. +# Verifies every dependency a doc contributor needs: Python >= 3.11, the +# te CLI, plus docfx (either the repo-local dotnet tool or a global +# install). Reports everything that is missing in one pass; installs +# nothing. The script-development tools have their own check: +# `./run scripts setup`. # # Usage: # ./run setup @@ -51,18 +52,14 @@ check() { local failed=0 # dotnet is not in this list: _docfx_ok requires it only when no global # docfx exists, so it is reported exactly once and only when relevant. - require python shellcheck te uvx || failed=1 + require python te || failed=1 if ! _docfx_ok; then error "docfx not found: neither 'dotnet docfx --version' nor 'docfx --version' succeeded." info 'See build_scripts/run_scripts/README.md for installation instructions.' failed=1 fi if [ "$failed" -eq 0 ]; then - local tool var - for tool in python shellcheck te uvx; do - var="$(run_var_name "$tool")" - printf ' %-11s %s\n' "$tool" "$(command -v "${!var}")" - done + print_resolved python te printf ' %-11s %s\n' docfx "$(_docfx_desc)" info 'All dependencies found.' fi diff --git a/build_scripts/run_scripts/watch.sh b/build_scripts/run_scripts/watch.sh index ae438cc7..78c59486 100755 --- a/build_scripts/run_scripts/watch.sh +++ b/build_scripts/run_scripts/watch.sh @@ -11,6 +11,8 @@ # Run alongside `./run serve` in another terminal, then refresh the browser # after each rebuild. # +# Tool overrides (env vars): RUN_PYTHON. +# # Usage: # ./run watch # ./run watch help diff --git a/build_scripts/te_script_runner.py b/build_scripts/te_script_runner.py index 5c8d1c35..74acfb88 100755 --- a/build_scripts/te_script_runner.py +++ b/build_scripts/te_script_runner.py @@ -84,11 +84,7 @@ def _parse_json(stdout: str) -> dict[str, Any] | None: def _output_lines(data: dict[str, Any]) -> list[str]: """The text of every output-level message, in order. Pure.""" messages = data.get("messages") or [] - return [ - str(m.get("text", "")) - for m in messages - if isinstance(m, dict) and m.get("level") == "output" - ] + return [str(m.get("text", "")) for m in messages if isinstance(m, dict) and m.get("level") == "output"] def _diagnostic_lines(data: dict[str, Any]) -> list[str]: @@ -139,11 +135,7 @@ def summarize(stdout: str, exit_code: int) -> Result: """ data = _parse_json(stdout) if data is None: - diagnostics = ( - [f"[te] failed with exit {exit_code} and no json output"] - if exit_code != 0 - else [] - ) + diagnostics = [f"[te] failed with exit {exit_code} and no json output"] if exit_code != 0 else [] return Result(exit_code, exit_code == 0, "", diagnostics) if data.get("dryRun"): return _summarize_dry_run(data, exit_code) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..941fc2b6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +# Configuration for the repo's Python tooling (build_scripts/*.py). +# +# The ruff `include` and mypy `files` lists below are the same list and must +# stay in sync: the build scripts that meet the bar enforced by +# `./run scripts check`. Add scripts to both lists as they qualify. + +[tool.ruff] +line-length = 120 +target-version = "py311" +lint.select = ["F", "B", "SIM", "I", "UP"] +include = [ + "build_scripts/check_links.py", + "build_scripts/csharp_doctest.py", + "build_scripts/te_script_runner.py", +] + +[tool.mypy] +files = [ + "build_scripts/check_links.py", + "build_scripts/csharp_doctest.py", + "build_scripts/te_script_runner.py", +] +strict = true