diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index de13acd0..4cc07131 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -253,17 +253,29 @@ markdown links, which is why there is no submodule to hold the pin. ### How the pin moves -The workflow runs weekly, resolves the latest release tag, and opens a bump PR -for every release the pin does not already contain. Two shapes come out of it: +The workflow runs weekly and resolves the latest release tag. What comes out of +it depends on what the release carries: -- **Pages changed.** The usual case: review the diff. -- **Nothing under `docs/` changed.** The pages are byte-identical and the only - diff is `source_ref` on each of them, but the PR still opens, because that is - what moves the pin off a commit and onto a release tag. Skipping these would +- **Pages changed.** A PR to review the diff. The usual case. +- **Nothing under `docs/` changed, and the pin is a commit.** A PR whose only + diff is the pin and the `source_ref` each page records. It opens because that + is what moves the pin off a commit and onto a release tag; skipping it would strand a temporary commit pin for good. - -The PR body says which of the two it is. The recipe version readers type is a -separate axis, covered by the `static-site` entry under `watched`. +- **Nothing under `docs/` changed, and the pin is a tag.** Nothing. A PR would + carry an empty page diff for someone to review and merge. The pin then lags + the latest release and stays accurate, since it records the ref this copy came + from and the copy still matches it, and the release is not missed, because the + recipe that deploys this canister releases in lockstep and is tracked under + `watched`. + +The PR body says which of the first two it is. + +To sync a ref rather than a release, dispatch the workflow with `ref`: a sha, +tag, or branch. That is for a docs fix that has shipped upstream but is not in a +release, and it leaves the pin on a commit until the next release moves it onto +a tag. The release checks do not apply to a dispatched ref, so it can also move +the pin backwards, which a rollback wants and a mistyped sha does not: the run +says so and the PR body repeats it. To sync by hand, or to trial a ref before pinning it: diff --git a/.github/workflows/sync-static-site.yml b/.github/workflows/sync-static-site.yml index 419e8e43..eee1986a 100644 --- a/.github/workflows/sync-static-site.yml +++ b/.github/workflows/sync-static-site.yml @@ -4,6 +4,14 @@ on: schedule: - cron: '0 9 * * 3' # Weekly on Wednesday workflow_dispatch: + inputs: + ref: + description: >- + Ref to sync, for a docs fix that has shipped upstream but not been + released. Any commit-ish the clone can resolve. Leave empty to sync + the latest release, which is what the weekly run does. + required: false + type: string jobs: sync: @@ -31,25 +39,70 @@ jobs: id: check env: GH_TOKEN: ${{ steps.app-token.outputs.token }} + # A dispatch input reaches the shell as data, never as script. + INPUT_REF: ${{ inputs.ref }} run: | PIN=$(node -p "require('./.sources/upstream.json').synced.find(e => e.repo === 'dfinity/certified-assets').pinned") echo "pin=$PIN" >> $GITHUB_OUTPUT - # Stable releases only, the same pattern the upstream.json watcher uses. - # `^v[0-9]` would accept v0.4.0-rc.1 and publish docs for a prerelease. - TAG=$(git -C /tmp/certified-assets tag --sort=-version:refname \ - | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1) - echo "Pinned: $PIN. Latest release: $TAG." - echo "tag=$TAG" >> $GITHUB_OUTPUT + if [ -n "$INPUT_REF" ]; then + # A ref given by hand is the maintainer's call, so the release + # checks below do not apply: the reason to pass one is a docs fix + # that is not in a release yet. + # + # Tried as given and then under `origin/`, because a clone creates a + # local branch only for the default branch and leaves every other + # one reachable as `origin/` alone. A sha and a tag resolve + # on the first attempt. + REF_COMMIT=$(git -C /tmp/certified-assets rev-parse --verify --quiet "${INPUT_REF}^{commit}" \ + || git -C /tmp/certified-assets rev-parse --verify --quiet "origin/${INPUT_REF}^{commit}") || { + echo "::error::Cannot resolve ref '$INPUT_REF' in dfinity/certified-assets." + exit 1 + } + TAG=$(git -C /tmp/certified-assets rev-parse --short "$REF_COMMIT") + echo "Manual ref: $INPUT_REF resolved to $TAG. Pinned: $PIN." - # The pin is allowed to sit ahead of the latest release while a docs - # fix has shipped but a release has not (the state this sync started - # in). Syncing the tag then would publish older prose. - if git -C /tmp/certified-assets merge-base --is-ancestor "$TAG" "$PIN"; then - echo "Pin already contains $TAG. Nothing to sync." - echo "needed=false" >> $GITHUB_OUTPUT - exit 0 + # Peel the pin to a commit before comparing. The tags here are + # annotated, so an unpeeled tag name resolves to the tag object and + # would never equal a commit: dispatching the pinned tag would then + # read as a change and rewrite the pin from that tag to its own + # commit sha, moving a tag pin onto a commit for no reason. + PIN_COMMIT=$(git -C /tmp/certified-assets rev-parse --verify --quiet "${PIN}^{commit}") || { + echo "::error::Pin '$PIN' does not resolve in dfinity/certified-assets." + exit 1 + } + if [ "$REF_COMMIT" = "$PIN_COMMIT" ]; then + echo "That is the pin already. Nothing to sync." + echo "needed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # A ref the pin already contains moves the pin backwards. That is + # what a rollback is, so it runs, but it is also what a mistyped sha + # looks like, so it is not silent: the run warns and the PR body + # says so, which is the difference between the two. + if git -C /tmp/certified-assets merge-base --is-ancestor \ + "$REF_COMMIT" "$PIN_COMMIT"; then + echo "::warning::$INPUT_REF is behind the pin $PIN. This will move the pin backwards." + echo "backwards=true" >> $GITHUB_OUTPUT + fi + else + # Stable releases only, the same pattern the upstream.json watcher uses. + # `^v[0-9]` would accept v0.4.0-rc.1 and publish docs for a prerelease. + TAG=$(git -C /tmp/certified-assets tag --sort=-version:refname \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1) + echo "Pinned: $PIN. Latest release: $TAG." + + # The pin is allowed to sit ahead of the latest release while a docs + # fix has shipped but a release has not (the state this sync started + # in). Syncing the tag then would publish older prose. + if git -C /tmp/certified-assets merge-base --is-ancestor "$TAG" "$PIN"; then + echo "Pin already contains $TAG. Nothing to sync." + echo "needed=false" >> $GITHUB_OUTPUT + exit 0 + fi fi + echo "tag=$TAG" >> $GITHUB_OUTPUT # Skip only when a PR is actually open. A branch on its own proves # nothing: if a previous run pushed and then failed at `gh pr create`, @@ -68,18 +121,34 @@ jobs: git push origin --delete "$BRANCH" fi - # A release that ships canister changes without touching docs/ leaves - # the synced pages byte-identical, so there is no content to review. - # The pin still has to move: it is allowed to sit on a commit only - # while no release carries the pages, and skipping here would strand - # it on that commit for good. So the PR is opened either way, and the - # body says which of the two it is. CHANGED=$(git -C /tmp/certified-assets diff --name-only "${PIN}..${TAG}" -- docs/) - echo "needed=true" >> $GITHUB_OUTPUT if [ -z "$CHANGED" ]; then - echo "No docs/ changes between $PIN and $TAG: advancing the pin only." - echo "pin_only=true" >> $GITHUB_OUTPUT + # A release that ships canister changes without touching docs/ leaves + # the synced pages byte-identical, so there is nothing to review. Two + # cases, and only one of them is worth a pull request. + if [ -n "$INPUT_REF" ] || ! git -C /tmp/certified-assets show-ref \ + --verify --quiet "refs/tags/${PIN}"; then + # The pin is a commit (or a ref was dispatched by hand). Moving it + # onto the tag is the point: a commit pin is allowed only while no + # release carries the pages, and skipping here would strand it + # there for good. + echo "No docs/ changes between $PIN and $TAG: advancing the pin only." + echo "needed=true" >> $GITHUB_OUTPUT + echo "pin_only=true" >> $GITHUB_OUTPUT + else + # The pin is already a tag, so a pull request would carry an empty + # page diff and a bumped ref, for a reader to review and merge with + # nothing in it. The pin lags the release and stays accurate: it + # says which ref this copy came from, and the copy still matches it. + # The release itself is not lost, since the recipe that deploys this + # canister releases in lockstep and is tracked under `watched`. + echo "No docs/ changes between $PIN and $TAG, and the pin is a tag." + echo "Nothing to sync." + echo "needed=false" >> $GITHUB_OUTPUT + exit 0 + fi else + echo "needed=true" >> $GITHUB_OUTPUT echo "Changed upstream pages:" echo "$CHANGED" echo "pin_only=false" >> $GITHUB_OUTPUT @@ -158,11 +227,24 @@ jobs: echo "" echo "Automated sync of the certified-assets user docs." echo "" - echo "**Release:** \`$TAG\` (pinned from \`$PIN\`)" + if [ -n "$INPUT_REF" ]; then + echo "**Ref:** \`$TAG\` (pinned from \`$PIN\`), dispatched by hand as \`$INPUT_REF\`." + echo "A ref is synced by hand when a docs fix has shipped upstream but not" + echo "been released; the next release moves the pin back onto a tag." + if [ "$BACKWARDS" = "true" ]; then + echo "" + echo "> [!WARNING]" + echo "> This ref is **behind** the pin, so the pin moves backwards and any" + echo "> page below is reverted to the older text. Intended for a rollback." + echo "> If you meant to move forward, close this and dispatch the right ref." + fi + else + echo "**Release:** \`$TAG\` (pinned from \`$PIN\`)" + fi echo "" if [ "$PIN_ONLY" = "true" ]; then - echo "No page changed in this range. The pin moves off a commit and onto" - echo "the release tag, so the only diff is \`source_ref\` on each page." + echo "No page changed in this range, so the only diff is the pin and the" + echo "\`source_ref\` each page records." else echo "**Changed upstream files:**" while IFS= read -r f; do @@ -192,3 +274,5 @@ jobs: PIN: ${{ steps.check.outputs.pin }} CHANGED: ${{ steps.check.outputs.changed_files }} PIN_ONLY: ${{ steps.check.outputs.pin_only }} + INPUT_REF: ${{ inputs.ref }} + BACKWARDS: ${{ steps.check.outputs.backwards }} diff --git a/.sources/upstream.json b/.sources/upstream.json index f8b90955..c5390506 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -65,6 +65,11 @@ "range contains before pinning past a tag, and prefer a tag whenever one", "carries the pages. The sync workflow moves the pin onto a tag at the next", "release even when no page changed, so a commit pin is never permanent.", + "Once it is a tag, a release that changes no page is skipped instead: the", + "pin then lags the latest release and stays accurate, since it records the", + "ref this copy came from and the copy still matches it. The release is not", + "missed either way, because the recipe that deploys this canister releases", + "in lockstep and is tracked under `watched`.", "History: the first sync used d9cb7df, because certified-assets#124 landed", "the frontmatter contract this sync requires after the v0.3.3 tag." ] diff --git a/scripts/sync-static-site.mjs b/scripts/sync-static-site.mjs index c967f4c3..3d4e9aab 100644 --- a/scripts/sync-static-site.mjs +++ b/scripts/sync-static-site.mjs @@ -4,9 +4,12 @@ // // That repo is the single source of truth for how the canister behaves. Its // docs/ pages carry Starlight frontmatter and page order upstream, so this -// script copies rather than authors: no headings are added, and nothing is -// rewritten beyond the mechanical transformations listed below, each of which -// exists because this site enforces a rule the source repo does not. +// script copies rather than authors: the prose that lands here is the prose +// upstream wrote, word for word. Nothing is reworded, and no command is +// translated. Where a page breaks a rule this site enforces, the sync says so +// and the fix belongs upstream, in the source of truth, rather than in a +// pipeline that would hide the divergence and have to be re-applied on every +// sync. // // Unlike the motoko and internet-identity syncs, nothing here is a submodule. // The build resolves no file from certified-assets, only markdown links, so the @@ -14,15 +17,16 @@ // that ref. Read content through raw.githubusercontent.com, never the contents // API, which returns base64 that gets truncated for larger files. // -// Transformations: -// - Rewrite absolute links to this docs site into relative .md links (LINK_MAP) -// - Record provenance as source_repo / source_ref in the frontmatter -// - Normalize what the brand rules lock: em dash (U+2014), en dash (U+2013) -// as a prose separator, and "tamperproof" as one word. Prose only, never -// inside a fence. These are near no-ops today (upstream dropped its em -// dashes in certified-assets#125) and exist so a future page cannot -// reintroduce them silently. -// - Append a do-not-edit marker +// What is touched, none of it prose: +// - Link targets: an absolute link to this docs site becomes the relative .md +// link that reaches the same page (LINK_MAP). Upstream writes these as +// absolute URLs because its docs/ is also read on GitHub. Keeping them +// verbatim is not an option: scripts/validate.js rejects both +// docs.internetcomputer.org and the retired internetcomputer.org/docs on +// every page in this repo, the synced tree included. +// - Frontmatter: source_repo / source_ref appended, recording what this copy +// came from. +// - A do-not-edit marker appended. // // Validation (exits non-zero on failure, so a bad sync never lands quietly): // - Frontmatter must carry title, description and sidebar.order. This is the @@ -32,7 +36,14 @@ // upstream linked a page LINK_MAP does not know about yet. // - Every relative .md link must resolve on disk, including the ones pointing // out of the synced tree. -// - No banned character may survive normalization. +// - No em dash, en-dash separator, or `dfx` command. These three are what +// scripts/validate.js and AGENTS.md ban outright, so publishing one would +// fail this repo's own CI on a page nobody here can edit. The sync stops +// instead, naming the page, and upstream fixes it. +// +// Softer house style (one-word "tamperproof", for instance) is reported at the +// end of a run and not enforced: it is worth an upstream issue, not a blocked +// sync. // // Usage: node scripts/sync-static-site.mjs [--ref ] // or: npm run sync:static-site @@ -74,25 +85,13 @@ function canonicalize(url) { .replace(/\/(#|$)/, '$1'); } -// Brand rules of record: https://jgwns-tqaaa-aaaao-ba5ua-cai.icp0.io/rules.json -// (banned_characters, one_word_spellings). Prose only: a fence can hold a -// hyphenated identifier or a range that is none of our business. -const PROSE_RULES = [ - { re: /\s*—\s*/g, to: ': ', what: 'em dash' }, - { re: /\s–\s/g, to: ', ', what: 'en dash as separator' }, - { re: /tamper[- ]proof/gi, to: 'tamperproof', what: 'hyphenated tamperproof' }, -]; - -// `dfx` is banned in this repo (AGENTS.md "Never"), and a command reaches a -// reader from inside a fence, where the prose rules deliberately do not go. Only -// this exact shape is rewritten, checked against `icp canister call` in icp-cli -// v1.5.0: accepts a principal, `-e` selects the network, and the -// argument has to be written out, because `icp canister call` with no argument -// opens an interactive prompt instead of sending an empty one. dfx infers `()`; -// icp does not, so the rewrite adds it. Any other `dfx` occurrence fails the -// sync rather than being guessed at. -const DFX_CALL = /^(\s*)dfx canister call (\S+) (\S+) --network ic[ \t]*$/gm; -const rewriteDfx = (text) => text.replace(DFX_CALL, "$1icp canister call $2 $3 '()' -e ic"); +// House style this sync reports but does not apply. The brand rules of record +// (https://jgwns-tqaaa-aaaao-ba5ua-cai.icp0.io/rules.json, one_word_spellings) +// lock these spellings for prose written here; a synced page is prose written +// upstream, so a hit is an upstream issue to raise, not something to patch on +// the way in. Prose only: a fence can hold a hyphenated identifier that is none +// of our business. +const HOUSE_STYLE = [{ re: /tamper[- ]proof/i, what: '"tamperproof" split in two' }]; // Prose means prose: not a fenced block, not the frontmatter, and within a line, // not an inline code span and not a link destination. A rule that reached into @@ -120,19 +119,13 @@ function mapProse(text, fn) { return text.slice(0, cut) + body; } -function normalizeProse(text) { - const applied = []; - const out = mapProse(text, (line) => { - let l = line; - for (const { re, to, what } of PROSE_RULES) { - if (re.test(l)) { - applied.push(what); - l = l.replace(re, to); - } - } - return l; +function scanProse(text) { + const found = []; + mapProse(text, (line) => { + for (const { re, what } of HOUSE_STYLE) if (re.test(line)) found.push(what); + return line; }); - return { out, applied: [...new Set(applied)] }; + return [...new Set(found)]; } function rewriteSiteLinks(text) { @@ -312,9 +305,9 @@ async function main() { const pages = await sourcePages(ref); const written = []; - const normalized = []; + const deviations = []; - // Transform every page before writing any of them. A page that violates the + // Prepare every page before writing any of them. A page that violates the // contract then leaves the tree exactly as it was, rather than a mix of two // refs that still builds and still validates. const prepared = new Map(); @@ -332,12 +325,10 @@ async function main() { ); } - const deDfxed = rewriteDfx(linked); - const { out: clean, applied } = normalizeProse(deDfxed); - if (deDfxed !== linked) applied.push('dfx command rewritten to icp'); - if (applied.length) normalized.push(`${file}: ${applied.join(', ')}`); + const housePeeves = scanProse(linked); + if (housePeeves.length) deviations.push(`${file}: ${housePeeves.join(', ')}`); - prepared.set(file, stampProvenance(clean, file, ref) + marker(file, ref)); + prepared.set(file, stampProvenance(linked, file, ref) + marker(file, ref)); } // Check everything while it is still only in memory, so a failure leaves the @@ -348,13 +339,16 @@ async function main() { problems.push(`${file}: broken link ${href}`); } if (/—|\s–\s/.test(content)) { - problems.push(`${file}: a banned character survived normalization`); + problems.push( + `${file}: holds an em dash or an en-dash separator, which ` + + `scripts/validate.js rejects on every page in this repo. Fix it ` + + `upstream; this sync does not rewrite prose.` + ); } if (/\bdfx\b/.test(content)) { problems.push( - `${file}: publishes a \`dfx\` command this script does not know how to ` + - `rewrite. dfx is banned here (AGENTS.md "Never"), so fix it upstream, ` + - `or extend DFX_CALL if the shape is safe to translate.` + `${file}: holds a \`dfx\` command, which is banned here (AGENTS.md ` + + `"Never"). Fix it upstream; this sync does not translate commands.` ); } } @@ -400,9 +394,9 @@ async function main() { console.log(`\nWrote ${written.length} page(s) to ${TARGET_DIR}/:`); for (const file of written) console.log(` ${file}`); if (stale.length) console.log(`Removed ${stale.length} page(s) gone upstream: ${stale.join(', ')}`); - if (normalized.length) { - console.log('\nNormalized (report upstream so these become no-ops):'); - for (const line of normalized) console.log(` ${line}`); + if (deviations.length) { + console.log('\nHouse style upstream does not follow (raise it there; nothing was changed here):'); + for (const line of deviations) console.log(` ${line}`); } }