sources: the site's sources pinned and checked in Gate, and a daily sync (R5, R6, R7)

.github/workflows/pages.yml

@@ -1,10 +1,13 @@
 name: Pages
 
-# Builds the site on every push and pull request with task gate, as a developer does: fails on a
-# broken link or a page without the experimental notice, and publishes main to GitHub Pages. The
-# reference pages are committed. The gate holds what they take from spin-terraform-aws and brand,
-# both public, to those repositories at the commits src/data/reference.json records; what they
-# take from spin is held by `task reference:check`, which this job runs once it can check spin out.
+# Builds the site on every push and pull request as a developer checks it, and publishes main to
+# GitHub Pages: task check, the reference against spin, spin-terraform-aws and brand at the
+# commits versions.yaml pins, and the gate - the build, every link, a page without the
+# experimental notice. spin is private: the sources App's token reads it, in the one step that
+# fetches, and the steps after it build and run spin's CLI with no token at all. A pull request
+# from a fork has no secret, so it runs task check:public, which holds the site to the public
+# sources and keeps what it says of spin as reference.json has it. Dependabot's pull requests
+# have the App as a Dependabot secret, and run the whole check.
 
 on:
   push:
@@ -24,6 +27,9 @@ jobs:
     # The check main's ruleset requires, named Gate in every spin-stack repository: the build.
     name: Gate
     runs-on: ubuntu-latest
+    env:
+      # Whether this run has the sources App: a secret cannot be asked in an if, its presence can.
+      SOURCES_APP: ${{ secrets.SOURCES_APP_KEY != '' }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -38,9 +44,33 @@ jobs:
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-      # What a developer runs, with the Task go.mod pins: task gate, the install through Socket
-      # Firewall among it.
-      - run: go tool task gate
+          # The modules and the build cache, spin's among them: spin is pinned in versions.yaml,
+          # so a bump of it is a new key and every other run builds its CLI from the cache.
+          cache-dependency-path: |
+            go.sum
+            versions.yaml
+      # Read-only, on spin alone: the public sources need no token.
+      - id: sources-token
+        if: env.SOURCES_APP == 'true'
+        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
+        with:
+          app-id: ${{ secrets.SOURCES_APP_ID }}
+          private-key: ${{ secrets.SOURCES_APP_KEY }}
+          owner: spin-stack
+          repositories: spin
+          permission-contents: read
+      - name: Sources
+        if: env.SOURCES_APP == 'true'
+        env:
+          # tools/sources hands it to its own git commands as GIT_CONFIG_*, and writes it nowhere.
+          SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }}
+        run: go tool task sources
+      # What a developer runs, with the Task go.mod pins: task check, the install through Socket
+      # Firewall among it. The sources are fetched, so this asks nothing of spin.
+      - if: env.SOURCES_APP == 'true'
+        run: go tool task check
+      - if: env.SOURCES_APP != 'true'
+        run: go tool task check:public
       - if: github.ref == 'refs/heads/main'
         uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
         with:

.github/workflows/sync.yml

@@ -0,0 +1,116 @@
+name: Sync
+
+# The site against what it is generated from, so a sync no longer waits for somebody to remember
+# it. Every day, and when a source's release dispatches `sources-released`: the pins in
+# versions.yaml move to the newest releases (spin's only to one that holds the commit pinned), the
+# reference is regenerated from them, and the result is force-pushed to sync/sources as the sync
+# App, whose push starts Gate as a person's would, with its one pull request opened or updated.
+# A person reviews and merges it: it publishes spin's help, API and board on a public site, and
+# behaviour that changed under the hand-written pages is theirs to write.
+#
+# spin is private and this repository public, so the pull request says which pins moved and which
+# files changed. What spin's commits say - grouped by the pages each path of spin feeds - and the
+# release chain go in one issue in spin, "Site vs spin main", written with the sources App. Every
+# Monday it also says what spin main has past the pin: advice, which fails nothing.
+
+on:
+  schedule:
+    - cron: '23 5 * * *'
+    - cron: '41 5 * * 1'
+  repository_dispatch:
+    types: [sources-released]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: sync
+  cancel-in-progress: false
+
+jobs:
+  release:
+    if: github.event.schedule != '41 5 * * 1'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # The push is the sync App's, with its own token; the checkout's would start no Gate.
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: |
+            go.sum
+            versions.yaml
+      # Read-only: spin's code and releases, and the release chain's repositories, with issues
+      # on spin for the one issue this writes there.
+      - id: sources-token
+        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
+        with:
+          app-id: ${{ secrets.SOURCES_APP_ID }}
+          private-key: ${{ secrets.SOURCES_APP_KEY }}
+          owner: spin-stack
+          repositories: spin,spin-terraform-aws,spin-machine,spinbox.dev-deploy
+          permission-contents: read
+          permission-actions: read
+          permission-issues: write
+      - name: Bump the pins
+        env:
+          SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }}
+        run: go tool task sources:bump
+      # With no token: spin's CLI is built and run here. A page the sources no longer agree with
+      # still has its regenerated files pushed, and the pull request says so.
+      - id: reference
+        continue-on-error: true
+        run: go tool task reference
+      - name: Write spin's issue
+        env:
+          SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }}
+          BOT: ${{ steps.sources-token.outputs.app-slug }}[bot]
+          MODE: release
+        run: go tool task sync:issue
+      - id: sync-token
+        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
+        with:
+          app-id: ${{ secrets.SYNC_APP_ID }}
+          private-key: ${{ secrets.SYNC_APP_KEY }}
+          owner: spin-stack
+          repositories: website
+          permission-contents: write
+          permission-pull-requests: write
+      - name: Open the pull request
+        env:
+          SYNC_TOKEN: ${{ steps.sync-token.outputs.token }}
+          BOT: ${{ steps.sync-token.outputs.app-slug }}[bot]
+          REFERENCE: ${{ steps.reference.outcome }}
+        run: go tool task sync:pr
+
+  main:
+    if: github.event.schedule == '41 5 * * 1' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    # Advice: spin main changes an interface nearly every day, and nobody is to be stopped by it.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+      - id: sources-token
+        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
+        with:
+          app-id: ${{ secrets.SOURCES_APP_ID }}
+          private-key: ${{ secrets.SOURCES_APP_KEY }}
+          owner: spin-stack
+          repositories: spin,spin-terraform-aws,spin-machine,spinbox.dev-deploy
+          permission-contents: read
+          permission-actions: read
+          permission-issues: write
+      - name: Write spin's issue
+        env:
+          SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }}
+          BOT: ${{ steps.sources-token.outputs.app-slug }}[bot]
+          MODE: main
+        run: go tool task sync:issue

README.md

@@ -10,21 +10,40 @@ check that fails the build, or it does not exist. Everything below is one of tho
 ```
 go tool task install      # dependencies, through Socket Firewall (go tool sfw)
 go tool task dev          # http://localhost:4321, live reload
-go tool task gate         # what CI runs: the gates, the install, the tests, the public reference, the build, every link
-go tool task check        # the gate, and the reference against the sources
+go tool task sources      # spin, spin-terraform-aws and brand into .sources/, at the commits versions.yaml pins
+go tool task check        # what CI runs: the reference against the sources, and the gate
+go tool task gate         # the gates, the install, the tests, the public reference, the build, every link
 go tool task exposure     # what this public repository shows: every ref, its PRs and issues
 go tool task --list       # everything else
 ```
 
 Task is a tool in `go.mod`, so `go tool task` is the version CI runs; the tasks below are named
-without it. `task gate` is what CI runs, its one step: it fails first on an install that skips
-the firewall (`go tool sfw check`) or a workflow step that is not one task (`go tool taskgate`),
-and `task check` is that and more: CI cannot read spin while it is private, so the reference is
-checked against checkouts of spin, spin-terraform-aws and brand beside this one (`SPIN=`,
-`TERRAFORM=`, `BRAND=` to point elsewhere). The half that needs no spin is in the gate:
-`task reference:public` fetches spin-terraform-aws and brand into `.sources/`, at the commits
-`src/data/reference.json` records, and runs `refgen -only terraform,brand -check`, which builds
-no CLI and keeps what the site says of spin as `reference.json` has it.
+without it. `task check` is what CI runs: the reference against its sources, then `task gate`,
+which fails first on an install that skips the firewall (`go tool sfw check`) or a workflow step
+that is not one task (`go tool taskgate`).
+
+- **The sources are pinned** in `versions.yaml`, go-tools' versions format, held by
+  `tools/sources/repo_test.go` as every spin-stack repository is: spin, spin-terraform-aws and
+  brand, each at one commit. `task sources` (`tools/sources`) fetches exactly those commits, with
+  their history and tags, into `.sources/`, which is where `SPIN`, `TERRAFORM` and `BRAND` point
+  unless given (`SPIN=../spin` for a checkout of your own). refgen fails when
+  `src/data/reference.json` records another commit than the pins, so the two never disagree.
+  spin is private: `SOURCES_TOKEN`, or git's own credentials, reads it. In CI the read-only
+  sources App's token reaches git only as `GIT_CONFIG_*` in the one step that fetches.
+- **Without spin** - a fork's pull request has no secret - CI runs `task check:public`: the gate,
+  whose `task reference:public` fetches spin-terraform-aws and brand at their pins and runs
+  `refgen -only terraform,brand -check`, which builds no CLI and keeps what the site says of spin
+  as `reference.json` has it.
+- **The sync** is `.github/workflows/sync.yml`, every day: `task sources:bump` moves the pins to
+  the newest releases (spin only to one that holds the commit pinned, brand to its main), `task
+  reference` regenerates, and the sync App force-pushes branch `sync/sources` and opens or
+  updates its one pull request, which a person reviews; nothing merges it. spin is private and
+  this repository public, so the pull request lists the pins that moved and the files that
+  changed, and what spin's commits say - grouped by the pages each path of spin feeds - goes in
+  one issue in spin, "Site vs spin main", with the release chain: spin-machine as spin pins it
+  against its newest release, the last runs of `machine.yml`, `propose.yml` and
+  `fresh-install.yml`, and what spinbox.dev runs. Every Monday the same issue says what spin main
+  has past the pin. `tools/sync` writes all of it.
 
 - **The repository is public**, and so is every pull request the moment it is opened.
   `task exposure` (`go tool exposure`) reads every ref's commits, pull request heads included,
@@ -37,8 +56,7 @@ no CLI and keeps what the site says of spin as `reference.json` has it.
   topic's groups at a time (`starlight-sidebar-topics`). Images and diagrams zoom in place
   (`starlight-image-zoom`), and every page title offers its Markdown twin, copied or opened.
 - **Reference pages** under `src/content/docs/reference/` are generated by `task reference` from
-  checkouts of spin and spin-terraform-aws beside this one (`SPIN=`, `TERRAFORM=` to point
-  elsewhere). Each names its source in its frontmatter (`source: repository:path`), and its footer
+  spin and spin-terraform-aws at their pins (`SPIN=`, `TERRAFORM=` to point elsewhere). Each names its source in its frontmatter (`source: repository:path`), and its footer
   links that source at the documented commit instead of an edit link. Edit the source, not the
   page. Every page's footer says which spin release an installation runs, and the tree the
   reference is generated from.
@@ -99,7 +117,7 @@ no CLI and keeps what the site says of spin as `reference.json` has it.
   expression or a directive `src/lib/pages.ts` does not translate.
 - **The look** is spin-stack/brand's: its palette, checked there for contrast and perceptual
   distance on real panels and through colour-vision deficiencies, and the organization's mark.
-  `task reference` copies them from a checkout beside this one (`BRAND=` to point elsewhere) into
+  `task reference` copies them from brand at its pin (`BRAND=` to point elsewhere) into
   `src/styles/brand-*.css`, `src/assets/` and `public/`, with the app's screenshots, and
   `task reference:check` fails on a copy that is not brand's, on screenshots of another release
   than installations run, and on one whose test spin no longer has; `src/styles/custom.css` says only how the site's parts use them.

Taskfile.yml

@@ -3,12 +3,13 @@ version: '3'
 # The site: Astro Starlight under src/, and two Go tools under tools/ - refgen writes the
 # reference pages from the sources they describe, linkcheck fails on a link the build did not
 # produce. SPIN, TERRAFORM and BRAND are checkouts of spin-stack/spin, spin-stack/spin-terraform-aws
-# and spin-stack/brand.
+# and spin-stack/brand: by default the ones task sources fetches into .sources/, at the commits
+# versions.yaml pins.
 
 vars:
-  SPIN: '{{.SPIN | default "../spin"}}'
-  TERRAFORM: '{{.TERRAFORM | default "../spin-terraform-aws"}}'
-  BRAND: '{{.BRAND | default "../brand"}}'
+  SPIN: '{{.SPIN | default ".sources/spin"}}'
+  TERRAFORM: '{{.TERRAFORM | default ".sources/spin-terraform-aws"}}'
+  BRAND: '{{.BRAND | default ".sources/brand"}}'
   BASE_PATH: '{{.BASE_PATH | default "/website"}}'
 
 tasks:
@@ -22,42 +23,45 @@ tasks:
     cmds:
       - pnpm dev
 
+  sources:
+    desc: Fetch spin, spin-terraform-aws and brand into .sources/, each at the commit versions.yaml pins. spin is private - SOURCES_TOKEN, or git's own credentials, reads it.
+    cmds:
+      - go run ./tools/sources spin spin-terraform-aws brand
+
+  sources:bump:
+    desc: Move the pins to the newest releases - spin only to one that holds the commit pinned, brand to its main - and fetch them. Then task reference.
+    cmds:
+      - go run ./tools/sources -bump spin spin-terraform-aws brand
+
+  sync:pr:
+    desc: Commit what sources:bump and reference changed to branch sync/sources as the sync App (SYNC_TOKEN, BOT its login), and open its one pull request or update it. sync.yml runs it.
+    cmds:
+      - go run ./tools/sync pr -bot '{{.BOT}}' -reference '{{.REFERENCE | default "success"}}'
+
+  sync:issue:
+    desc: Write spin's issue "Site vs spin main" as the sources App (SOURCES_TOKEN, BOT its login) - MODE release, the commits a bump takes in, or main, what spin main has past the pin - with the release chain. sync.yml runs it.
+    cmds:
+      - go run ./tools/sync issue -mode '{{.MODE}}' -bot '{{.BOT}}'
+
   reference:
     desc: Write the reference pages (CLI, Terraform, config file, API) and what the site copies from the spin, spin-terraform-aws and brand checkouts.
     cmds:
+      - task: sources
       - go run ./tools/refgen -spin {{.SPIN}} -terraform {{.TERRAFORM}} -brand {{.BRAND}}
 
   reference:check:
     desc: Fail if a reference page is not what the sources say.
     cmds:
+      - task: sources
       - go run ./tools/refgen -spin {{.SPIN}} -terraform {{.TERRAFORM}} -brand {{.BRAND}} -check
 
   reference:public:
-    desc: Fail if what the site has of spin-terraform-aws and brand is not what they say at the commits src/data/reference.json records. Needs no spin checkout, nor any credential.
-    # CI cannot check spin out, and this half needs nothing of it: what the site says of spin
-    # stays as reference.json has it, and is held by reference:check.
-    cmds:
-      # With its history and tags: a page pins the module at the newest release the commit holds.
-      - task: source
-        vars: { REPO: spin-terraform-aws, COMMIT: '{{(fromJson .REFERENCE).terraform.commit}}', TAGS: 'true' }
-      - task: source
-        vars: { REPO: brand, COMMIT: '{{(fromJson .REFERENCE).brand.commit}}' }
+    desc: Fail if what the site has of spin-terraform-aws and brand is not what they say at the commits versions.yaml pins. Needs no spin checkout, nor any credential.
+    # A fork's pull request has no token to read spin with, and this half needs nothing of it:
+    # what the site says of spin stays as reference.json has it, and is held by reference:check.
+    cmds:
+      - go run ./tools/sources spin-terraform-aws brand
       - go run ./tools/refgen -only terraform,brand -terraform .sources/spin-terraform-aws -brand .sources/brand -check
-    vars:
-      REFERENCE:
-        sh: cat src/data/reference.json
-
-  source:
-    internal: true
-    desc: Fetch one public spin-stack repository into .sources/ at one commit, and with TAGS, its history and its tags.
-    status:
-      - test "$(git -C .sources/{{.REPO}} rev-parse HEAD 2>/dev/null)" = "{{.COMMIT}}"
-      - test -z "{{.TAGS}}" || test "$(git -C .sources/{{.REPO}} rev-parse --is-shallow-repository)" = false
-    cmds:
-      - rm -rf .sources/{{.REPO}}
-      - git init -q .sources/{{.REPO}}
-      - git -C .sources/{{.REPO}} fetch -q {{if .TAGS}}--tags{{else}}--depth 1{{end}} https://github.com/spin-stack/{{.REPO}} {{.COMMIT}}
-      - git -C .sources/{{.REPO}} checkout -q --detach FETCH_HEAD
 
   build:
     desc: Build the site into dist/, with llms.txt, llms-full.txt and a .md twin of every page.
@@ -95,7 +99,12 @@ tasks:
       - go tool exposure -allow .github/exposure-allow.txt -gitleaks-config .gitleaks.toml -private ami,ci,shipset,skills,spin,spinbox.dev-deploy spin-stack/website
 
   check:
-    desc: Everything CI runs, and the reference against the sources.
+    desc: What CI runs with the token that reads spin - the reference against the sources at their pins, and the gate.
     cmds:
       - task: reference:check
       - task: gate
+
+  check:public:
+    desc: What CI runs with no token to read spin, a fork's pull request - the gate, whose reference:public needs none.
+    cmds:
+      - task: gate