diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index b7303a8..b20d949 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,10 +1,13 @@ name: Pages -# Builds the site on every push and pull request with task gate, as a developer does: fails on a -# broken link or a page without the experimental notice, and publishes main to GitHub Pages. The -# reference pages are committed. The gate holds what they take from spin-terraform-aws and brand, -# both public, to those repositories at the commits src/data/reference.json records; what they -# take from spin is held by `task reference:check`, which this job runs once it can check spin out. +# Builds the site on every push and pull request as a developer checks it, and publishes main to +# GitHub Pages: task check, the reference against spin, spin-terraform-aws and brand at the +# commits versions.yaml pins, and the gate - the build, every link, a page without the +# experimental notice. spin is private: the sources App's token reads it, in the one step that +# fetches, and the steps after it build and run spin's CLI with no token at all. A pull request +# from a fork has no secret, so it runs task check:public, which holds the site to the public +# sources and keeps what it says of spin as reference.json has it. Dependabot's pull requests +# have the App as a Dependabot secret, and run the whole check. on: push: @@ -24,6 +27,9 @@ jobs: # The check main's ruleset requires, named Gate in every spin-stack repository: the build. name: Gate runs-on: ubuntu-latest + env: + # Whether this run has the sources App: a secret cannot be asked in an if, its presence can. + SOURCES_APP: ${{ secrets.SOURCES_APP_KEY != '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -38,9 +44,33 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - # What a developer runs, with the Task go.mod pins: task gate, the install through Socket - # Firewall among it. - - run: go tool task gate + # The modules and the build cache, spin's among them: spin is pinned in versions.yaml, + # so a bump of it is a new key and every other run builds its CLI from the cache. + cache-dependency-path: | + go.sum + versions.yaml + # Read-only, on spin alone: the public sources need no token. + - id: sources-token + if: env.SOURCES_APP == 'true' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.SOURCES_APP_ID }} + private-key: ${{ secrets.SOURCES_APP_KEY }} + owner: spin-stack + repositories: spin + permission-contents: read + - name: Sources + if: env.SOURCES_APP == 'true' + env: + # tools/sources hands it to its own git commands as GIT_CONFIG_*, and writes it nowhere. + SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }} + run: go tool task sources + # What a developer runs, with the Task go.mod pins: task check, the install through Socket + # Firewall among it. The sources are fetched, so this asks nothing of spin. + - if: env.SOURCES_APP == 'true' + run: go tool task check + - if: env.SOURCES_APP != 'true' + run: go tool task check:public - if: github.ref == 'refs/heads/main' uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000..a2b2994 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,116 @@ +name: Sync + +# The site against what it is generated from, so a sync no longer waits for somebody to remember +# it. Every day, and when a source's release dispatches `sources-released`: the pins in +# versions.yaml move to the newest releases (spin's only to one that holds the commit pinned), the +# reference is regenerated from them, and the result is force-pushed to sync/sources as the sync +# App, whose push starts Gate as a person's would, with its one pull request opened or updated. +# A person reviews and merges it: it publishes spin's help, API and board on a public site, and +# behaviour that changed under the hand-written pages is theirs to write. +# +# spin is private and this repository public, so the pull request says which pins moved and which +# files changed. What spin's commits say - grouped by the pages each path of spin feeds - and the +# release chain go in one issue in spin, "Site vs spin main", written with the sources App. Every +# Monday it also says what spin main has past the pin: advice, which fails nothing. + +on: + schedule: + - cron: '23 5 * * *' + - cron: '41 5 * * 1' + repository_dispatch: + types: [sources-released] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: sync + cancel-in-progress: false + +jobs: + release: + if: github.event.schedule != '41 5 * * 1' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The push is the sync App's, with its own token; the checkout's would start no Gate. + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache-dependency-path: | + go.sum + versions.yaml + # Read-only: spin's code and releases, and the release chain's repositories, with issues + # on spin for the one issue this writes there. + - id: sources-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.SOURCES_APP_ID }} + private-key: ${{ secrets.SOURCES_APP_KEY }} + owner: spin-stack + repositories: spin,spin-terraform-aws,spin-machine,spinbox.dev-deploy + permission-contents: read + permission-actions: read + permission-issues: write + - name: Bump the pins + env: + SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }} + run: go tool task sources:bump + # With no token: spin's CLI is built and run here. A page the sources no longer agree with + # still has its regenerated files pushed, and the pull request says so. + - id: reference + continue-on-error: true + run: go tool task reference + - name: Write spin's issue + env: + SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }} + BOT: ${{ steps.sources-token.outputs.app-slug }}[bot] + MODE: release + run: go tool task sync:issue + - id: sync-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.SYNC_APP_ID }} + private-key: ${{ secrets.SYNC_APP_KEY }} + owner: spin-stack + repositories: website + permission-contents: write + permission-pull-requests: write + - name: Open the pull request + env: + SYNC_TOKEN: ${{ steps.sync-token.outputs.token }} + BOT: ${{ steps.sync-token.outputs.app-slug }}[bot] + REFERENCE: ${{ steps.reference.outcome }} + run: go tool task sync:pr + + main: + if: github.event.schedule == '41 5 * * 1' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + # Advice: spin main changes an interface nearly every day, and nobody is to be stopped by it. + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - id: sources-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.SOURCES_APP_ID }} + private-key: ${{ secrets.SOURCES_APP_KEY }} + owner: spin-stack + repositories: spin,spin-terraform-aws,spin-machine,spinbox.dev-deploy + permission-contents: read + permission-actions: read + permission-issues: write + - name: Write spin's issue + env: + SOURCES_TOKEN: ${{ steps.sources-token.outputs.token }} + BOT: ${{ steps.sources-token.outputs.app-slug }}[bot] + MODE: main + run: go tool task sync:issue diff --git a/README.md b/README.md index aac8ddf..7cee670 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,40 @@ check that fails the build, or it does not exist. Everything below is one of tho ``` go tool task install # dependencies, through Socket Firewall (go tool sfw) go tool task dev # http://localhost:4321, live reload -go tool task gate # what CI runs: the gates, the install, the tests, the public reference, the build, every link -go tool task check # the gate, and the reference against the sources +go tool task sources # spin, spin-terraform-aws and brand into .sources/, at the commits versions.yaml pins +go tool task check # what CI runs: the reference against the sources, and the gate +go tool task gate # the gates, the install, the tests, the public reference, the build, every link go tool task exposure # what this public repository shows: every ref, its PRs and issues go tool task --list # everything else ``` Task is a tool in `go.mod`, so `go tool task` is the version CI runs; the tasks below are named -without it. `task gate` is what CI runs, its one step: it fails first on an install that skips -the firewall (`go tool sfw check`) or a workflow step that is not one task (`go tool taskgate`), -and `task check` is that and more: CI cannot read spin while it is private, so the reference is -checked against checkouts of spin, spin-terraform-aws and brand beside this one (`SPIN=`, -`TERRAFORM=`, `BRAND=` to point elsewhere). The half that needs no spin is in the gate: -`task reference:public` fetches spin-terraform-aws and brand into `.sources/`, at the commits -`src/data/reference.json` records, and runs `refgen -only terraform,brand -check`, which builds -no CLI and keeps what the site says of spin as `reference.json` has it. +without it. `task check` is what CI runs: the reference against its sources, then `task gate`, +which fails first on an install that skips the firewall (`go tool sfw check`) or a workflow step +that is not one task (`go tool taskgate`). + +- **The sources are pinned** in `versions.yaml`, go-tools' versions format, held by + `tools/sources/repo_test.go` as every spin-stack repository is: spin, spin-terraform-aws and + brand, each at one commit. `task sources` (`tools/sources`) fetches exactly those commits, with + their history and tags, into `.sources/`, which is where `SPIN`, `TERRAFORM` and `BRAND` point + unless given (`SPIN=../spin` for a checkout of your own). refgen fails when + `src/data/reference.json` records another commit than the pins, so the two never disagree. + spin is private: `SOURCES_TOKEN`, or git's own credentials, reads it. In CI the read-only + sources App's token reaches git only as `GIT_CONFIG_*` in the one step that fetches. +- **Without spin** - a fork's pull request has no secret - CI runs `task check:public`: the gate, + whose `task reference:public` fetches spin-terraform-aws and brand at their pins and runs + `refgen -only terraform,brand -check`, which builds no CLI and keeps what the site says of spin + as `reference.json` has it. +- **The sync** is `.github/workflows/sync.yml`, every day: `task sources:bump` moves the pins to + the newest releases (spin only to one that holds the commit pinned, brand to its main), `task + reference` regenerates, and the sync App force-pushes branch `sync/sources` and opens or + updates its one pull request, which a person reviews; nothing merges it. spin is private and + this repository public, so the pull request lists the pins that moved and the files that + changed, and what spin's commits say - grouped by the pages each path of spin feeds - goes in + one issue in spin, "Site vs spin main", with the release chain: spin-machine as spin pins it + against its newest release, the last runs of `machine.yml`, `propose.yml` and + `fresh-install.yml`, and what spinbox.dev runs. Every Monday the same issue says what spin main + has past the pin. `tools/sync` writes all of it. - **The repository is public**, and so is every pull request the moment it is opened. `task exposure` (`go tool exposure`) reads every ref's commits, pull request heads included, @@ -37,8 +56,7 @@ no CLI and keeps what the site says of spin as `reference.json` has it. topic's groups at a time (`starlight-sidebar-topics`). Images and diagrams zoom in place (`starlight-image-zoom`), and every page title offers its Markdown twin, copied or opened. - **Reference pages** under `src/content/docs/reference/` are generated by `task reference` from - checkouts of spin and spin-terraform-aws beside this one (`SPIN=`, `TERRAFORM=` to point - elsewhere). Each names its source in its frontmatter (`source: repository:path`), and its footer + spin and spin-terraform-aws at their pins (`SPIN=`, `TERRAFORM=` to point elsewhere). Each names its source in its frontmatter (`source: repository:path`), and its footer links that source at the documented commit instead of an edit link. Edit the source, not the page. Every page's footer says which spin release an installation runs, and the tree the reference is generated from. @@ -99,7 +117,7 @@ no CLI and keeps what the site says of spin as `reference.json` has it. expression or a directive `src/lib/pages.ts` does not translate. - **The look** is spin-stack/brand's: its palette, checked there for contrast and perceptual distance on real panels and through colour-vision deficiencies, and the organization's mark. - `task reference` copies them from a checkout beside this one (`BRAND=` to point elsewhere) into + `task reference` copies them from brand at its pin (`BRAND=` to point elsewhere) into `src/styles/brand-*.css`, `src/assets/` and `public/`, with the app's screenshots, and `task reference:check` fails on a copy that is not brand's, on screenshots of another release than installations run, and on one whose test spin no longer has; `src/styles/custom.css` says only how the site's parts use them. diff --git a/Taskfile.yml b/Taskfile.yml index 07c8b1d..6e52c5f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,12 +3,13 @@ version: '3' # The site: Astro Starlight under src/, and two Go tools under tools/ - refgen writes the # reference pages from the sources they describe, linkcheck fails on a link the build did not # produce. SPIN, TERRAFORM and BRAND are checkouts of spin-stack/spin, spin-stack/spin-terraform-aws -# and spin-stack/brand. +# and spin-stack/brand: by default the ones task sources fetches into .sources/, at the commits +# versions.yaml pins. vars: - SPIN: '{{.SPIN | default "../spin"}}' - TERRAFORM: '{{.TERRAFORM | default "../spin-terraform-aws"}}' - BRAND: '{{.BRAND | default "../brand"}}' + SPIN: '{{.SPIN | default ".sources/spin"}}' + TERRAFORM: '{{.TERRAFORM | default ".sources/spin-terraform-aws"}}' + BRAND: '{{.BRAND | default ".sources/brand"}}' BASE_PATH: '{{.BASE_PATH | default "/website"}}' tasks: @@ -22,42 +23,45 @@ tasks: cmds: - pnpm dev + sources: + desc: Fetch spin, spin-terraform-aws and brand into .sources/, each at the commit versions.yaml pins. spin is private - SOURCES_TOKEN, or git's own credentials, reads it. + cmds: + - go run ./tools/sources spin spin-terraform-aws brand + + sources:bump: + desc: Move the pins to the newest releases - spin only to one that holds the commit pinned, brand to its main - and fetch them. Then task reference. + cmds: + - go run ./tools/sources -bump spin spin-terraform-aws brand + + sync:pr: + desc: Commit what sources:bump and reference changed to branch sync/sources as the sync App (SYNC_TOKEN, BOT its login), and open its one pull request or update it. sync.yml runs it. + cmds: + - go run ./tools/sync pr -bot '{{.BOT}}' -reference '{{.REFERENCE | default "success"}}' + + sync:issue: + desc: Write spin's issue "Site vs spin main" as the sources App (SOURCES_TOKEN, BOT its login) - MODE release, the commits a bump takes in, or main, what spin main has past the pin - with the release chain. sync.yml runs it. + cmds: + - go run ./tools/sync issue -mode '{{.MODE}}' -bot '{{.BOT}}' + reference: desc: Write the reference pages (CLI, Terraform, config file, API) and what the site copies from the spin, spin-terraform-aws and brand checkouts. cmds: + - task: sources - go run ./tools/refgen -spin {{.SPIN}} -terraform {{.TERRAFORM}} -brand {{.BRAND}} reference:check: desc: Fail if a reference page is not what the sources say. cmds: + - task: sources - go run ./tools/refgen -spin {{.SPIN}} -terraform {{.TERRAFORM}} -brand {{.BRAND}} -check reference:public: - desc: Fail if what the site has of spin-terraform-aws and brand is not what they say at the commits src/data/reference.json records. Needs no spin checkout, nor any credential. - # CI cannot check spin out, and this half needs nothing of it: what the site says of spin - # stays as reference.json has it, and is held by reference:check. - cmds: - # With its history and tags: a page pins the module at the newest release the commit holds. - - task: source - vars: { REPO: spin-terraform-aws, COMMIT: '{{(fromJson .REFERENCE).terraform.commit}}', TAGS: 'true' } - - task: source - vars: { REPO: brand, COMMIT: '{{(fromJson .REFERENCE).brand.commit}}' } + desc: Fail if what the site has of spin-terraform-aws and brand is not what they say at the commits versions.yaml pins. Needs no spin checkout, nor any credential. + # A fork's pull request has no token to read spin with, and this half needs nothing of it: + # what the site says of spin stays as reference.json has it, and is held by reference:check. + cmds: + - go run ./tools/sources spin-terraform-aws brand - go run ./tools/refgen -only terraform,brand -terraform .sources/spin-terraform-aws -brand .sources/brand -check - vars: - REFERENCE: - sh: cat src/data/reference.json - - source: - internal: true - desc: Fetch one public spin-stack repository into .sources/ at one commit, and with TAGS, its history and its tags. - status: - - test "$(git -C .sources/{{.REPO}} rev-parse HEAD 2>/dev/null)" = "{{.COMMIT}}" - - test -z "{{.TAGS}}" || test "$(git -C .sources/{{.REPO}} rev-parse --is-shallow-repository)" = false - cmds: - - rm -rf .sources/{{.REPO}} - - git init -q .sources/{{.REPO}} - - git -C .sources/{{.REPO}} fetch -q {{if .TAGS}}--tags{{else}}--depth 1{{end}} https://github.com/spin-stack/{{.REPO}} {{.COMMIT}} - - git -C .sources/{{.REPO}} checkout -q --detach FETCH_HEAD build: desc: Build the site into dist/, with llms.txt, llms-full.txt and a .md twin of every page. @@ -95,7 +99,12 @@ tasks: - go tool exposure -allow .github/exposure-allow.txt -gitleaks-config .gitleaks.toml -private ami,ci,shipset,skills,spin,spinbox.dev-deploy spin-stack/website check: - desc: Everything CI runs, and the reference against the sources. + desc: What CI runs with the token that reads spin - the reference against the sources at their pins, and the gate. cmds: - task: reference:check - task: gate + + check:public: + desc: What CI runs with no token to read spin, a fork's pull request - the gate, whose reference:public needs none. + cmds: + - task: gate diff --git a/tools/internal/github/github.go b/tools/internal/github/github.go new file mode 100644 index 0000000..a6b32e4 --- /dev/null +++ b/tools/internal/github/github.go @@ -0,0 +1,109 @@ +// Package github is the little of GitHub's REST API the site's tools ask: a client whose base +// is a field, so that a test hands it an httptest server and needs no network. +package github + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// API is the address every request is made to when a Client names none. +const API = "https://api.github.com" + +// Client asks GitHub's REST API as Token, which may be empty for what is public. +type Client struct { + Base string + Token string + HTTP *http.Client +} + +// New is a client of api.github.com as token. +func New(token string) *Client { + return &Client{Base: API, Token: token, HTTP: &http.Client{Timeout: time.Minute}} +} + +// Do sends body, as JSON when it is not nil, to path and decodes the answer into out when out is +// not nil. An answer that is not a 2xx is an error that carries what GitHub said. +func (c *Client) Do(ctx context.Context, method, path string, body, out any) error { + raw, err := c.send(ctx, method, path, body, "application/vnd.github+json") + if err != nil || out == nil { + return err + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("%s %s: %w", method, path, err) + } + return nil +} + +// Get is Do for a GET. +func (c *Client) Get(ctx context.Context, path string, out any) error { + return c.Do(ctx, http.MethodGet, path, nil, out) +} + +// File is a file of repo at ref, as its bytes. +func (c *Client) File(ctx context.Context, repo, path, ref string) ([]byte, error) { + return c.send(ctx, http.MethodGet, "/repos/"+repo+"/contents/"+path+"?ref="+ref, nil, "application/vnd.github.raw") +} + +func (c *Client) send(ctx context.Context, method, path string, body any, accept string) ([]byte, error) { + var in io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return nil, err + } + in = bytes.NewReader(raw) + } + req, err := http.NewRequestWithContext(ctx, method, strings.TrimSuffix(c.Base, "/")+path, in) + if err != nil { + return nil, err + } + req.Header.Set("Accept", accept) + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.Token != "" { + req.Header.Set("Authorization", "Bearer "+c.Token) + } + hc := c.HTTP + if hc == nil { + hc = http.DefaultClient + } + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() // read-only + raw, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", method, path, err) + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(raw))) + } + return raw, nil +} + +// GitAuth is what a git command's environment takes token as: an Authorization header for +// github.com, handed in GIT_CONFIG_* so it is never written to a repository's config, where a +// later step would find it. Basic, as actions/checkout sends it: git's HTTP answers no Bearer. +// No token, no header: what is public needs none. +func GitAuth(token string) []string { + if token == "" { + return nil + } + basic := base64.StdEncoding.EncodeToString([]byte("x-access-token:" + token)) + return []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=http.https://github.com/.extraheader", + "GIT_CONFIG_VALUE_0=AUTHORIZATION: basic " + basic, + } +} diff --git a/tools/refgen/commands.go b/tools/refgen/commands.go index 476cf33..3051f0b 100644 --- a/tools/refgen/commands.go +++ b/tools/refgen/commands.go @@ -306,14 +306,15 @@ func flagsOf(help string) map[string]flagSpec { return flags } -// taskNames lists every task spin's Taskfile has. --no-status: listing with status writes the -// checksums of the tasks it looks at. +// taskNames lists every task spin's Taskfile has, as the Task spin's go.mod pins reads it: the +// one its Taskfile is written for, and one CI has without installing a Task of its own. +// --no-status: listing with status writes the checksums of the tasks it looks at. func taskNames(spin string) (map[string]bool, error) { - cmd := exec.Command("task", "--list-all", "--json", "--no-status") + cmd := exec.Command("go", "tool", "task", "--list-all", "--json", "--no-status") cmd.Dir = spin out, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("task --list-all in %s: %w", spin, err) + return nil, fmt.Errorf("go tool task --list-all in %s: %w", spin, err) } var list struct { Tasks []struct { diff --git a/tools/refgen/main.go b/tools/refgen/main.go index 7fd24c3..2e825fb 100644 --- a/tools/refgen/main.go +++ b/tools/refgen/main.go @@ -40,6 +40,7 @@ func run() error { outputsDir := flag.String("outputs", "src/data/output", "where what the CLI prints, as spin's goldens hold it, is copied") assets := flag.String("diagrams", "src/assets/diagrams", "where the diagrams the source repositories draw are copied") figuresData := flag.String("figures", "src/data/figures.json", "each diagram's source and words, for its Markdown twin") + pinned := flag.String("versions", "versions.yaml", "what the sources are pinned at, which reference.json must record") check := flag.Bool("check", false, "write nothing; fail if a page on disk differs from what the sources say") flag.Parse() @@ -159,6 +160,7 @@ func run() error { for _, c := range []func() ([]string, error){ func() ([]string, error) { return checkDescriptions(*docs) }, func() ([]string, error) { return checkRepos(*docs) }, + func() ([]string, error) { return checkPins(*pinned, r) }, func() ([]string, error) { return checkDomains(*docs, *tf) }, func() ([]string, error) { return checkTofu(*docs, *tf) }, } { diff --git a/tools/refgen/pins.go b/tools/refgen/pins.go new file mode 100644 index 0000000..52a77aa --- /dev/null +++ b/tools/refgen/pins.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + + "github.com/spin-stack/go-tools/versions" +) + +// checkPins holds reference.json to versions.yaml: the commits the reference was generated from +// are the ones pinned, and a pin at a release is that release. Two records of the same thing +// that could disagree would have the sync move one and the site go on documenting the other; a +// reference generated from a checkout that is not the pin is one CI cannot reproduce. +func checkPins(file string, r reference) ([]string, error) { + v, err := versions.Load(file) + if err != nil { + return nil, err + } + var bad []string + for _, c := range []struct { + name, commit, release, field string + }{ + {"spin", r.Spin.Commit, r.Spin.Tag, "spin.tag"}, + {"spin-terraform-aws", r.Terraform.Commit, r.Terraform.Release, "terraform.release"}, + {"brand", r.Brand.Commit, "", ""}, + } { + e, err := v.Get(c.name) + if err != nil { + return nil, err + } + if c.commit != e.Pin { + bad = append(bad, fmt.Sprintf("reference.json: %s at %s; %s pins %s (task sources, then task reference)", c.name, c.commit, file, e.Pin)) + } + if c.field != "" && releaseParts.MatchString(e.Version) && c.release != e.Version { + bad = append(bad, fmt.Sprintf("reference.json: %s is %q; %s pins %s at the release %s", c.field, c.release, file, c.name, e.Version)) + } + } + return bad, nil +} diff --git a/tools/refgen/pins_test.go b/tools/refgen/pins_test.go new file mode 100644 index 0000000..f7df5b4 --- /dev/null +++ b/tools/refgen/pins_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const pinsFile = `- name: spin + kind: git + source: https://github.com/spin-stack/spin + version: main + pin: 77380b13da641c24e3f79b9c7586c39888f4e7ba + track: github-release +- name: spin-terraform-aws + kind: git + source: https://github.com/spin-stack/spin-terraform-aws + version: v20261024.01 + pin: 3df623f0ca2b7f91650e0ac857ba0bde295e0481 + track: github-release +- name: brand + kind: git + source: https://github.com/spin-stack/brand + version: main + pin: c882b6f0f3901787525ef16ef29058c9c1418cbe + track: branch +` + +// reference.json records the commits versions.yaml pins, and the release a pin at one names. +func TestReferenceRecordsThePins(t *testing.T) { + file := filepath.Join(t.TempDir(), "versions.yaml") + if err := os.WriteFile(file, []byte(pinsFile), 0o644); err != nil { + t.Fatal(err) + } + var r reference + r.Spin.Commit = "77380b13da641c24e3f79b9c7586c39888f4e7ba" + r.Terraform.Commit, r.Terraform.Release = "3df623f0ca2b7f91650e0ac857ba0bde295e0481", "v20261024.01" + r.Brand.Commit = "c882b6f0f3901787525ef16ef29058c9c1418cbe" + if bad, err := checkPins(file, r); err != nil || len(bad) != 0 { + t.Fatalf("checkPins of the pins = %q, %v; want nothing", bad, err) + } + + moved := r + moved.Brand.Commit = strings.Repeat("ab", 20) + moved.Terraform.Release = "v20261019.01" + bad, err := checkPins(file, moved) + if err != nil { + t.Fatal(err) + } + if len(bad) != 2 || !strings.Contains(bad[0], "terraform.release") || !strings.Contains(bad[1], "brand at "+moved.Brand.Commit) { + t.Errorf("checkPins = %q; want terraform.release and brand's commit", bad) + } +} diff --git a/tools/sources/bump.go b/tools/sources/bump.go new file mode 100644 index 0000000..32bec46 --- /dev/null +++ b/tools/sources/bump.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "fmt" + "os" + "slices" + "strconv" + "strings" + + "github.com/spin-stack/go-tools/versions" + "github.com/spin-stack/go-tools/versions/upstream" + + "github.com/spin-stack/website/tools/internal/github" +) + +// bumpFile moves each source named to the version next says, and writes file when one moved. +func bumpFile(ctx context.Context, file string, names []string, gh *github.Client) error { + v, err := versions.Load(file) + if err != nil { + return err + } + var out []byte + for _, st := range upstream.Check(ctx, v) { + if !slices.Contains(names, st.Entry.Name) { + continue + } + to, err := next(st, func(tag string) (bool, error) { return holds(ctx, gh, st.Entry, tag) }) + if err != nil { + return fmt.Errorf("%s: %w", st.Entry.Name, err) + } + if to == "" { + fmt.Printf("%s: stays at %s %s\n", st.Entry.Name, st.Entry.Version, st.Entry.Pin) + continue + } + e, b, err := upstream.Bump(ctx, v, st.Entry.Name, to) + if err != nil { + return err + } + if v, err = versions.Parse(b); err != nil { + return err + } + out = b + fmt.Printf("%s: %s %s -> %s %s\n", e.Name, st.Entry.Version, st.Entry.Pin, e.Version, e.Pin) + } + if out == nil { + return nil + } + return os.WriteFile(file, out, 0o644) +} + +// next is the version a source moves to from where check found it, "" when it stays. holds says +// whether a release tag holds the commit pinned. +// +// - A branch (brand's main) moves when the branch did: the newest of it is what it has to say. +// - A release moves only to a newer release, never back. +// - A commit of main, pinned until a release holds it, moves to the newest release only once +// that release holds the commit: the release before it documents less than the site does, +// and the site would go backwards. +func next(st upstream.Status, holds func(tag string) (bool, error)) (string, error) { + e := st.Entry + if st.Newest == "" { + return "", fmt.Errorf("its newest is unknown: %s", st.Note) + } + if !st.Behind { + return "", nil + } + if e.Track == "branch" { + return e.Version, nil + } + // The go-tools track spells the tag as the version is spelled, and main has no v: the tag + // does. + newest := st.Newest + if !strings.HasPrefix(newest, "v") && release.MatchString("v"+newest) { + newest = "v" + newest + } + if !release.MatchString(newest) { + return "", fmt.Errorf("its newest, %s, is not a release v.", newest) + } + if release.MatchString(e.Version) { + switch c := compareReleases(newest, e.Version); { + case c > 0: + return newest, nil + case c == 0: + // The same release at another commit: a tag moved, which is for a person to look at. + return "", fmt.Errorf("%s is now %s; versions.yaml pins %s", e.Version, st.Note, e.Pin) + } + return "", nil + } + ok, err := holds(newest) + if err != nil || !ok { + return "", err + } + return newest, nil +} + +// compareReleases orders two release names by date and then by number: v20261019.10 is after +// v20261019.9, which the strings' order would not say. +func compareReleases(a, b string) int { + ma, mb := release.FindStringSubmatch(a), release.FindStringSubmatch(b) + da, _ := strconv.Atoi(ma[1]) + db, _ := strconv.Atoi(mb[1]) + if da != db { + return da - db + } + na, _ := strconv.Atoi(ma[2]) + nb, _ := strconv.Atoi(mb[2]) + return na - nb +} + +// holds is whether tag of e's repository is e's pin or a descendant of it, as GitHub's compare +// says: a bump runs before the new pin is fetched, and the API knows the history without it. +func holds(ctx context.Context, gh *github.Client, e versions.Entry, tag string) (bool, error) { + repo := strings.TrimPrefix(e.Source, "https://github.com/") + var cmp struct { + Status string `json:"status"` + } + if err := gh.Get(ctx, "/repos/"+repo+"/compare/"+e.Pin+"..."+tag, &cmp); err != nil { + return false, err + } + return cmp.Status == "ahead" || cmp.Status == "identical", nil +} diff --git a/tools/sources/fetch.go b/tools/sources/fetch.go new file mode 100644 index 0000000..7812bc3 --- /dev/null +++ b/tools/sources/fetch.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + "github.com/spin-stack/go-tools/versions" +) + +// release is a version that names a release tag, v., as spin and spin-terraform-aws +// cut them; anything else (main) is a branch the pin is a commit of. +var release = regexp.MustCompile(`^v(20\d{6})\.(\d+)$`) + +// fetch makes dir a checkout of e's pin, with the history that leads to it and every tag: +// refgen asks which release tags point at or are merged into it, and reads spin's versions.yaml +// at the release an installation runs. A checkout already at the pin, whole, is left as it is, +// so a second run asks nothing of the network and needs no token. +func fetch(ctx context.Context, dir string, e versions.Entry, auth []string) error { + if head, err := git(ctx, dir, nil, "rev-parse", "HEAD"); err == nil && head == e.Pin { + if shallow, err := git(ctx, dir, nil, "rev-parse", "--is-shallow-repository"); err == nil && shallow == "false" { + return tagged(ctx, dir, e) + } + } + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if _, err := git(ctx, dir, nil, "init", "-q"); err != nil { + return err + } + if _, err := git(ctx, dir, auth, "fetch", "-q", "--tags", e.Source, e.Pin); err != nil { + return err + } + if err := tagged(ctx, dir, e); err != nil { + return err + } + _, err := git(ctx, dir, nil, "checkout", "-q", "--detach", e.Pin) + return err +} + +// tagged fails when e's version is a release whose tag is not its pin: a tag moved, or a pin +// written by hand, would have the site name one release while documenting another tree. +func tagged(ctx context.Context, dir string, e versions.Entry) error { + if !release.MatchString(e.Version) { + return nil + } + got, err := git(ctx, dir, nil, "rev-parse", "--verify", "-q", "refs/tags/"+e.Version+"^{commit}") + if err != nil { + return fmt.Errorf("%s has no tag %s", e.Source, e.Version) + } + if got != e.Pin { + return fmt.Errorf("%s's tag %s is %s, and versions.yaml pins %s", e.Source, e.Version, got, e.Pin) + } + return nil +} + +// git runs git in dir with env added to this process's, its output trimmed. It never asks for a +// password: a source the token cannot read fails rather than waits. +func git(ctx context.Context, dir string, env []string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(append(os.Environ(), "GIT_TERMINAL_PROMPT=0"), env...) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(out)), nil +} diff --git a/tools/sources/main.go b/tools/sources/main.go new file mode 100644 index 0000000..7b020bd --- /dev/null +++ b/tools/sources/main.go @@ -0,0 +1,82 @@ +// Command sources fetches what the site is generated from - each repository versions.yaml pins - +// into .sources/, at exactly the pinned commit, and with -bump moves the pins to the newest +// releases first: +// +// go run ./tools/sources [-bump] ... +// +// A private repository is fetched with SOURCES_TOKEN, which reaches git only as GIT_CONFIG_* in +// the environment of the git commands this runs: nothing is written to a checkout's config, so +// the steps after this one, which build and run what was fetched, never see it. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "path/filepath" + "strings" + + "github.com/spin-stack/go-tools/versions" + + "github.com/spin-stack/website/tools/internal/github" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + err := run(ctx) + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "sources:", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + file := flag.String("file", versions.File, "the versions.yaml that pins the sources") + dir := flag.String("dir", ".sources", "where each source is fetched, into a directory of its name") + bump := flag.Bool("bump", false, "first move each pin named to the newest release it can move to, and write the file") + flag.Parse() + names := flag.Args() + if len(names) == 0 { + return errors.New("name the sources, as versions.yaml names them") + } + token := os.Getenv("SOURCES_TOKEN") + if *bump { + // What the go-tools resolvers ask of GitHub's API they ask as GITHUB_TOKEN, and of git + // with what git's environment carries. + if token != "" { + if err := os.Setenv("GITHUB_TOKEN", token); err != nil { + return err + } + for _, kv := range github.GitAuth(token) { + k, v, _ := strings.Cut(kv, "=") + if err := os.Setenv(k, v); err != nil { + return err + } + } + } + if err := bumpFile(ctx, *file, names, github.New(token)); err != nil { + return err + } + } + v, err := versions.Load(*file) + if err != nil { + return err + } + for _, n := range names { + e, err := v.Get(n) + if err != nil { + return err + } + if e.Kind != versions.Git { + return fmt.Errorf("%s is a %s; a source is a git repository", n, e.Kind) + } + if err := fetch(ctx, filepath.Join(*dir, n), e, github.GitAuth(token)); err != nil { + return fmt.Errorf("%s: %w", n, err) + } + } + return nil +} diff --git a/tools/sources/repo_test.go b/tools/sources/repo_test.go new file mode 100644 index 0000000..1b61cfe --- /dev/null +++ b/tools/sources/repo_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "regexp" + "testing" + + "github.com/spin-stack/go-tools/versions/gate" +) + +// The site is held to its versions.yaml as any spin-stack repository is: a commit written anywhere +// else is a pin no bump moves. What refgen writes records the commits the reference was generated +// from rather than pins them, and refgen fails when reference.json's are not the pins. A source +// is read where the Taskfile names it to this command. +func TestThisRepositoryPassesTheGate(t *testing.T) { + g := gate.Gate{ + Root: "../..", + Elsewhere: []string{ + "go.sum", + // What refgen writes: reference.json names the commits, and the rest link spin's + // tree at its commit (the status page's tests, the security pages, the diagrams) or + // copy what spin and brand recorded (the CLI's goldens' digests, the screenshots'). + "src/data/reference.json", + "src/data/figures.json", + "src/data/shots.json", + "src/data/output/", + "src/content/docs/reference/", + }, + Reads: append([]*regexp.Regexp{regexp.MustCompile(`go run \./tools/sources(?: -bump)?((?: [a-z][a-z0-9-]*)+)`)}, gate.DefaultReads...), + } + if err := g.Check(); err != nil { + t.Error(err) + } +} diff --git a/tools/sources/sources_test.go b/tools/sources/sources_test.go new file mode 100644 index 0000000..e24aaee --- /dev/null +++ b/tools/sources/sources_test.go @@ -0,0 +1,216 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/spin-stack/go-tools/versions" + "github.com/spin-stack/go-tools/versions/upstream" + + "github.com/spin-stack/website/tools/internal/github" +) + +// upstreamRepo is a bare repository with two commits, the first tagged v20261024.01, served from +// disk as GitHub serves one: any commit its history holds can be asked for. +func upstreamRepo(t *testing.T) (url, first, second string) { + t.Helper() + dir := t.TempDir() + work, bare := filepath.Join(dir, "work"), filepath.Join(dir, "bare.git") + run := func(d string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", d}, args...)...) + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + if err := os.MkdirAll(work, 0o755); err != nil { + t.Fatal(err) + } + run(work, "init", "-q", "-b", "main") + for i, body := range []string{"one\n", "two\n"} { + if err := os.WriteFile(filepath.Join(work, "f"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + run(work, "add", "f") + run(work, "commit", "-q", "-m", body) + if i == 0 { + first = run(work, "rev-parse", "HEAD") + run(work, "tag", "-a", "-m", "release", "v20261024.01") + } + } + second = run(work, "rev-parse", "HEAD") + run(dir, "clone", "-q", "--bare", work, bare) + run(bare, "config", "uploadpack.allowReachableSHA1InWant", "true") + return "file://" + bare, first, second +} + +func head(t *testing.T, dir string) string { + t.Helper() + got, err := git(context.Background(), dir, nil, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + return got +} + +// A source is fetched at exactly its pin, whole and with its tags, whatever its branch is at, and +// moved when the pin moves. +func TestFetchChecksOutThePinWithItsHistoryAndTags(t *testing.T) { + ctx := context.Background() + url, first, second := upstreamRepo(t) + dir := filepath.Join(t.TempDir(), "src") + + if err := fetch(ctx, dir, versions.Entry{Name: "src", Source: url, Version: "main", Pin: second}, nil); err != nil { + t.Fatal(err) + } + if got := head(t, dir); got != second { + t.Errorf("HEAD is %s; want the pin %s", got, second) + } + if tags, _ := git(ctx, dir, nil, "tag", "--merged", "HEAD"); tags != "v20261024.01" { + t.Errorf("tags merged into the pin: %q; want v20261024.01, which refgen reads the module's release from", tags) + } + if shallow, _ := git(ctx, dir, nil, "rev-parse", "--is-shallow-repository"); shallow != "false" { + t.Error("the checkout is shallow; refgen reads its history") + } + + if err := fetch(ctx, dir, versions.Entry{Name: "src", Source: url, Version: "v20261024.01", Pin: first}, nil); err != nil { + t.Fatal(err) + } + if got := head(t, dir); got != first { + t.Errorf("after the pin moved, HEAD is %s; want %s", got, first) + } +} + +// A checkout already at its pin is not fetched again: a run after the one with the token needs +// none, and asks nothing of the network. +func TestFetchLeavesACheckoutAtItsPin(t *testing.T) { + ctx := context.Background() + url, _, second := upstreamRepo(t) + dir := filepath.Join(t.TempDir(), "src") + e := versions.Entry{Name: "src", Source: url, Version: "main", Pin: second} + if err := fetch(ctx, dir, e, nil); err != nil { + t.Fatal(err) + } + e.Source = "file:///nowhere" + if err := fetch(ctx, dir, e, nil); err != nil { + t.Errorf("a checkout at its pin was fetched again: %v", err) + } +} + +// A release whose tag is not its pin is refused: the site would name one release and document +// another tree. +func TestFetchRefusesATagThatIsNotThePin(t *testing.T) { + url, _, second := upstreamRepo(t) + err := fetch(context.Background(), filepath.Join(t.TempDir(), "src"), versions.Entry{Name: "src", Source: url, Version: "v20261024.01", Pin: second}, nil) + if err == nil || !strings.Contains(err.Error(), "tag v20261024.01 is") { + t.Errorf("fetch = %v; want the tag named as not the pin", err) + } +} + +// A pin the repository does not have fails, and leaves nothing a later run takes for a checkout. +func TestFetchFailsOnACommitTheRepositoryDoesNotHave(t *testing.T) { + url, _, _ := upstreamRepo(t) + dir := filepath.Join(t.TempDir(), "src") + missing := strings.Repeat("ab", 20) + if err := fetch(context.Background(), dir, versions.Entry{Name: "src", Source: url, Version: "main", Pin: missing}, nil); err == nil { + t.Fatal("fetched a commit the repository does not have") + } + if _, err := git(context.Background(), dir, nil, "rev-parse", "--verify", "-q", "HEAD"); err == nil { + t.Error("a failed fetch left a checkout with a HEAD") + } +} + +// The token reaches git as configuration in the environment, never as a file, and only when there +// is one. +func TestTheTokenIsAnExtraHeaderInTheEnvironment(t *testing.T) { + if got := github.GitAuth(""); got != nil { + t.Errorf("GitAuth with no token = %q; want nothing", got) + } + env := strings.Join(github.GitAuth("ghs_x"), "\n") + for _, want := range []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http.https://github.com/.extraheader", "GIT_CONFIG_VALUE_0=AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3g="} { + if !strings.Contains(env, want) { + t.Errorf("GitAuth lacks %s:\n%s", want, env) + } + } +} + +// Where each kind of pin moves, and where it does not. +func TestNextMovesForwardOnly(t *testing.T) { + const pin = "77380b13da641c24e3f79b9c7586c39888f4e7ba" + main := versions.Entry{Name: "spin", Version: "main", Pin: pin, Track: "github-release"} + tagged := versions.Entry{Name: "spin-terraform-aws", Version: "v20261024.01", Pin: pin, Track: "github-release"} + branch := versions.Entry{Name: "brand", Version: "main", Pin: pin, Track: "branch"} + holds := func(ok bool) func(string) (bool, error) { return func(string) (bool, error) { return ok, nil } } + for _, c := range []struct { + name string + st upstream.Status + holds bool + want string + }{ + {"a branch that moved moves along it", upstream.Status{Entry: branch, Newest: "main", Behind: true}, false, "main"}, + {"a branch that did not stays", upstream.Status{Entry: branch, Newest: "main"}, false, ""}, + {"a newer release", upstream.Status{Entry: tagged, Newest: "v20261025.01", Behind: true}, false, "v20261025.01"}, + {"a newer release the same day, by number", upstream.Status{Entry: tagged, Newest: "v20261024.10", Behind: true}, false, "v20261024.10"}, + {"never back to an older release", upstream.Status{Entry: tagged, Newest: "v20261019.02", Behind: true}, false, ""}, + {"main to a release that holds the commit", upstream.Status{Entry: main, Newest: "v20261025.01", Behind: true}, true, "v20261025.01"}, + {"main to a release the track spells without its v", upstream.Status{Entry: main, Newest: "20261025.01", Behind: true}, true, "v20261025.01"}, + {"main stays past a release that does not", upstream.Status{Entry: main, Newest: "v20261019.02", Behind: true}, false, ""}, + } { + got, err := next(c.st, holds(c.holds)) + if err != nil || got != c.want { + t.Errorf("%s: next = %q, %v; want %q", c.name, got, err, c.want) + } + } + for _, st := range []upstream.Status{ + {Entry: tagged, Note: "versions: the API answered 404"}, + {Entry: tagged, Newest: "latest", Behind: true}, + {Entry: tagged, Newest: "v20261024.01", Behind: true, Note: "v20261024.01 is now " + strings.Repeat("cd", 20)}, + } { + if got, err := next(st, holds(true)); err == nil { + t.Errorf("next(%+v) = %q; want an error", st, got) + } + } + boom := errors.New("boom") + if _, err := next(upstream.Status{Entry: main, Newest: "v20261025.01", Behind: true}, func(string) (bool, error) { return false, boom }); !errors.Is(err, boom) { + t.Errorf("next lost the compare's error: %v", err) + } +} + +// Whether a release holds the pin is GitHub's compare from the pin to the tag: ahead or the same. +func TestHoldsIsTheCompareFromThePin(t *testing.T) { + const pin = "77380b13da641c24e3f79b9c7586c39888f4e7ba" + status := map[string]string{"v20261025.01": "ahead", "v20261026.01": "identical", "v20261019.02": "behind", "v20261020.01": "diverged"} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer ghs_x" { + http.Error(w, "no token", http.StatusNotFound) + return + } + tag, ok := strings.CutPrefix(r.URL.Path, "/repos/spin-stack/spin/compare/"+pin+"...") + if !ok || status[tag] == "" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`{"status":"` + status[tag] + `"}`)) + })) + defer srv.Close() + gh := &github.Client{Base: srv.URL, Token: "ghs_x"} + e := versions.Entry{Source: "https://github.com/spin-stack/spin", Pin: pin} + for tag, want := range map[string]bool{"v20261025.01": true, "v20261026.01": true, "v20261019.02": false, "v20261020.01": false} { + if got, err := holds(context.Background(), gh, e, tag); err != nil || got != want { + t.Errorf("holds(%s) = %v, %v; want %v", tag, got, err, want) + } + } + if _, err := holds(context.Background(), gh, e, "v20990101.01"); err == nil { + t.Error("holds took a 404 for an answer") + } +} diff --git a/tools/sync/chain.go b/tools/sync/chain.go new file mode 100644 index 0000000..71736fc --- /dev/null +++ b/tools/sync/chain.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/spin-stack/go-tools/versions" + + "github.com/spin-stack/website/tools/internal/github" +) + +// runs are the workflows whose last run the chain report gives: each is a link of the chain from a +// spin-machine release to an installation, and the one that stalls is the one nobody sees. +var runs = []struct{ repo, workflow, what string }{ + {"spin-stack/spin", "machine.yml", "a spin-machine release proposed to spin"}, + {"spin-stack/spin-terraform-aws", "propose.yml", "the module proposed to the deployments"}, + {"spin-stack/spin-terraform-aws", "fresh-install.yml", "an installation from nothing, weekly"}, +} + +var ( + tfvarsVersion = regexp.MustCompile(`(?m)^spin_version\s*=\s*"([^"]+)"`) + moduleRef = regexp.MustCompile(`spin-terraform-aws\?ref=([^"\s]+)`) +) + +// chain is the release-chain report: spin-machine as spin main pins it against its newest +// release, when each link's workflow last ran and how, and what spinbox.dev runs. Each line says +// what it could not read rather than stopping the report: it is advice, and the rest of it holds. +func chain(ctx context.Context, gh *github.Client) string { + var b strings.Builder + b.WriteString("| | |\n|---|---|\n") + row := func(k, v string) { fmt.Fprintf(&b, "| %s | %s |\n", k, v) } + + pinned, err := machinePin(ctx, gh) + var latest struct { + Tag string `json:"tag_name"` + } + if err == nil { + err = gh.Get(ctx, "/repos/spin-stack/spin-machine/releases/latest", &latest) + } + switch { + case err != nil: + row("spin-machine", "could not read: "+err.Error()) + case pinned == latest.Tag: + row("spin-machine", fmt.Sprintf("spin main pins %s, the newest release", pinned)) + default: + row("spin-machine", fmt.Sprintf("spin main pins %s; the newest release is **%s**", pinned, latest.Tag)) + } + + for _, r := range runs { + var list struct { + Runs []struct { + Created string `json:"created_at"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + URL string `json:"html_url"` + } `json:"workflow_runs"` + } + key := fmt.Sprintf("`%s` %s", r.workflow, r.what) + switch err := gh.Get(ctx, "/repos/"+r.repo+"/actions/workflows/"+r.workflow+"/runs?per_page=1", &list); { + case err != nil: + row(key, "could not read: "+err.Error()) + case len(list.Runs) == 0: + row(key, "never ran") + default: + last := list.Runs[0] + how := last.Conclusion + if how == "" { + how = last.Status + } + row(key, fmt.Sprintf("last ran %s, [%s](%s)", day(last.Created), how, last.URL)) + } + } + + version, ref, err := deployment(ctx, gh) + if err != nil { + row("spinbox.dev", "could not read: "+err.Error()) + } else { + row("spinbox.dev", fmt.Sprintf("runs %s, with the module at `%s`", version, ref)) + } + return b.String() +} + +// machinePin is the spin-machine release spin's main pins in its versions.yaml. +func machinePin(ctx context.Context, gh *github.Client) (string, error) { + raw, err := gh.File(ctx, "spin-stack/spin", versions.File, "main") + if err != nil { + return "", err + } + v, err := versions.Parse(raw) + if err != nil { + return "", err + } + e, err := v.Get("spin-machine") + return e.Version, err +} + +// deployment is the release spinbox.dev runs and the module ref it takes the installation from. +func deployment(ctx context.Context, gh *github.Client) (string, string, error) { + tfvars, err := gh.File(ctx, "spin-stack/spinbox.dev-deploy", "terraform.tfvars", "main") + if err != nil { + return "", "", err + } + main, err := gh.File(ctx, "spin-stack/spinbox.dev-deploy", "main.tf", "main") + if err != nil { + return "", "", err + } + v, r := tfvarsVersion.FindSubmatch(tfvars), moduleRef.FindSubmatch(main) + if v == nil || r == nil { + return "", "", fmt.Errorf("spinbox.dev-deploy names no spin_version in terraform.tfvars or no module ref in main.tf") + } + return string(v[1]), string(r[1]), nil +} + +// day is the date of a GitHub timestamp. +func day(ts string) string { + d, _, _ := strings.Cut(ts, "T") + return d +} diff --git a/tools/sync/digest.go b/tools/sync/digest.go new file mode 100644 index 0000000..488abf0 --- /dev/null +++ b/tools/sync/digest.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/spin-stack/website/tools/internal/github" +) + +// feed is a path of spin's tree and the pages of the site it feeds: a commit that touches it is +// one a reviewer reads those pages against. +type feed struct { + Path string + Pages []string +} + +// read are the paths refgen reads that no page names as its source: what it holds the hand-written +// pages to. Each is every page of a kind, not a list of them, since the check is what finds which. +var read = []feed{ + {"internal/features/features.go", []string{"the landing page's count and cards", "every feature: link"}}, + {"hack/env-catalog.txt", []string{"every page that names a SPIN_ variable"}}, + {"Taskfile.yml", []string{"every page that quotes a task command"}}, + {"internal/storage/qcow/manager.go", []string{"every page with a of constants.json"}}, + {"internal/controlplane/compat/compat.go", []string{"every page with a of constants.json"}}, + {"versions.yaml", []string{"reference/release"}}, +} + +var ( + sourceLine = regexp.MustCompile(`(?m)^source: "spin:([^"]+)"$`) + figureName = regexp.MustCompile(`= total { + break + } + } + out = out[:min(len(out), maxCommits)] + for i := range out { + var c struct { + Files []struct { + Name string `json:"filename"` + } `json:"files"` + } + if err := gh.Get(ctx, "/repos/"+repo+"/commits/"+out[i].SHA, &c); err != nil { + return nil, 0, err + } + for _, f := range c.Files { + out[i].Files = append(out[i].Files, f.Name) + } + } + return out, total, nil +} + +// digest is the commits grouped by the path of spin each touches that feeds the site, each group +// headed by the pages that path feeds, and a count of the rest: what a reviewer reads the site +// against, and nothing they need not. +func digest(cs []commit, total int, fs []feed) string { + var b strings.Builder + if total == 0 { + return "No commits.\n" + } + fmt.Fprintf(&b, "%d commit%s", total, map[bool]string{true: "s"}[total != 1]) + if total > len(cs) { + fmt.Fprintf(&b, ", the first %d read", len(cs)) + } + b.WriteString(".\n") + touched := map[string]bool{} + for _, f := range fs { + var lines []string + for _, c := range cs { + if slices.ContainsFunc(c.Files, f.feeds) { + lines = append(lines, fmt.Sprintf("- %s %s", short(c.SHA), c.Subject)) + touched[c.SHA] = true + } + } + if lines == nil { + continue + } + fmt.Fprintf(&b, "\n#### `%s`\n\nFeeds: %s\n\n%s\n", f.Path, strings.Join(f.Pages, ", "), strings.Join(lines, "\n")) + } + if n := len(cs) - len(touched); n > 0 { + fmt.Fprintf(&b, "\n%d of them touch no path the site reads.\n", n) + } + return b.String() +} + +func short(sha string) string { return sha[:min(len(sha), 10)] } diff --git a/tools/sync/issue.go b/tools/sync/issue.go new file mode 100644 index 0000000..fa88355 --- /dev/null +++ b/tools/sync/issue.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/spin-stack/website/tools/internal/github" +) + +// issueTitle is the one issue in spin that the sync keeps: spin is private and the website public, +// so what quotes spin's commits is written where only those who read spin read it. +const issueTitle = "Site vs spin main" + +// issueHead opens the issue's body, above its sections. +const issueHead = "What the site at https://spin-stack.github.io/website/ documents of spin, against what spin has. " + + "Written by spin-stack/website's `.github/workflows/sync.yml`, section by section; an edit by hand is written over. " + + "Advice: nothing here fails a check.\n" + +// sections, in the order the body has them: each job writes its own and leaves the others. +var sections = []struct{ key, title string }{ + {"release", "The last release sync"}, + {"main", "spin main past the pin"}, + {"chain", "The release chain"}, +} + +// withSection is body with the section key's text replaced by text, the others as they were; +// a body that has none of them is begun again with the head. +func withSection(body, key, text string) string { + have := map[string]string{} + for _, s := range sections { + start, end := "", "" + if _, rest, ok := strings.Cut(body, start); ok { + if in, _, ok := strings.Cut(rest, end); ok { + have[s.key] = in + } + } + } + have[key] = "\n" + strings.TrimSpace(text) + "\n" + var b strings.Builder + b.WriteString(issueHead) + for _, s := range sections { + in, ok := have[s.key] + if !ok { + in = "\nNot written yet.\n" + } + fmt.Fprintf(&b, "\n## %s\n\n%s\n", s.title, s.key, in, s.key) + } + return b.String() +} + +// writeIssue writes the sections given into the open issue issueTitle in repo, which bot opened, +// and opens it when there is none. It is found by its author, not searched for: GitHub's search +// is behind its writes, and a second issue is what that would make. +func writeIssue(ctx context.Context, gh *github.Client, repo, bot string, texts map[string]string) (string, error) { + var open []struct { + Number int `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + URL string `json:"html_url"` + PR any `json:"pull_request"` + } + q := url.Values{"state": {"open"}, "creator": {bot}, "per_page": {"100"}} + if err := gh.Get(ctx, "/repos/"+repo+"/issues?"+q.Encode(), &open); err != nil { + return "", err + } + body, number, link := "", 0, "" + for _, i := range open { + if i.Title == issueTitle && i.PR == nil { + body, number, link = i.Body, i.Number, i.URL + break + } + } + for _, s := range sections { + if t, ok := texts[s.key]; ok { + body = withSection(body, s.key, t) + } + } + var out struct { + URL string `json:"html_url"` + } + if number == 0 { + err := gh.Do(ctx, http.MethodPost, "/repos/"+repo+"/issues", map[string]string{"title": issueTitle, "body": body}, &out) + return out.URL, err + } + err := gh.Do(ctx, http.MethodPatch, fmt.Sprintf("/repos/%s/issues/%d", repo, number), map[string]string{"body": body}, &out) + if out.URL == "" { + out.URL = link + } + return out.URL, err +} diff --git a/tools/sync/main.go b/tools/sync/main.go new file mode 100644 index 0000000..c5fc228 --- /dev/null +++ b/tools/sync/main.go @@ -0,0 +1,140 @@ +// Command sync writes what the daily sync of the site with its sources says, and where: +// +// go run ./tools/sync pr -bot [bot] the sync pull request, as the sync App (SYNC_TOKEN) +// go run ./tools/sync issue -mode release -bot [bot] spin's issue: the commits the bump took in, and the release chain (SOURCES_TOKEN) +// go run ./tools/sync issue -mode main -bot [bot] spin's issue: spin main past the pin, and the release chain +// +// The website is public and spin private: the pull request says which pins moved and which +// files changed, and what spin's commits say goes in one issue in spin. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "time" + + "github.com/spin-stack/go-tools/versions" + + "github.com/spin-stack/website/tools/internal/github" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + err := run(ctx, os.Args[1:]) + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "sync:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, argv []string) error { + if len(argv) == 0 { + return errors.New("pr or issue") + } + fs := flag.NewFlagSet(argv[0], flag.ContinueOnError) + bot := fs.String("bot", "", "the App's bot login, [bot], that writes") + base := fs.String("base", "origin/main", "the ref whose versions.yaml the pins moved from") + mode := fs.String("mode", "", "issue: release, what the bump took in, or main, what spin main has past the pin") + reference := fs.String("reference", "success", "pr: how task reference ended, as the step's outcome says") + dry := fs.Bool("print", false, "issue: print the sections and write nothing, to read them before the issue does") + docs := fs.String("docs", "src/content/docs", "the pages, whose sources the digest groups commits by") + figures := fs.String("figures", "src/data/figures.json", "the diagrams' sources") + if err := fs.Parse(argv[1:]); err != nil { + return err + } + if *bot == "" { + return errors.New("-bot names who writes") + } + now, err := versions.Load(versions.File) + if err != nil { + return err + } + raw, err := gitOut(ctx, nil, "show", *base+":"+versions.File) + if err != nil { + return err + } + was, err := versions.Parse([]byte(raw)) + if err != nil { + return err + } + switch argv[0] { + case "pr": + ms := moves(was, now) + changed, err := changedFiles(ctx) + if err != nil { + return err + } + if len(ms) == 0 && len(changed) == 0 { + fmt.Println("nothing moved: every pin is at its newest, and the reference is what they say") + return nil + } + token := os.Getenv("SYNC_TOKEN") + link, err := publish(ctx, github.New(token), "spin-stack/website", "https://github.com/spin-stack/website", *bot, token, ms, prBody(ms, changed, *reference, today())) + if err != nil { + return err + } + fmt.Println(link) + return nil + case "issue": + gh := github.New(os.Getenv("SOURCES_TOKEN")) + fds, err := feeds(*docs, *figures) + if err != nil { + return err + } + text, err := issueText(ctx, gh, *mode, was, now, fds) + if err != nil { + return err + } + texts := map[string]string{*mode: text, "chain": chain(ctx, gh)} + if *dry { + fmt.Print(withSection(withSection("", *mode, text), "chain", texts["chain"])) + return nil + } + link, err := writeIssue(ctx, gh, "spin-stack/spin", *bot, texts) + if err != nil { + return err + } + fmt.Println(link) + return nil + } + return fmt.Errorf("%s is neither pr nor issue", argv[0]) +} + +// issueText is the issue's section for mode: the commits the bump took into the site, or those +// spin main has past the pin, each digest dated so a reader sees how old it is. +func issueText(ctx context.Context, gh *github.Client, mode string, was, now *versions.Versions, fds []feed) (string, error) { + from, err := was.Get("spin") + if err != nil { + return "", err + } + to, err := now.Get("spin") + if err != nil { + return "", err + } + var head, base, intro string + switch mode { + case "release": + if from.Pin == to.Pin { + return fmt.Sprintf("%s: the pin stays at %s: no release newer than it is one the site can move to.\n", today(), at(to.Version, to.Pin)), nil + } + base, head = from.Pin, to.Pin + intro = fmt.Sprintf("%s: the sync pull request moves spin from %s to %s.", today(), at(from.Version, from.Pin), at(to.Version, to.Pin)) + case "main": + base, head = to.Pin, "main" + intro = fmt.Sprintf("%s: the site documents spin %s; this is what main has past it, for the next release to bring.", today(), at(to.Version, to.Pin)) + default: + return "", fmt.Errorf("-mode %q is neither release nor main", mode) + } + cs, total, err := commits(ctx, gh, "spin-stack/spin", base, head) + if err != nil { + return "", err + } + return intro + "\n\n" + digest(cs, total, fds), nil +} + +func today() string { return time.Now().UTC().Format(time.DateOnly) } diff --git a/tools/sync/pr.go b/tools/sync/pr.go new file mode 100644 index 0000000..91dd310 --- /dev/null +++ b/tools/sync/pr.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "strings" + + "github.com/spin-stack/go-tools/versions" + + "github.com/spin-stack/website/tools/internal/github" +) + +// branch is the one branch, and so the one pull request, the sync writes: a run force-pushes it, +// and the pull request open from it is updated rather than another opened beside it. +const branch = "sync/sources" + +// move is a pin that changed between main and the bump. +type move struct { + Name string + FromV, FromPin string + ToV, ToPin string +} + +// moves is every entry whose version or pin differs from base to now, in now's order. +func moves(base, now *versions.Versions) []move { + var out []move + for _, e := range now.Entries { + was, err := base.Get(e.Name) + if err == nil && was.Version == e.Version && was.Pin == e.Pin { + continue + } + out = append(out, move{e.Name, was.Version, was.Pin, e.Version, e.Pin}) + } + return out +} + +// prBody is the sync pull request's description. The website is public and spin private, so it +// says which pins moved and which files changed, and nothing of what spin's commits say: that is +// in spin's issue, where only those who read spin read it. +func prBody(ms []move, changed []string, reference, today string) string { + var b strings.Builder + fmt.Fprintf(&b, "The sources moved to their newest releases, and the reference regenerated from them, on %s.\n\n", today) + b.WriteString("| source | from | to |\n|---|---|---|\n") + for _, m := range ms { + fmt.Fprintf(&b, "| %s | %s | %s |\n", m.Name, at(m.FromV, m.FromPin), at(m.ToV, m.ToPin)) + } + b.WriteString("\n") + switch reference { + case "success": + b.WriteString("`task reference` wrote the files below and found nothing on the hand-written pages that the sources do not have.\n") + default: + fmt.Fprintf(&b, "`task reference` did not pass (%s): the hand-written pages say something the sources no longer do. Gate's `task check` names each line; fix them on this branch.\n", reference) + } + fmt.Fprintf(&b, "\n
%d files changed\n\n", len(changed)) + for _, f := range changed { + fmt.Fprintf(&b, "- `%s`\n", f) + } + b.WriteString("\n
\n\n") + fmt.Fprintf(&b, "What spin's commits since the last pin change, grouped by the pages each feeds, is in spin's issue %q, beside the release chain. ", issueTitle) + b.WriteString("Read the pages they feed against them: what is generated is here, what is written by hand is a person's to change. ") + b.WriteString("Nothing merges this on its own; the next run writes over this branch.\n") + return b.String() +} + +func at(version, pin string) string { + if version == "" { + return "-" + } + return fmt.Sprintf("%s `%s`", version, short(pin)) +} + +// changedFiles is every path git sees changed or added in the work tree. +func changedFiles(ctx context.Context) ([]string, error) { + out, err := gitOut(ctx, nil, "status", "--porcelain", "-z", "--untracked-files=all") + if err != nil { + return nil, err + } + var files []string + for _, rec := range strings.Split(strings.TrimSuffix(out, "\x00"), "\x00") { + if len(rec) > 3 { + files = append(files, rec[3:]) + } + } + return files, nil +} + +// commitMessage is the sync commit's, in the repository's voice: what moved. +func commitMessage(ms []move) string { + var parts []string + for _, m := range ms { + parts = append(parts, fmt.Sprintf("%s %s", m.Name, at(m.ToV, m.ToPin))) + } + return "sources: " + strings.ReplaceAll(strings.Join(parts, ", "), "`", "") + "\n\nWritten by sync.yml: versions.yaml bumped by task sources:bump, the reference by task reference.\n" +} + +// publish commits the work tree as bot, force-pushes it to branch of remote, and opens the pull request +// from it, or updates the one open. +func publish(ctx context.Context, gh *github.Client, repo, remote, bot, token string, ms []move, body string) (string, error) { + var user struct { + ID int `json:"id"` + } + if err := gh.Get(ctx, "/users/"+url.PathEscape(bot), &user); err != nil { + return "", err + } + id := []string{ + "GIT_AUTHOR_NAME=" + bot, "GIT_COMMITTER_NAME=" + bot, + fmt.Sprintf("GIT_AUTHOR_EMAIL=%d+%s@users.noreply.github.com", user.ID, bot), + fmt.Sprintf("GIT_COMMITTER_EMAIL=%d+%s@users.noreply.github.com", user.ID, bot), + } + for _, args := range [][]string{ + {"switch", "-q", "-C", branch}, + {"add", "-A"}, + {"commit", "-q", "-m", commitMessage(ms)}, + } { + if _, err := gitOut(ctx, id, args...); err != nil { + return "", err + } + } + if _, err := gitOut(ctx, github.GitAuth(token), "push", "-q", "--force", remote, "HEAD:refs/heads/"+branch); err != nil { + return "", err + } + owner, _, _ := strings.Cut(repo, "/") + var open []struct { + Number int `json:"number"` + URL string `json:"html_url"` + } + q := url.Values{"head": {owner + ":" + branch}, "state": {"open"}} + if err := gh.Get(ctx, "/repos/"+repo+"/pulls?"+q.Encode(), &open); err != nil { + return "", err + } + title := strings.TrimSuffix(strings.SplitN(commitMessage(ms), "\n", 2)[0], ".") + var pr struct { + URL string `json:"html_url"` + } + if len(open) > 0 { + err := gh.Do(ctx, http.MethodPatch, fmt.Sprintf("/repos/%s/pulls/%d", repo, open[0].Number), map[string]string{"title": title, "body": body}, &pr) + return pr.URL, err + } + err := gh.Do(ctx, http.MethodPost, "/repos/"+repo+"/pulls", map[string]string{"title": title, "body": body, "head": branch, "base": "main"}, &pr) + return pr.URL, err +} + +// gitOut runs git here with env added, its output untrimmed. It never asks for a password. +func gitOut(ctx context.Context, env []string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(append(os.Environ(), "GIT_TERMINAL_PROMPT=0"), env...) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) + } + return string(out), nil +} diff --git a/tools/sync/sync_test.go b/tools/sync/sync_test.go new file mode 100644 index 0000000..e928fcd --- /dev/null +++ b/tools/sync/sync_test.go @@ -0,0 +1,324 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/spin-stack/go-tools/versions" + + "github.com/spin-stack/website/tools/internal/github" +) + +const ( + oldPin = "77380b13da641c24e3f79b9c7586c39888f4e7ba" + newPin = "b3598142dbcd6ef24135066332441a400dd35980" +) + +func write(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +// api serves routes, each a path and its query as GitHub is asked them, and fails the test on +// anything else: a request the code was not meant to make is one to know about. +func api(t *testing.T, routes map[string]string) *github.Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := routes[r.URL.RequestURI()] + if !ok { + t.Errorf("unexpected request %s %s", r.Method, r.URL.RequestURI()) + http.NotFound(w, r) + return + } + if body == "404" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return &github.Client{Base: srv.URL, HTTP: srv.Client()} +} + +// The digest puts each commit under every path of spin it touches that feeds a page, with the +// pages, and counts the commits that feed nothing rather than listing them. +func TestDigestGroupsCommitsByThePagesTheyFeed(t *testing.T) { + dir := t.TempDir() + docs := filepath.Join(dir, "docs") + write(t, filepath.Join(docs, "reference/cli/workspace.md"), "---\ntitle: \"spin workspace\"\nsource: \"spin:cmd/cli\"\n---\n") + write(t, filepath.Join(docs, "reference/api/index.md"), "---\nsource: \"spin:internal/controlplane/apidocs/spin.v1.openapi.json\"\n---\n") + write(t, filepath.Join(docs, "concepts/network.mdx"), "Some words.\n\n
\n") + figures := filepath.Join(dir, "figures.json") + write(t, figures, `{"datapath": {"source": "https://github.com/spin-stack/spin/blob/`+oldPin+`/docs/network/datapath"}}`) + fds, err := feeds(docs, figures) + if err != nil { + t.Fatal(err) + } + + gh := api(t, map[string]string{ + "/repos/spin-stack/spin/compare/" + oldPin + "...main?per_page=100&page=1": `{"total_commits": 3, "commits": [ + {"sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "commit": {"message": "cli: spin open\n\nbody"}}, + {"sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "commit": {"message": "network: the datapath drawn again"}}, + {"sha": "cccccccccccccccccccccccccccccccccccccccc", "commit": {"message": "runner: a fix"}}]}`, + "/repos/spin-stack/spin/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": `{"files": [{"filename": "cmd/cli/cmd/open.go"}, {"filename": "hack/env-catalog.txt"}]}`, + "/repos/spin-stack/spin/commits/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": `{"files": [{"filename": "docs/network/datapath.dot"}]}`, + "/repos/spin-stack/spin/commits/cccccccccccccccccccccccccccccccccccccccc": `{"files": [{"filename": "internal/runner/x.go"}]}`, + }) + cs, total, err := commits(context.Background(), gh, "spin-stack/spin", oldPin, "main") + if err != nil { + t.Fatal(err) + } + got := digest(cs, total, fds) + for _, want := range []string{ + "3 commits.", + "#### `cmd/cli`\n\nFeeds: reference/cli/workspace\n\n- aaaaaaaaaa cli: spin open\n", + "#### `hack/env-catalog.txt`\n\nFeeds: every page that names a SPIN_ variable\n\n- aaaaaaaaaa cli: spin open\n", + "#### `docs/network/datapath`\n\nFeeds: concepts/network\n\n- bbbbbbbbbb network: the datapath drawn again\n", + "1 of them touch no path the site reads.", + } { + if !strings.Contains(got, want) { + t.Errorf("digest lacks %q:\n%s", want, got) + } + } + if strings.Contains(got, "runner: a fix") || strings.Contains(got, "apidocs") { + t.Errorf("digest lists a commit or a path that feeds nothing it touched:\n%s", got) + } +} + +// A directory of more than three pages is said once, with how many: the CLI feeds a page per +// command. +func TestFeedsSayADirectoryOfManyPagesOnce(t *testing.T) { + got := collapse([]string{"reference/cli/b", "reference/cli", "reference/cli/a", "reference/cli/c", "reference/cli/d", "reference/status", "reference/cli/a"}) + want := []string{"reference/cli", "reference/cli/ (4 pages)", "reference/status"} + if !slices.Equal(got, want) { + t.Errorf("collapse = %q; want %q", got, want) + } +} + +// The chain report reads each link, and says of one it cannot read that it could not, instead +// of giving up the rest. +func TestChainReportsEachLinkAndWhatItCouldNotRead(t *testing.T) { + gh := api(t, map[string]string{ + "/repos/spin-stack/spin/contents/versions.yaml?ref=main": `- name: spin-machine + kind: download + source: https://github.com/spin-stack/spin-machine/releases/download/{version}/spin-machine-{version}-linux-x86_64.tar.gz + version: v20261014.01 + pin: 10b1685e82b8b3e34785e0a7a17d618bac711aa9edee5f13856d1586e345a44e + track: github-release https://github.com/spin-stack/spin-machine +`, + "/repos/spin-stack/spin-machine/releases/latest": `{"tag_name": "v20261023.01"}`, + "/repos/spin-stack/spin/actions/workflows/machine.yml/runs?per_page=1": `{"workflow_runs": [{"created_at": "2026-10-14T09:00:00Z", "status": "completed", "conclusion": "success", "html_url": "https://github.com/spin-stack/spin/actions/runs/1"}]}`, + "/repos/spin-stack/spin-terraform-aws/actions/workflows/propose.yml/runs?per_page=1": `{"workflow_runs": []}`, + "/repos/spin-stack/spin-terraform-aws/actions/workflows/fresh-install.yml/runs?per_page=1": "404", + "/repos/spin-stack/spinbox.dev-deploy/contents/terraform.tfvars?ref=main": "spin_version = \"v20260928.07\"\ndomain = \"spinbox.dev\"\n", + "/repos/spin-stack/spinbox.dev-deploy/contents/main.tf?ref=main": "module \"spin\" {\n source = \"github.com/spin-stack/spin-terraform-aws?ref=1d7e2a2\"\n}\n", + }) + got := chain(context.Background(), gh) + for _, want := range []string{ + "| spin-machine | spin main pins v20261014.01; the newest release is **v20261023.01** |", + "| `machine.yml` a spin-machine release proposed to spin | last ran 2026-10-14, [success](https://github.com/spin-stack/spin/actions/runs/1) |", + "| `propose.yml` the module proposed to the deployments | never ran |", + "| `fresh-install.yml` an installation from nothing, weekly | could not read: GET /repos/spin-stack/spin-terraform-aws/actions/workflows/fresh-install.yml/runs?per_page=1: 404 Not Found", + "| spinbox.dev | runs v20260928.07, with the module at `1d7e2a2` |", + } { + if !strings.Contains(got, want) { + t.Errorf("chain report lacks %q:\n%s", want, got) + } + } +} + +// issues is spin's issues as the sync sees them: the bot's, and the requests that write them. +type issues struct { + mu sync.Mutex + byNum map[int]map[string]any + created int +} + +func (s *issues) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/spin-stack/spin/issues": + if r.URL.Query().Get("creator") != "sources[bot]" || r.URL.Query().Get("state") != "open" { + t.Errorf("issues listed by %q", r.URL.RawQuery) + } + list := []map[string]any{{"number": 7, "title": "Something else", "body": "x"}} + for _, i := range s.byNum { + list = append(list, i) + } + _ = json.NewEncoder(w).Encode(list) + case r.Method == http.MethodPost && r.URL.Path == "/repos/spin-stack/spin/issues": + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + s.created++ + n := 100 + s.created + in["number"], in["html_url"] = n, fmt.Sprintf("https://github.com/spin-stack/spin/issues/%d", n) + s.byNum[n] = in + _ = json.NewEncoder(w).Encode(in) + case r.Method == http.MethodPatch && strings.HasPrefix(r.URL.Path, "/repos/spin-stack/spin/issues/"): + var n int + _, _ = fmt.Sscanf(strings.TrimPrefix(r.URL.Path, "/repos/spin-stack/spin/issues/"), "%d", &n) + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + s.byNum[n]["body"] = in["body"] + _ = json.NewEncoder(w).Encode(s.byNum[n]) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.RequestURI()) + http.NotFound(w, r) + } + } +} + +// One issue: the first write opens it, the next finds it and writes its own section, keeping the +// others. +func TestTheIssueIsOneAndEachWriteKeepsTheOtherSections(t *testing.T) { + st := &issues{byNum: map[int]map[string]any{}} + srv := httptest.NewServer(st.handler(t)) + defer srv.Close() + gh := &github.Client{Base: srv.URL, HTTP: srv.Client()} + ctx := context.Background() + + if _, err := writeIssue(ctx, gh, "spin-stack/spin", "sources[bot]", map[string]string{"release": "moved to v20261025.01", "chain": "chain one"}); err != nil { + t.Fatal(err) + } + link, err := writeIssue(ctx, gh, "spin-stack/spin", "sources[bot]", map[string]string{"main": "main is 40 past", "chain": "chain two"}) + if err != nil { + t.Fatal(err) + } + if st.created != 1 || len(st.byNum) != 1 { + t.Fatalf("%d issues opened; want one", st.created) + } + if link != "https://github.com/spin-stack/spin/issues/101" { + t.Errorf("link = %s", link) + } + body := st.byNum[101]["body"].(string) + if st.byNum[101]["title"] != issueTitle { + t.Errorf("title = %v", st.byNum[101]["title"]) + } + for _, want := range []string{"moved to v20261025.01", "main is 40 past", "chain two"} { + if !strings.Contains(body, want) { + t.Errorf("the body lacks %q:\n%s", want, body) + } + } + if strings.Contains(body, "chain one") || strings.Index(body, "## The last release sync") > strings.Index(body, "## spin main past the pin") { + t.Errorf("a section was kept that was written over, or the order moved:\n%s", body) + } +} + +func pins(t *testing.T, spin, brand string) *versions.Versions { + t.Helper() + v, err := versions.Parse([]byte(fmt.Sprintf(`- name: spin + kind: git + source: https://github.com/spin-stack/spin + version: %s + pin: %s + track: github-release +- name: brand + kind: git + source: https://github.com/spin-stack/brand + version: main + pin: %s + track: branch +`, map[string]string{oldPin: "main", newPin: "v20261025.01"}[spin], spin, brand))) + if err != nil { + t.Fatal(err) + } + return v +} + +// The public pull request names the pins that moved and the files that changed, and nothing of +// what spin's commits say. +func TestThePullRequestSaysWhatMovedAndNothingOfSpinsCommits(t *testing.T) { + brand := "c882b6f0f3901787525ef16ef29058c9c1418cbe" + ms := moves(pins(t, oldPin, brand), pins(t, newPin, brand)) + if len(ms) != 1 || ms[0].Name != "spin" || ms[0].FromV != "main" || ms[0].ToV != "v20261025.01" { + t.Fatalf("moves = %+v; want spin alone, from main to v20261025.01", ms) + } + body := prBody(ms, []string{"versions.yaml", "src/data/reference.json"}, "failure", "2026-10-25") + for _, want := range []string{ + "| spin | main `77380b13da` | v20261025.01 `b3598142db` |", + "- `src/data/reference.json`", + "`task reference` did not pass (failure)", + "on 2026-10-25", + `spin's issue "Site vs spin main"`, + } { + if !strings.Contains(body, want) { + t.Errorf("the body lacks %q:\n%s", want, body) + } + } + if got := commitMessage(ms); !strings.HasPrefix(got, "sources: spin v20261025.01 b3598142db\n") { + t.Errorf("commit message = %q", got) + } +} + +// The sync commits as the App's bot, force-pushes the one branch, and updates the pull request +// open from it rather than opening a second. +func TestPublishPushesTheOneBranchAndUpdatesItsPullRequest(t *testing.T) { + dir := t.TempDir() + work, bare := filepath.Join(dir, "work"), filepath.Join(dir, "bare.git") + git := func(d string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", d}, args...)...) + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write(t, filepath.Join(work, "versions.yaml"), "old\n") + git(dir, "init", "-q", "--bare", bare) + git(work, "init", "-q", "-b", "main") + git(work, "add", "-A") + git(work, "commit", "-q", "-m", "init") + write(t, filepath.Join(work, "versions.yaml"), "new\n") + t.Chdir(work) + + var patched map[string]string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method + " " + r.URL.RequestURI() { + case "GET /users/sync%5Bbot%5D": + _, _ = w.Write([]byte(`{"id": 42}`)) + case "GET /repos/spin-stack/website/pulls?head=spin-stack%3Async%2Fsources&state=open": + _, _ = w.Write([]byte(`[{"number": 9, "html_url": "https://github.com/spin-stack/website/pull/9"}]`)) + case "PATCH /repos/spin-stack/website/pulls/9": + _ = json.NewDecoder(r.Body).Decode(&patched) + _, _ = w.Write([]byte(`{"html_url": "https://github.com/spin-stack/website/pull/9"}`)) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.RequestURI()) + http.NotFound(w, r) + } + })) + defer srv.Close() + gh := &github.Client{Base: srv.URL, HTTP: srv.Client()} + ms := []move{{Name: "brand", FromV: "main", FromPin: oldPin, ToV: "main", ToPin: newPin}} + link, err := publish(context.Background(), gh, "spin-stack/website", bare, "sync[bot]", "", ms, "the body") + if err != nil { + t.Fatal(err) + } + if link != "https://github.com/spin-stack/website/pull/9" || patched["body"] != "the body" || patched["title"] != "sources: brand main b3598142db" { + t.Errorf("publish = %s, patched %v", link, patched) + } + if got := git(bare, "show", "sync/sources:versions.yaml"); got != "new" { + t.Errorf("the branch has versions.yaml %q; want the bumped one", got) + } + if got := git(bare, "log", "-1", "--format=%an <%ae>", "sync/sources"); got != "sync[bot] <42+sync[bot]@users.noreply.github.com>" { + t.Errorf("the commit is %s's; want the App's bot", got) + } +} diff --git a/versions.yaml b/versions.yaml new file mode 100644 index 0000000..71adfcf --- /dev/null +++ b/versions.yaml @@ -0,0 +1,36 @@ +# What the site is generated from, each written once: `task sources` fetches every pin into +# .sources/, refgen fails when src/data/reference.json records another commit, and +# `task sources:bump` moves them to the newest releases, which .github/workflows/sync.yml does +# every day for a person to review. It is go-tools' versions format, held by +# tools/sources/repo_test.go as any repository's is. + +- name: spin + kind: git + source: https://github.com/spin-stack/spin + version: main + pin: 77380b13da641c24e3f79b9c7586c39888f4e7ba + track: github-release + note: >- + A commit of main, not a release: v20261019.02, the newest, has no feature board, and the site + documents the release cut from main after it. A bump moves to a release only once one holds + this commit, never back to an older one. Private: fetched with the sources App's token. + +- name: spin-terraform-aws + kind: git + source: https://github.com/spin-stack/spin-terraform-aws + version: v20261024.01 + pin: 3df623f0ca2b7f91650e0ac857ba0bde295e0481 + track: github-release + note: >- + The module the AWS pages describe and pin, and images.json, the releases an installation can + run. Fetched with its tags: a page names the newest release the commit holds. + +- name: brand + kind: git + source: https://github.com/spin-stack/brand + version: main + pin: c882b6f0f3901787525ef16ef29058c9c1418cbe + track: branch + note: >- + The palette, the logos and the screenshots. brand cuts no releases, so its main is what it + has to say; the screenshots are compared at the release they were taken of.