release.yml

name: Release

# Publish the release assets of this repository. For each port of scripts/ports:
#
#   pacman-static-<arch>, .json   a static pacman and its evidence file
#   rootfs-<arch>.tar.gz          the filesystem of the published image
#   rootfs-<arch>.json            the evidence file of that image
#   bootstrap-set-<arch>.txt      its package set: name version sha256
#   oci-<arch>.tar.gz             the image, for docker load or podman load
#
# and manifest.json (every published tag) and SHA256SUMS (every asset).
#
# Triggers:
#   push of a v* tag   publish that tag
#   schedule           day 1 of each month: publish the next patch tag after
#                      the newest vX.Y.Z tag. The release makes the tag.
#   workflow_dispatch  build the assets and stop. Nothing is published.
#
# ⭐ A run is idempotent. A new release starts as a draft at the commit of the
# run. Each asset is uploaded with --clobber, then the draft is published,
# which makes the tag. A second run at the same commit finishes the draft. A
# draft at another commit is deleted and made again at the commit of the run.
#
# ⛔ A run never changes a published release. To make one again, delete it
# first: gh release delete TAG.
#
# ⛔ A port with no published image is left out, and the notes name it. Any
# other failure publishes nothing.
#
# ⚠ A git tag (vX.Y.Z) is a version of the tooling and the assets. Image tags
# are dates. The two namespaces are separate.
on:
  push:
    tags:
      - "v*"
  schedule:
    - cron: "15 6 1 * *"
  workflow_dispatch:

defaults:
  run:
    shell: bash

permissions: {}

concurrency:
  group: release-${{ github.ref }}
  cancel-in-progress: false

jobs:
  #--------------------------------------------------------------------------#
  # ⛔ First. Choose the tag, refuse a tag that is not on the default branch,
  # and run the static suite before any build.
  #--------------------------------------------------------------------------#
  guard:
    name: Check the tag
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    permissions:
      contents: read
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      publish: ${{ steps.tag.outputs.publish }}
      matrix: ${{ steps.ports.outputs.matrix }}
    steps:
      # fetch-depth 0: the tag check reads the history and every tag.
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Choose the tag
        id: tag
        env:
          EVENT: ${{ github.event_name }}
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          case "$EVENT" in
            push)
              tag="$GITHUB_REF_NAME"
              default="$(timeout -k 5 60 gh api "repos/$GITHUB_REPOSITORY" --jq .default_branch)"
              if ! git merge-base --is-ancestor HEAD "origin/$default"; then
                echo "::error::$tag is not on origin/$default. A release builds only from the default branch."
                echo "reproduce: git merge-base --is-ancestor $tag origin/$default"
                exit 1
              fi
              publish=true
              ;;
            schedule)
              # A draft has no git tag, so a later run gets the same tag. The
              # publish job deletes a draft at another commit and makes it
              # again.
              tag="$(git tag -l 'v*' | scripts/release-notes --next-tag)"
              publish=true
              ;;
            *)
              tag=""
              publish=false
              ;;
          esac
          echo "tag=$tag" >> "$GITHUB_OUTPUT"
          echo "publish=$publish" >> "$GITHUB_OUTPUT"
          echo "::notice::tag $tag, publish $publish"

      - name: Read the port table
        id: ports
        run: |
          set -euo pipefail
          echo "matrix=$(scripts/ports matrix)" >> "$GITHUB_OUTPUT"

      - name: Run the static suite
        id: static
        run: tests/run.sh -j 4 static

  #--------------------------------------------------------------------------#
  # The static pacman binaries, from the workflow that owns the recipe.
  #--------------------------------------------------------------------------#
  pacman:
    name: Static pacman
    needs: guard
    uses: ./.github/workflows/pacman-static.yml
    permissions:
      contents: read

  #--------------------------------------------------------------------------#
  # One job per port: take four assets out of the published image.
  #
  # ⛔ The published image, never a new build: the assets are the bytes that
  # consumers pull, and the digest is the digest of the registry.
  #--------------------------------------------------------------------------#
  rootfs:
    name: Rootfs ${{ matrix.arch }}
    needs: guard
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.guard.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # GHCR <first alias> is the rolling tag of the port. 404 means that the
      # port has no published image: the port is left out. Any other answer
      # that is not 200 stops the job.
      - name: Find the published image
        id: image
        env:
          ARCH: ${{ matrix.arch }}
          OWNER: ${{ github.repository_owner }}
        run: |
          set -euo pipefail
          PROG=release
          . scripts/lib.sh
          mkdir -p assets meta
          alias="$(scripts/ports aliases "$ARCH" | awk 'NR == 1')"
          repo="$(printf '%s' "$OWNER" | tr '[:upper:]' '[:lower:]')/archlinux"
          rc=0
          res="$(reg_head ghcr.io "$repo" "$alias")" || rc=$?
          [ "$rc" -eq 0 ] || die "no answer from ghcr.io for $repo:$alias" \
            "reproduce: curl -sSI --connect-timeout 15 --max-time 60 https://ghcr.io/v2/$repo/manifests/$alias"
          read -r code digest <<< "$res"
          case "$code" in
            200)
              echo "state=published" >> "$GITHUB_OUTPUT"
              echo "image=ghcr.io/$repo@$digest" >> "$GITHUB_OUTPUT"
              echo "digest=$digest" >> "$GITHUB_OUTPUT"
              echo "::notice::$ARCH is ghcr.io/$repo:$alias at $digest"
              ;;
            404)
              echo "state=absent" >> "$GITHUB_OUTPUT"
              jq -n --arg arch "$ARCH" --arg reason "ghcr.io/$repo:$alias does not exist" \
                '{arch: $arch, reason: $reason}' > "meta/absent-$ARCH.json"
              echo "::warning::$ARCH has no published image at ghcr.io/$repo:$alias. The release leaves it out."
              ;;
            *)
              die "ghcr.io answered HTTP $code for $repo:$alias" \
                "only 404 means that no image is published" \
                "reproduce: curl -sSI --connect-timeout 15 --max-time 60 https://ghcr.io/v2/$repo/manifests/$alias"
              ;;
          esac

      - name: Register the emulator
        if: steps.image.outputs.state == 'published'
        env:
          ARCH: ${{ matrix.arch }}
        run: scripts/binfmt-register "$ARCH"

      # --platform is necessary: the pushed image is an index with an
      # attestation, and a pull with no platform picks the runner's.
      - name: Pull the published image
        if: steps.image.outputs.state == 'published'
        env:
          IMAGE: ${{ steps.image.outputs.image }}
          PLATFORM: ${{ matrix.platform }}
        run: timeout -k 10 900 docker pull --platform "$PLATFORM" "$IMAGE"

      # The evidence file of the build comes from the status branch when its
      # digest is the published digest. Else gen-evidence makes it again from
      # the current databases, and the notes say so. ⚠ That can fail when the
      # databases moved after the build.
      #
      # ⛔ CONTAINER_RUNTIME=docker: the image is in the docker store, and
      # gen-evidence picks podman when it finds one.
      - name: Get the evidence file
        id: evidence
        if: steps.image.outputs.state == 'published'
        env:
          ARCH: ${{ matrix.arch }}
          IMAGE: ${{ steps.image.outputs.image }}
          DIGEST: ${{ steps.image.outputs.digest }}
          PLATFORM: ${{ matrix.platform }}
          CONTAINER_RUNTIME: docker
        run: |
          set -euo pipefail
          ev="assets/rootfs-$ARCH.json"
          source=""
          reason=""
          rc=0
          timeout -k 5 120 git fetch -q --depth 1 origin status || rc=$?
          if [ "$rc" -ne 0 ]; then
            reason="git fetch origin status exited $rc"
          elif ! git cat-file -e "FETCH_HEAD:evidence/$ARCH.json"; then
            reason="the status branch has no evidence/$ARCH.json"
          else
            git show "FETCH_HEAD:evidence/$ARCH.json" > "$ev"
            got="$(jq -r '.digest // "-"' "$ev")"
            if [ "$got" = "$DIGEST" ]; then
              source=status-branch
            else
              reason="evidence/$ARCH.json is for $got, the published digest is $DIGEST"
              rm -f "$ev"
            fi
          fi
          if [ -z "$source" ]; then
            echo "::warning::$ARCH: $reason. Making the evidence again without a database snapshot."
            # ⛔ No SOURCE_COMMIT and no BUILD_DATE: gen-evidence then reads
            # them from the labels of the image. The commit and the time of
            # this run are not the build of the image.
            env -u SOURCE_COMMIT -u BUILD_DATE scripts/gen-evidence "$ARCH" "$IMAGE" "$PLATFORM" "$ev"
            source=regenerated
          fi
          jq -n --arg arch "$ARCH" --arg image "$IMAGE" --arg digest "$DIGEST" \
            --arg evidence "$source" --arg reason "$reason" \
            '{arch: $arch, image: $image, digest: $digest, evidence: $evidence, reason: $reason}' \
            > "meta/rootfs-$ARCH.json"
          echo "::notice::$ARCH evidence: $source"

      - name: Test the image
        if: steps.image.outputs.state == 'published'
        env:
          ARCH: ${{ matrix.arch }}
          IMAGE: ${{ steps.image.outputs.image }}
          PLATFORM: ${{ matrix.platform }}
        run: |
          set -euo pipefail
          IMAGE="$IMAGE" PLATFORM="$PLATFORM" CONTAINER_RUNTIME=docker \
            EVIDENCE="assets/rootfs-$ARCH.json" tests/run.sh image

      # docker export writes the filesystem of a container. docker save writes
      # the image with its layers and metadata. Both are assets.
      - name: Export the rootfs and the OCI archive
        if: steps.image.outputs.state == 'published'
        env:
          ARCH: ${{ matrix.arch }}
          IMAGE: ${{ steps.image.outputs.image }}
          PLATFORM: ${{ matrix.platform }}
        run: |
          set -euo pipefail
          cid="$(docker create --platform "$PLATFORM" "$IMAGE" true)"
          docker export "$cid" | gzip -9 > "assets/rootfs-$ARCH.tar.gz"
          docker rm -f "$cid" > /dev/null
          docker save "$IMAGE" | gzip -9 > "assets/oci-$ARCH.tar.gz"

          # ⛔ gzip -t reads the whole stream. A small or broken archive stops
          # the job.
          for f in "assets/rootfs-$ARCH.tar.gz" "assets/oci-$ARCH.tar.gz"; do
            gzip -t "$f"
            bytes="$(stat -c %s "$f")"
            if [ "$bytes" -lt 1000000 ]; then
              echo "::error::$f has $bytes bytes, too small for a root filesystem"
              exit 1
            fi
            echo "$f: $bytes bytes"
          done
          members="$(tar -tzf "assets/rootfs-$ARCH.tar.gz" | awk '$0 == "etc/os-release" || $0 == "./etc/os-release" { n++ } END { print n + 0 }')"
          if [ "$members" -eq 0 ]; then
            echo "::error::assets/rootfs-$ARCH.tar.gz has no etc/os-release"
            echo "reproduce: tar -tzf assets/rootfs-$ARCH.tar.gz | grep os-release"
            exit 1
          fi

      - name: Write the package set
        if: steps.image.outputs.state == 'published'
        env:
          ARCH: ${{ matrix.arch }}
        run: |
          set -euo pipefail
          scripts/gen-bootstrap-set "assets/rootfs-$ARCH.json" "assets/bootstrap-set-$ARCH.txt"
          awk 'NR <= 14' "assets/bootstrap-set-$ARCH.txt"

      - name: Upload the assets
        if: steps.image.outputs.state == 'published'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: rootfs-${{ matrix.arch }}
          path: assets/*
          if-no-files-found: error
          overwrite: true
          retention-days: 7

      - name: Upload the notes
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: meta-${{ matrix.arch }}
          path: meta/*
          if-no-files-found: error
          overwrite: true
          retention-days: 7

  #--------------------------------------------------------------------------#
  # Every published tag with its digest, read from the registry.
  #--------------------------------------------------------------------------#
  manifest:
    name: Manifest of tags
    needs: guard
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Generate
        env:
          OWNER: ${{ github.repository_owner }}
        run: |
          set -euo pipefail
          mkdir -p assets
          owner="$(printf '%s' "$OWNER" | tr '[:upper:]' '[:lower:]')"
          SOURCE_COMMIT="$GITHUB_SHA" scripts/gen-manifest "ghcr.io/$owner/archlinux" assets/manifest.json
          jq '{schema, image, tag_count, resolved_count, generated}' assets/manifest.json

      - name: Upload
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: manifest
          path: assets/*
          if-no-files-found: error
          overwrite: true
          retention-days: 7

  #--------------------------------------------------------------------------#
  # Publish. Only when guard chose to publish and every job above succeeded.
  # ⛔ contents: write is on this job only.
  #--------------------------------------------------------------------------#
  publish:
    name: Publish the release
    needs: [guard, pacman, rootfs, manifest]
    if: needs.guard.outputs.publish == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    permissions:
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # Each artifact in its own directory: dl/<artifact name>/.
      - name: Download every artifact
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          path: dl

      - name: Sort the assets and the notes
        run: |
          set -euo pipefail
          mkdir -p assets meta
          for d in dl/*/; do
            case "$(basename "$d")" in
              meta-*) cp -- "$d"* meta/ ;;
              *) cp -- "$d"* assets/ ;;
            esac
          done
          ls -l assets meta

      # ⛔ release-notes refuses a release with a hole. Each port of the port
      # table has every asset, or meta/absent-<arch>.json says it has no image.
      - name: Write the notes and SHA256SUMS
        id: notes
        env:
          TAG: ${{ needs.guard.outputs.tag }}
        run: |
          set -euo pipefail
          RELEASE_ARCHES="$(scripts/ports list | tr '\n' ' ')" \
            scripts/release-notes assets "$TAG" meta > body.md
          cat body.md >> "$GITHUB_STEP_SUMMARY"

      # A new release starts as a draft. A draft has no git tag. The tag
      # comes when the draft is published, after every asset is in place.
      #
      # The output state: "draft" when the draft has every asset of this run,
      # "published" when the release is published already. ⛔ Then nothing is
      # uploaded and the steps below do not run.
      #
      # ⛔ A draft at another commit is from an older run: its assets and its
      # target are not this run. It is deleted and made again. gh release
      # delete with no --cleanup-tag removes no git tag.
      - name: Publish
        id: publish
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ needs.guard.outputs.tag }}
          TARGET: ${{ github.sha }}
        run: |
          set -euo pipefail
          rc=0
          view="$(timeout -k 5 120 gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \
            --json isDraft,targetCommitish 2> view.err)" || rc=$?
          make=1
          if [ "$rc" -ne 0 ]; then
            if ! awk 'index($0, "release not found") { hit = 1 } END { exit hit ? 0 : 1 }' view.err; then
              echo "::error::gh release view $TAG exited $rc: $(tr '\n' ' ' < view.err)"
              exit 1
            fi
          else
            draft="$(jq -r '.isDraft' <<< "$view")"
            at="$(jq -r '.targetCommitish' <<< "$view")"
            if [ "$draft" != true ]; then
              echo "state=published" >> "$GITHUB_OUTPUT"
              echo "::notice::$TAG is published already, at $at. This run does not change it. To make it again, delete it first: gh release delete $TAG"
              exit 0
            fi
            if [ "$at" = "$TARGET" ]; then
              make=0
              echo "::notice::$TAG is a draft at $TARGET. This run finishes it."
            else
              echo "::warning::$TAG is a draft at $at, and this run is at $TARGET. Deleting the draft to make it again."
              timeout -k 5 120 gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes
            fi
          fi
          if [ "$make" -eq 1 ]; then
            timeout -k 5 300 gh release create "$TAG" --draft --repo "$GITHUB_REPOSITORY" \
              --target "$TARGET" --title "$TAG" --notes-file body.md
            echo "::notice::made the draft release $TAG at $TARGET"
          fi
          # ⛔ An asset that this run did not make is removed. SHA256SUMS
          # covers only the assets of this run.
          timeout -k 5 120 gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json assets \
            --jq '.assets[].name' > remote-names.txt
          while IFS= read -r name; do
            [ -n "$name" ] || continue
            if [ ! -f "assets/$name" ]; then
              echo "::notice::removing $name from the draft $TAG: this run did not make it"
              timeout -k 5 120 gh release delete-asset "$TAG" "$name" --repo "$GITHUB_REPOSITORY" --yes
            fi
          done < remote-names.txt
          for n in 1 2 3; do
            rc=0
            timeout -k 10 3600 gh release upload "$TAG" --repo "$GITHUB_REPOSITORY" --clobber assets/* || rc=$?
            [ "$rc" -eq 0 ] && break
            echo "::warning::gh release upload exited $rc (try $n of 3)"
            [ "$n" -lt 3 ] && sleep $((n * 30))
          done
          [ "$rc" -eq 0 ] || { echo "::error::the upload failed 3 times. Run the workflow again to finish $TAG."; exit 1; }
          timeout -k 5 300 gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --notes-file body.md
          echo "state=draft" >> "$GITHUB_OUTPUT"

      # ⛔ The draft has each local asset with the same size, and no other
      # asset, before it is published.
      - name: Check the uploaded assets
        id: verify
        if: steps.publish.outputs.state == 'draft'
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ needs.guard.outputs.tag }}
        run: |
          set -euo pipefail
          timeout -k 5 120 gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json assets \
            --jq '.assets[] | "\(.name) \(.size)"' > remote.txt
          missing=""
          for f in assets/*; do
            want="$(basename "$f") $(stat -c %s "$f")"
            if ! awk -v w="$want" '$0 == w { hit = 1 } END { exit hit ? 0 : 1 }' remote.txt; then
              missing="$missing $(basename "$f")"
            fi
          done
          if [ -n "$missing" ]; then
            echo "::error::these assets are missing on $TAG or have another size:$missing"
            echo "reproduce: gh release view $TAG --json assets"
            exit 1
          fi
          extra="$(awk 'NR == FNR { local[$1] = 1; next } !($1 in local) { printf "%s ", $1 }' \
            <(find assets -maxdepth 1 -type f -printf '%f\n') remote.txt)"
          if [ -n "$extra" ]; then
            echo "::error::$TAG also has assets that this run did not make: $extra"
            echo "reproduce: gh release view $TAG --json assets"
            exit 1
          fi
          echo "::notice::$(awk 'END { print NR }' remote.txt) assets on $TAG"

      - name: Publish the draft
        id: undraft
        if: steps.publish.outputs.state == 'draft'
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ needs.guard.outputs.tag }}
        run: |
          set -euo pipefail
          timeout -k 5 300 gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --draft=false
          echo "::notice::published https://github.com/$GITHUB_REPOSITORY/releases/tag/$TAG"

  #--------------------------------------------------------------------------#
  # ⭐ A scheduled run fires once a month, so one failure raises
  # failing-release. scripts/watch-schedules raises the same key only after 3
  # failed runs, and leaves it open while the newest run failed. A scheduled
  # run that succeeds clears it.
  #--------------------------------------------------------------------------#
  report:
    name: Report
    needs: [guard, pacman, rootfs, manifest, publish]
    if: ${{ always() && github.event_name == 'schedule' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    permissions:
      contents: read
      issues: write
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Raise or clear failing-release
        id: report
        env:
          GH_TOKEN: ${{ github.token }}
          NEEDS: ${{ toJSON(needs) }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -euo pipefail
          results="$(jq -r 'to_entries | map("\(.key) \(.value.result)") | join(", ")' <<< "$NEEDS")"
          failed="$(jq -r '[.[].result | select(. == "failure" or . == "cancelled")] | length' <<< "$NEEDS")"
          if [ "$failed" -eq 0 ]; then
            scripts/track-issue clear failing-release "The scheduled release run succeeded: $RUN_URL"
            exit 0
          fi
          body="$RUNNER_TEMP/failing-release.md"
          {
            echo "The scheduled release run failed: $RUN_URL"
            echo
            echo "Job results: $results"
            echo
            echo "reproduce: gh run view ${RUN_URL##*/} --log-failed"
          } > "$body"
          scripts/track-issue raise failing-release "release: the scheduled run failed" "$body"