.github: Several CI-related improvements

.github/workflows/cacerts-apply-patch.sh

@@ -6,11 +6,6 @@ source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
 
 prepare_git_repo
 
-if ! check_remote_branch "cacerts-${VERSION_NEW}-${TARGET_BRANCH}"; then
-    echo "remote branch already exists, nothing to do"
-    exit 0
-fi
-
 pushd "${SDK_OUTER_OVERLAY}"
 
 # Parse the Manifest file for already present source files and keep the latest version in the current series
@@ -32,7 +27,4 @@ generate_update_changelog 'ca-certificates' "${VERSION_NEW}" "${URL}" 'ca-certif
 
 commit_changes app-misc/ca-certificates "${VERSION_OLD}" "${VERSION_NEW}"
 
-cleanup_repo
-
-echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}"
-echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"
+push_changes_with_rebase

.github/workflows/cacerts-release.yaml

@@ -12,12 +12,11 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - name: Check out main scripts branch for GitHub workflow scripts only
+      - name: Check out scripts at ${{ github.ref }} for GHA scripts only
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: gha
-          ref: main
       - name: Figure out branch
         id: figure-out-branch
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
@@ -55,15 +54,3 @@ jobs:
           SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
           TARGET_BRANCH: ${{ steps.figure-out-branch.outputs.BRANCH }}
         run: gha/.github/workflows/cacerts-apply-patch.sh
-      - name: Create pull request
-        if: (steps.figure-out-branch.outputs.SKIP == 0) && (steps.apply-patch.outputs.UPDATE_NEEDED == 1)
-        uses: peter-evans/create-pull-request@v6
-        with:
-          token: ${{ secrets.BOT_PR_TOKEN }}
-          path: work
-          branch: "cacerts-${{ steps.nss-latest-release.outputs.NSS_VERSION }}-${{ steps.figure-out-branch.outputs.BRANCH }}"
-          base: ${{ steps.figure-out-branch.outputs.BRANCH }}
-          title: Update ca-certificates in ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.nss-latest-release.outputs.NSS_VERSION }}
-          body: Subject says it all.
-          labels: ${{ steps.figure-out-branch.outputs.LABEL }}
-          signoff: true

.github/workflows/ci.yaml

@@ -279,6 +279,30 @@ jobs:
           path: |
             scripts/image-changes-reports*.txt
 
+      - name: Post reports to PR
+        if: github.event.pull_request
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const globber = await glob.create('scripts/image-changes-reports*.txt');
+            let body = `### Image changes reports (${{ matrix.arch }})\n\n`;
+            for await (const file of globber.globGenerator()) {
+              const name = require('path').basename(file, '.txt');
+              const content = fs.readFileSync(file, 'utf8').trim();
+              if (content) {
+                body += `<details><summary>${name}</summary>\n\n\`\`\`diff\n${content}\n\`\`\`\n\n</details>\n\n`;
+              }
+            }
+            if (body.includes('<details>')) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: body,
+              });
+            }
+
       # Clean up what we uploaded already so the "vendor images" wildcard
       #  works when uploading artifacts in the next step.
       - name: Remove update, generic and devcontainer images

.github/workflows/common.sh

@@ -191,6 +191,16 @@ function commit_changes() {
   popd
 }
 
+# Push the changes to the current remote and branch. If this fails, clean up
+# (rebase needs this), pull, and rebase before trying again.
+function push_changes_with_rebase() {
+  if ! git -C "${SDK_OUTER_TOPDIR}" push; then
+    cleanup_repo
+    git -C "${SDK_OUTER_TOPDIR}" pull --rebase --depth=500
+    git -C "${SDK_OUTER_TOPDIR}" push
+  fi
+}
+
 # Prints the status of the git repo and cleans it up - reverts
 # uncommitted changes, removes untracked files. It's usually called at
 # the end of a script making changes to the repository in order to

.github/workflows/kernel-release.yaml

@@ -12,12 +12,11 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - name: Check out main scripts branch for GitHub workflow scripts only
+      - name: Check out scripts at ${{ github.ref }} for GHA scripts only
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: gha
-          ref: main
       - name: Figure out branch
         id: figure-out-branch
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
@@ -66,5 +65,5 @@ jobs:
           base: ${{ steps.figure-out-branch.outputs.BRANCH }}
           title: Upgrade Linux Kernel for ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.kernel-latest-release.outputs.KERNEL_VERSION }}
           body: Subject says it all.
-          labels: ${{ steps.figure-out-branch.outputs.LABEL }}
+          labels: ${{ steps.figure-out-branch.outputs.LABEL }},auto-ci
           signoff: true

.github/workflows/mantle-releases-main.yml

@@ -1,82 +1,52 @@
 name: Get the latest mantle release for branch
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
   schedule:
     - cron:  '0 7 * * 1'
 
 jobs:
   get-mantle-release:
     strategy:
       matrix:
-        branch: [main,alpha,beta,stable,lts,lts-old]
+        channel: [main,alpha,beta,stable,lts,lts-old]
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
+      - name: Check out scripts at ${{ github.ref }} for GHA scripts only
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: gha
       - name: Figure out branch
         id: figure-out-branch
-        run: |
-          set -euo pipefail # The line with major=$(curl | awk) requires pipefail for error handling
-          skip=0
-          branch=''
-          if [ ${{ matrix.branch }} = "main" ]; then
-            branch='main'
-          elif [[ ${{ matrix.branch }} = 'lts-old' ]]; then
-            curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 'https://lts.release.flatcar-linux.net/lts-info'
-            if [[ $(grep -e ':supported' lts-info | wc -l) -le 1 ]]; then
-              # Only one supported LTS, skip this workflow run
-              # as 'lts' matrix branch will handle updating the only
-              # supported LTS.
-              skip=1
-            else
-              major=$(grep -e ':supported' lts-info | sort -V | head -n 1 | awk -F: '{print $1}')
-              branch="flatcar-${major}"
-              # Drop this corner case when 2605 is not supported.
-              if [[ ${major} -eq 2605 ]]; then
-                branch='flatcar-lts-2605'
-              fi
-            fi
-            rm -f lts-info
-          else
-            major=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://${{ matrix.branch }}.release.flatcar-linux.net/amd64-usr/current/version.txt | awk -F= '/FLATCAR_BUILD=/{ print $2 }')
-            branch="flatcar-${major}"
-          fi
-          echo "BRANCH=${branch}" >>"${GITHUB_OUTPUT}"
-          echo "SKIP=${skip}" >>"${GITHUB_OUTPUT}"
-      - uses: actions/checkout@v4
-        if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
+        run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
+      - name: Check out work scripts branch for updating
+        if: steps.figure-out-branch.outputs.SKIP == 0
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          path: work
           ref: ${{ steps.figure-out-branch.outputs.BRANCH }}
       - name: Fetch latest mantle hash
-        if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
+        if: steps.figure-out-branch.outputs.SKIP == 0
         id: fetch-latest-mantle
         run: |
           set -euo pipefail
           commit=$(git ls-remote  https://github.com/flatcar/mantle refs/heads/main | cut -f1)
           echo "COMMIT=${commit}" >>"${GITHUB_OUTPUT}"
       - name: Try to apply patch
-        if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
+        if: steps.figure-out-branch.outputs.SKIP == 0
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/work"
         run: |
           set -euo pipefail
+          source gha/.github/workflows/common.sh
           set -x
+          cd "${WORK_SCRIPTS_DIR}"
           commit=${{ steps.fetch-latest-mantle.outputs.COMMIT }}
-          if ! grep -q "ghcr.io/flatcar/mantle:git-${commit}" sdk_container/.repo/manifests/mantle-container; then
-            echo "ghcr.io/flatcar/mantle:git-${commit}" > sdk_container/.repo/manifests/mantle-container
-            git add sdk_container/.repo/manifests/mantle-container
-          fi
-      - name: Create pull request for branch
-        if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
-        uses: peter-evans/create-pull-request@v6
-        with:
-          token: ${{ secrets.BOT_PR_TOKEN }}
-          base: ${{ steps.figure-out-branch.outputs.BRANCH }}
-          branch: mantle-update-${{ steps.figure-out-branch.outputs.BRANCH }}
-          author: Flatcar Buildbot <buildbot@flatcar-linux.org>
-          committer: Flatcar Buildbot <buildbot@flatcar-linux.org>
-          title: Upgrade mantle container image to latest HEAD in ${{ steps.figure-out-branch.outputs.BRANCH }}
-          commit-message: Update mantle container image to latest HEAD
-          delete-branch: true
-          signoff: true
+          file=sdk_container/.repo/manifests/mantle-container
+          echo "ghcr.io/flatcar/mantle:git-${commit}" > "${file}"
+          git status --porcelain "${file}" | grep -q "^ M" || exit 0
+          prepare_git_repo
+          git commit --signoff -m "Update mantle container image to latest HEAD" "${file}"
+          push_changes_with_rebase

.github/workflows/pr-workflows.yaml

@@ -10,40 +10,53 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pre_check:
-    name: "Check if we need to update the SDK"
+  approval_gate:
+    name: "Wait for approval"
     runs-on: ubuntu-latest
-    # Setting the environment is the more important reason we need this job.
-    # We use this job as a gate, so we can approve the PR workflow only once. If
+    # Use this job as a gate, so we can approve the PR workflow only once. If
     # we set this in the update_sdk job and in the build_image job, we would have
     # to approve the workflow for every job that kicks off. Given that the jobs
     # are sequenced, this is cumbersome. Use this job as a gate and make the rest
     # dependent on it.
-    environment: development
-    outputs:
-      sdk_changes: ${{ steps.step1.outputs.sdk_changes }}
+    #
+    # Skip the environment gate for auto-ci PRs from the bot to allow
+    # automatic triggering without manual approval.
+    environment: ${{ (github.actor != 'flatcar-infra' || !contains(github.event.pull_request.labels.*.name, 'auto-ci')) && 'development' || null }}
     steps:
-      - name: Set outputs
-        id: step1
-        shell: bash
-        run: |
-          echo "sdk_changes=${{ contains(github.event.pull_request.body, '/update-sdk') }}" >> $GITHUB_OUTPUT
+      - run: 'true'
 
   update_sdk:
     name: "Build an updated SDK container"
-    needs: [ pre_check ]
-    if: needs.pre_check.outputs.sdk_changes == 'true'
+    needs: [ approval_gate ]
+    if: "!contains(github.event.pull_request.labels.*.name, 'jenkins-ci') && contains(github.event.pull_request.body, '/update-sdk')"
     # SDK build needs access to bincache ssh secret
     secrets: inherit
     uses: ./.github/workflows/update-sdk.yaml
 
   build_image:
-    needs: [ update_sdk ]
+    needs: [ approval_gate, update_sdk ]
     # The update-sdk job may be skipped, which is fine. We only care if it tried to
     # run, but failed.
-    if: (always() && !cancelled()) && needs.update_sdk.result != 'failure'
+    if: "!contains(github.event.pull_request.labels.*.name, 'jenkins-ci') && always() && !cancelled() && needs.approval_gate.result == 'success' && needs.update_sdk.result != 'failure'"
     name: "Build the OS image"
     uses: ./.github/workflows/ci.yaml
     with:
       custom_sdk_version: ${{ needs.update_sdk.outputs.sdk_version }}
       image_formats: qemu_uefi pxe
+
+  jenkins_ci:
+    name: "Trigger Jenkins CI"
+    needs: [ approval_gate ]
+    if: contains(github.event.pull_request.labels.*.name, 'jenkins-ci')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger Jenkins build
+        run: |
+          curl -fsSL \
+            https://jenkins.flatcar.org/buildByToken/buildWithParameters \
+            --data-urlencode "token=${{ secrets.JENKINS_REMOTE_TRIGGER_TOKEN }}" \
+            --data-urlencode "job=container/packages_all_arches" \
+            --data-urlencode "cause=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --data-urlencode "version=main-9999.0.${{ github.run_number }}-github-${{ github.event.pull_request.number }}" \
+            --data-urlencode "scripts_ref=${{ github.event.pull_request.head.sha }}" \
+            --data-urlencode "git_sha=${{ github.event.pull_request.head.sha }}"