From f64d43dfa22e2d907f46a4003a784d1b88ed1ff3 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@flioz.org>
Date: Fri, 27 Feb 2026 07:21:38 +0000
Subject: [PATCH 1/8] feat: upload CI logs to S3

---
 .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6d811559..f9dbf154 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -309,6 +309,37 @@ jobs:
           echo "No container logs directory found"
         fi
 
+    # Upload state/latest directory to S3 for post-run inspection
+    # Path: s3://<CI_LOGS_BUCKET>/runs/<branch>/<run_id>/<run_attempt>/
+    # Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION secrets
+    # and CI_LOGS_BUCKET repository variable.
+    - name: "EXEC: {Install AWS CLI if missing}, independent"
+      if: always()
+      run: |
+        if command -v aws &>/dev/null; then
+          echo "AWS CLI already installed: $(aws --version)"
+        else
+          echo "AWS CLI not found, installing..."
+          curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+          unzip -q /tmp/awscliv2.zip -d /tmp/awscliv2
+          sudo /tmp/awscliv2/aws/install
+          rm -rf /tmp/awscliv2.zip /tmp/awscliv2
+          echo "AWS CLI installed: $(aws --version)"
+        fi
+
+    - name: "EXEC: {Upload state/latest to S3}, independent"
+      if: always()
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      run: |
+        BRANCH="${GITHUB_REF_NAME//\//-}"
+        S3_PATH="s3://${{ vars.CI_LOGS_BUCKET }}/runs/${BRANCH}/${{ github.run_id }}/${{ github.run_attempt }}/"
+        echo "Uploading ~/.foc-devnet/state/latest to ${S3_PATH}"
+        aws s3 sync ~/.foc-devnet/state/latest "${S3_PATH}" --no-progress
+        echo "Upload complete: ${S3_PATH}"
+
     # Verify cluster is running correctly
     - name: "EXEC: {Check cluster status}, independent"
       run: ./foc-devnet status

From f95cb2841c2b3baf4195eb62fa196f7a972c3739 Mon Sep 17 00:00:00 2001
From: Copilot <198982749+Copilot@users.noreply.github.com>
Date: Fri, 27 Feb 2026 13:12:06 +0530
Subject: [PATCH 2/8] Consolidate AWS CLI install and S3 upload into single
 conditional CI step (#67)

* Initial plan

* Consolidate AWS CLI install and S3 upload into a single conditional step

Co-authored-by: redpanda-f <181817029+redpanda-f@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: redpanda-f <181817029+redpanda-f@users.noreply.github.com>
---
 .github/workflows/ci.yml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f9dbf154..6ce43952 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -313,9 +313,18 @@ jobs:
     # Path: s3://<CI_LOGS_BUCKET>/runs/<branch>/<run_id>/<run_attempt>/
     # Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION secrets
     # and CI_LOGS_BUCKET repository variable.
-    - name: "EXEC: {Install AWS CLI if missing}, independent"
+    - name: "EXEC: {Install AWS CLI and upload state/latest to S3}, independent"
       if: always()
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+        CI_LOGS_BUCKET: ${{ vars.CI_LOGS_BUCKET }}
       run: |
+        if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_DEFAULT_REGION" ] || [ -z "$CI_LOGS_BUCKET" ]; then
+          echo "Skipping S3 upload: AWS credentials or CI_LOGS_BUCKET not configured"
+          exit 0
+        fi
         if command -v aws &>/dev/null; then
           echo "AWS CLI already installed: $(aws --version)"
         else
@@ -326,16 +335,8 @@ jobs:
           rm -rf /tmp/awscliv2.zip /tmp/awscliv2
           echo "AWS CLI installed: $(aws --version)"
         fi
-
-    - name: "EXEC: {Upload state/latest to S3}, independent"
-      if: always()
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-      run: |
         BRANCH="${GITHUB_REF_NAME//\//-}"
-        S3_PATH="s3://${{ vars.CI_LOGS_BUCKET }}/runs/${BRANCH}/${{ github.run_id }}/${{ github.run_attempt }}/"
+        S3_PATH="s3://${CI_LOGS_BUCKET}/runs/${BRANCH}/${{ github.run_id }}/${{ github.run_attempt }}/"
         echo "Uploading ~/.foc-devnet/state/latest to ${S3_PATH}"
         aws s3 sync ~/.foc-devnet/state/latest "${S3_PATH}" --no-progress
         echo "Upload complete: ${S3_PATH}"

From 40a97a0c52db7665e8593a8657248ffe64136916 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@flioz.org>
Date: Fri, 27 Feb 2026 07:32:03 +0000
Subject: [PATCH 3/8] feat: CI gets called every midnight

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6ce43952..aefcd230 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,6 +6,8 @@ on:
     branches: ['*']
   pull_request:
     branches: [main]
+  schedule:
+    - cron: '0 0 * * *' # Run every midnight
 
 jobs:
   fmt-clippy:

From d1499f2dcd9e9783e387c4ce693e6642d0864d19 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@filoz.org>
Date: Fri, 27 Feb 2026 08:34:57 +0000
Subject: [PATCH 4/8] fix: CI_LOGS_BUCKET uses secrets

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aefcd230..77fa08fe 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -321,7 +321,7 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        CI_LOGS_BUCKET: ${{ vars.CI_LOGS_BUCKET }}
+        CI_LOGS_BUCKET: ${{ secrets.CI_LOGS_BUCKET }}
       run: |
         if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_DEFAULT_REGION" ] || [ -z "$CI_LOGS_BUCKET" ]; then
           echo "Skipping S3 upload: AWS credentials or CI_LOGS_BUCKET not configured"

From 2a02de153af2dd721bcfc86d8d020707636463c8 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@filoz.org>
Date: Fri, 27 Feb 2026 08:59:34 +0000
Subject: [PATCH 5/8] fix: early return when AWS not configured correctly

---
 .github/workflows/ci.yml | 82 +++++++++++++++++++++++++++++++++-------
 1 file changed, 69 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 77fa08fe..2e09bc01 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,6 +35,64 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Check AWS S3 configuration, Install AWS CLI
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+        CI_LOGS_BUCKET: ${{ secrets.CI_LOGS_BUCKET }}
+      run: |
+        set -euo pipefail
+
+        echo "Checking AWS S3 configuration..."
+        MISSING=false
+        for VAR in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION CI_LOGS_BUCKET; do
+          if [ -z "${!VAR:-}" ]; then
+            echo "✖ $VAR is NOT set"
+            MISSING=true
+          else
+            if [ "$VAR" = "AWS_SECRET_ACCESS_KEY" ] || [ "$VAR" = "AWS_ACCESS_KEY_ID" ]; then
+              echo "✔ $VAR is set"
+            else
+              echo "✔ $VAR is set: ${!VAR}"
+            fi
+          fi
+        done
+
+        if [ "$MISSING" = "true" ]; then
+          echo "One or more AWS environment variables are missing. S3 upload will be skipped later."
+          echo "AWS_CONFIGURED=false" >> $GITHUB_ENV
+        else
+          echo "All required AWS environment variables are present."
+          echo "AWS_CONFIGURED=true" >> $GITHUB_ENV
+        fi
+
+        # Ensure unzip is available for AWS CLI installer
+        if ! command -v unzip &>/dev/null; then
+          echo "unzip not found; attempting to install unzip..."
+          sudo apt-get update -y || true
+          sudo apt-get install -y unzip || echo "Failed to install unzip; continuing"
+        fi
+
+        if command -v aws &>/dev/null; then
+          echo "AWS CLI already installed: $(aws --version)"
+        else
+          echo "AWS CLI not found; installing AWS CLI v2..."
+          curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+          unzip -q /tmp/awscliv2.zip -d /tmp/awscliv2
+          sudo /tmp/awscliv2/aws/install --update || sudo /tmp/awscliv2/aws/install || true
+          rm -rf /tmp/awscliv2.zip /tmp/awscliv2
+          if command -v aws &>/dev/null; then
+            echo "AWS CLI installed: $(aws --version)"
+          else
+            echo "Failed to install AWS CLI" >&2
+            if [ "${AWS_CONFIGURED:-false}" = "true" ]; then
+              echo "AWS required for uploads but installer failed; failing job." >&2
+              exit 1
+            fi
+          fi
+        fi
+
     # Free up disk space on GitHub Actions runner to avoid "no space left" errors
     - name: "EXEC: {Free up disk space}, independent"
       uses: endersonmenezes/free-disk-space@v3
@@ -323,24 +381,22 @@ jobs:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
         CI_LOGS_BUCKET: ${{ secrets.CI_LOGS_BUCKET }}
       run: |
-        if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_DEFAULT_REGION" ] || [ -z "$CI_LOGS_BUCKET" ]; then
-          echo "Skipping S3 upload: AWS credentials or CI_LOGS_BUCKET not configured"
+        set -euo pipefail
+
+        if [ "${AWS_CONFIGURED:-false}" != "true" ]; then
+          echo "AWS not configured; skipping S3 upload"
           exit 0
         fi
-        if command -v aws &>/dev/null; then
-          echo "AWS CLI already installed: $(aws --version)"
-        else
-          echo "AWS CLI not found, installing..."
-          curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
-          unzip -q /tmp/awscliv2.zip -d /tmp/awscliv2
-          sudo /tmp/awscliv2/aws/install
-          rm -rf /tmp/awscliv2.zip /tmp/awscliv2
-          echo "AWS CLI installed: $(aws --version)"
+
+        if ! command -v aws &>/dev/null; then
+          echo "AWS CLI not found; cannot upload logs to S3" >&2
+          exit 1
         fi
+
         BRANCH="${GITHUB_REF_NAME//\//-}"
-        S3_PATH="s3://${CI_LOGS_BUCKET}/runs/${BRANCH}/${{ github.run_id }}/${{ github.run_attempt }}/"
+        S3_PATH="s3://${CI_LOGS_BUCKET}/runs/${BRANCH}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
         echo "Uploading ~/.foc-devnet/state/latest to ${S3_PATH}"
-        aws s3 sync ~/.foc-devnet/state/latest "${S3_PATH}" --no-progress
+        aws s3 sync ~/.foc-devnet/state/latest "${S3_PATH}" --no-progress || echo "aws s3 sync returned non-zero"
         echo "Upload complete: ${S3_PATH}"
 
     # Verify cluster is running correctly

From a69a34d96a96b65481508880eda5b2c1cb251003 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@filoz.org>
Date: Fri, 27 Feb 2026 09:11:15 +0000
Subject: [PATCH 6/8] fix: AWS_DEFAULT_REGION

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2e09bc01..5e591b80 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,7 +39,7 @@ jobs:
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
         CI_LOGS_BUCKET: ${{ secrets.CI_LOGS_BUCKET }}
       run: |
         set -euo pipefail
@@ -378,7 +378,7 @@ jobs:
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
         CI_LOGS_BUCKET: ${{ secrets.CI_LOGS_BUCKET }}
       run: |
         set -euo pipefail

From a487fc1e8f06f03daeae91b79409a28873e94667 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@filoz.org>
Date: Fri, 27 Feb 2026 09:44:19 +0000
Subject: [PATCH 7/8] fix: BRANCH name corrections

---
 .github/workflows/ci.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5e591b80..af137f00 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -331,7 +331,7 @@ jobs:
     - name: "EXEC: {Start cluster}, independent"
       id: start_cluster
       continue-on-error: true
-      run: ./foc-devnet start --parallel
+      run: ./foc-devnet start --parallel --no-test
 
     # On failure, collect and print Docker container logs for debugging
     - name: "EXEC: {Collect Docker logs on failure}, independent"
@@ -393,7 +393,11 @@ jobs:
           exit 1
         fi
 
-        BRANCH="${GITHUB_REF_NAME//\//-}"
+        # derive branch name: use head ref for PRs, otherwise ref name
+        BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
+        # strip refs/heads/ prefix if present
+        BRANCH="${BRANCH#refs/heads/}"
+        BRANCH="${BRANCH//\//-}"
         S3_PATH="s3://${CI_LOGS_BUCKET}/runs/${BRANCH}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
         echo "Uploading ~/.foc-devnet/state/latest to ${S3_PATH}"
         aws s3 sync ~/.foc-devnet/state/latest "${S3_PATH}" --no-progress || echo "aws s3 sync returned non-zero"

From b27f07de99fad37cb8d77f18ef67f0b839795eb5 Mon Sep 17 00:00:00 2001
From: redpanda-f <redpanda@filoz.org>
Date: Fri, 27 Feb 2026 10:08:24 +0000
Subject: [PATCH 8/8] fix: no-test should be notest

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index af137f00..eae120c3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -331,7 +331,7 @@ jobs:
     - name: "EXEC: {Start cluster}, independent"
       id: start_cluster
       continue-on-error: true
-      run: ./foc-devnet start --parallel --no-test
+      run: ./foc-devnet start --parallel --notest
 
     # On failure, collect and print Docker container logs for debugging
     - name: "EXEC: {Collect Docker logs on failure}, independent"