From 3cc775d6af5f49d866ab56a68af7b676cec22173 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Tue, 21 Jul 2026 17:33:23 +0530
Subject: [PATCH 01/10] feat(EC-1818): add stress benchmark CI workflow

Add a report-only GitHub Actions workflow that runs the stress
benchmark on PRs to main, surfacing peak memory and execution time
in the job summary without blocking merges. Uses oras to pull
pre-built benchmark data from Quay with upstream regeneration as
fallback.

Ref: EC-1818

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 102 +++++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 .github/workflows/benchmark.yaml

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
new file mode 100644
index 000000000..ca1287c95
--- /dev/null
+++ b/.github/workflows/benchmark.yaml
@@ -0,0 +1,102 @@
+# Copyright The Conforma Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+name: Benchmark
+
+"on":
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+
+  Stress:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    env:
+      EC_STRESS_COMPONENTS: "10"
+      EC_STRESS_WORKERS: "5"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
+        with:
+          egress-policy: audit
+          disable-telemetry: true
+
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Go environment
+        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Setup ORAS
+        uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1
+
+      - name: Prepare benchmark data
+        run: |
+          cd benchmark/stress
+          ./prepare_data.sh
+
+      - name: Run stress benchmark
+        run: |
+          cd benchmark/stress
+          go run . 2>/dev/null | tee benchmark-output.txt
+
+      - name: Write job summary
+        if: always()
+        run: |
+          if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then
+            echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY"
+            echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          line=$(grep '^BenchmarkStress' benchmark/stress/benchmark-output.txt || true)
+          if [[ -z "$line" ]]; then
+            echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY"
+            echo "No benchmark results found in output." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          ns_op=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}')
+          peak_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}')
+          alloc=$(echo "$line" | grep -oP '[\d.]+ allocated-bytes/op' | awk '{print $1}')
+          heap=$(echo "$line" | grep -oP '[\d.]+ heap-bytes-from-system' | awk '{print $1}')
+
+          secs=$(awk "BEGIN {printf \"%.1f\", ${ns_op:-0} / 1000000000}")
+          rss_mb=$(awk "BEGIN {printf \"%.0f\", ${peak_rss:-0} / 1048576}")
+          alloc_mb=$(awk "BEGIN {printf \"%.0f\", ${alloc:-0} / 1048576}")
+          heap_mb=$(awk "BEGIN {printf \"%.0f\", ${heap:-0} / 1048576}")
+
+          {
+            echo "## Stress Benchmark"
+            echo ""
+            echo "| Metric | Value |"
+            echo "|--------|-------|"
+            echo "| Components | ${EC_STRESS_COMPONENTS} |"
+            echo "| Workers | ${EC_STRESS_WORKERS} |"
+            echo "| Execution time | ${secs}s |"
+            echo "| Peak RSS | ${rss_mb} MB |"
+            echo "| Allocated memory | ${alloc_mb} MB |"
+            echo "| Heap from system | ${heap_mb} MB |"
+          } >> "$GITHUB_STEP_SUMMARY"

From 26bde5cd6e4f423e9e616c6790e3a010ffe28505 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Tue, 21 Jul 2026 18:37:16 +0530
Subject: [PATCH 02/10] chore(EC-1818): bump stress benchmark to 40 components,
 10 workers

Initial CI run with 10/5 completed in under a minute. Increase to
40/10 to target 5-6 minute total job time on CI runners.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index ca1287c95..c483c32af 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -31,8 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     env:
-      EC_STRESS_COMPONENTS: "10"
-      EC_STRESS_WORKERS: "5"
+      EC_STRESS_COMPONENTS: "40"
+      EC_STRESS_WORKERS: "10"
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1

From 7b261b5d1a2ac7f7184427f65d96bd98bda6d7ed Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Tue, 21 Jul 2026 19:37:04 +0530
Subject: [PATCH 03/10] fix(EC-1818): address review feedback on benchmark
 workflow

Rename workflow to "Stress Benchmark", add workflow_dispatch trigger,
separate build from execution to surface compilation errors, and
document CI-specific env var overrides.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index c483c32af..072f26272 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -15,12 +15,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-name: Benchmark
+name: Stress Benchmark
 
 "on":
   pull_request:
     branches:
       - main
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -31,6 +32,8 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     env:
+      # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes.
+      # Code defaults are 10 components / 35 workers.
       EC_STRESS_COMPONENTS: "40"
       EC_STRESS_WORKERS: "10"
     steps:
@@ -57,10 +60,13 @@ jobs:
           cd benchmark/stress
           ./prepare_data.sh
 
+      - name: Build stress benchmark
+        run: go build -o benchmark/stress/stress ./benchmark/stress
+
       - name: Run stress benchmark
         run: |
           cd benchmark/stress
-          go run . 2>/dev/null | tee benchmark-output.txt
+          ./stress 2>/dev/null | tee benchmark-output.txt
 
       - name: Write job summary
         if: always()

From 4861700557a0b6b9fa3aa262d9118d3c76f20059 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Wed, 22 Jul 2026 17:27:48 +0530
Subject: [PATCH 04/10] chore(EC-1818): bump stress benchmark to 80 components

Doubles component count from 40 to 80 to increase peak RSS (~4 GB),
making memory regressions more visible in CI job summaries.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 072f26272..0d0eddc68 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -34,7 +34,7 @@ jobs:
     env:
       # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes.
       # Code defaults are 10 components / 35 workers.
-      EC_STRESS_COMPONENTS: "40"
+      EC_STRESS_COMPONENTS: "80"
       EC_STRESS_WORKERS: "10"
     steps:
       - name: Harden Runner

From d3506318028bf7813b8b11fbd7f20e6d37d7b8ac Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Wed, 22 Jul 2026 17:30:58 +0530
Subject: [PATCH 05/10] fix(EC-1818): capture stderr and use lowercase job ID

Redirect stderr to a file instead of /dev/null so panics and errors
are surfaced in the job summary. Rename job ID from Stress to stress
to match the dominant lowercase convention in the repo.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 0d0eddc68..8dede32ef 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -28,7 +28,7 @@ permissions:
 
 jobs:
 
-  Stress:
+  stress:
     runs-on: ubuntu-latest
     continue-on-error: true
     env:
@@ -66,11 +66,20 @@ jobs:
       - name: Run stress benchmark
         run: |
           cd benchmark/stress
-          ./stress 2>/dev/null | tee benchmark-output.txt
+          ./stress 2>benchmark-stderr.txt | tee benchmark-output.txt
 
       - name: Write job summary
         if: always()
         run: |
+          if [[ -s benchmark/stress/benchmark-stderr.txt ]]; then
+            {
+              echo "### Stderr"
+              echo '```'
+              tail -50 benchmark/stress/benchmark-stderr.txt
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
           if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then
             echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY"
             echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY"

From bb9a51653c63cd19480e7222049f56b3a9e00a3e Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Wed, 22 Jul 2026 17:50:23 +0530
Subject: [PATCH 06/10] chore(EC-1818): revert stress components to default 10
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All components use the same image digest, so internal caches
deduplicate after the first — higher counts don't add memory
pressure. Use the code default to avoid misleading numbers.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 8dede32ef..61737b6ca 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -34,7 +34,7 @@ jobs:
     env:
       # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes.
       # Code defaults are 10 components / 35 workers.
-      EC_STRESS_COMPONENTS: "80"
+      EC_STRESS_COMPONENTS: "10"
       EC_STRESS_WORKERS: "10"
     steps:
       - name: Harden Runner

From 5306b46ab9e3f1cc0d6d7659f5c3f58e52393cd2 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Wed, 22 Jul 2026 18:14:35 +0530
Subject: [PATCH 07/10] fix(EC-1818): address review feedback on benchmark
 workflow

- Add job name for GitHub Actions UI display
- Add cache restore step matching repo convention
- Use awk -v for variable passing instead of shell interpolation
- Add set -o pipefail so benchmark failures are not masked by tee

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 61737b6ca..d0514ac8d 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -29,6 +29,7 @@ permissions:
 jobs:
 
   stress:
+    name: Stress Benchmark
     runs-on: ubuntu-latest
     continue-on-error: true
     env:
@@ -46,6 +47,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
+      - name: Restore Cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          key: main
+          path: '**'
+
       - name: Setup Go environment
         uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
@@ -65,6 +72,7 @@ jobs:
 
       - name: Run stress benchmark
         run: |
+          set -o pipefail
           cd benchmark/stress
           ./stress 2>benchmark-stderr.txt | tee benchmark-output.txt
 
@@ -98,10 +106,10 @@ jobs:
           alloc=$(echo "$line" | grep -oP '[\d.]+ allocated-bytes/op' | awk '{print $1}')
           heap=$(echo "$line" | grep -oP '[\d.]+ heap-bytes-from-system' | awk '{print $1}')
 
-          secs=$(awk "BEGIN {printf \"%.1f\", ${ns_op:-0} / 1000000000}")
-          rss_mb=$(awk "BEGIN {printf \"%.0f\", ${peak_rss:-0} / 1048576}")
-          alloc_mb=$(awk "BEGIN {printf \"%.0f\", ${alloc:-0} / 1048576}")
-          heap_mb=$(awk "BEGIN {printf \"%.0f\", ${heap:-0} / 1048576}")
+          secs=$(awk -v val="${ns_op:-0}" 'BEGIN {printf "%.1f", val / 1000000000}')
+          rss_mb=$(awk -v val="${peak_rss:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
+          alloc_mb=$(awk -v val="${alloc:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
+          heap_mb=$(awk -v val="${heap:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
 
           {
             echo "## Stress Benchmark"

From 0bb84314cb324ae55395c9ae577a53b93b0aae24 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Mon, 27 Jul 2026 16:19:03 +0530
Subject: [PATCH 08/10] fix(EC-1818): add job timeout to benchmark workflow

Adds a 15-minute timeout to prevent hung oras pulls or stuck
benchmark processes from running for the default 6-hour limit.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index d0514ac8d..92fbbd031 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -31,6 +31,7 @@ jobs:
   stress:
     name: Stress Benchmark
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     continue-on-error: true
     env:
       # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes.

From 6b8e5bfa148ac00a8ac02ca24a060ab631d5c343 Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Mon, 27 Jul 2026 17:25:42 +0530
Subject: [PATCH 09/10] fix(EC-1818): only show stderr in job summary on
 failure

The ec CLI emits harmless logrus ERRO messages during normal
validation, which made every job summary start with a misleading
Stderr section. Gate it on step outcome so it only appears when
the benchmark actually fails.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 92fbbd031..a2024f210 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -72,6 +72,7 @@ jobs:
         run: go build -o benchmark/stress/stress ./benchmark/stress
 
       - name: Run stress benchmark
+        id: bench
         run: |
           set -o pipefail
           cd benchmark/stress
@@ -80,7 +81,7 @@ jobs:
       - name: Write job summary
         if: always()
         run: |
-          if [[ -s benchmark/stress/benchmark-stderr.txt ]]; then
+          if [[ "${{ steps.bench.outcome }}" == "failure" && -s benchmark/stress/benchmark-stderr.txt ]]; then
             {
               echo "### Stderr"
               echo '```'

From 45c5b81c9740749854375d7362eaadead1bbf80b Mon Sep 17 00:00:00 2001
From: Dheeraj <djodha@redhat.com>
Date: Mon, 27 Jul 2026 17:42:00 +0530
Subject: [PATCH 10/10] fix(EC-1818): add descriptions to job summary metrics
 table

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .github/workflows/benchmark.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index a2024f210..268b44755 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -116,12 +116,12 @@ jobs:
           {
             echo "## Stress Benchmark"
             echo ""
-            echo "| Metric | Value |"
-            echo "|--------|-------|"
-            echo "| Components | ${EC_STRESS_COMPONENTS} |"
-            echo "| Workers | ${EC_STRESS_WORKERS} |"
-            echo "| Execution time | ${secs}s |"
-            echo "| Peak RSS | ${rss_mb} MB |"
-            echo "| Allocated memory | ${alloc_mb} MB |"
-            echo "| Heap from system | ${heap_mb} MB |"
+            echo "| Metric | Value | Description |"
+            echo "|--------|-------|-------------|"
+            echo "| Components | ${EC_STRESS_COMPONENTS} | Snapshot components validated |"
+            echo "| Workers | ${EC_STRESS_WORKERS} | Parallel validation workers |"
+            echo "| Execution time | ${secs}s | Wall-clock time per iteration |"
+            echo "| Peak RSS | ${rss_mb} MB | Max physical memory used |"
+            echo "| Allocated memory | ${alloc_mb} MB | Total Go heap allocations |"
+            echo "| Heap from system | ${heap_mb} MB | Heap memory requested from OS |"
           } >> "$GITHUB_STEP_SUMMARY"