feat(bench): run the random access benchmark against S3

.github/workflows/develop-bench.yml

@@ -35,8 +35,9 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
+          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}{2}', github.run_id, matrix.benchmark.id, matrix.benchmark.variant_id)
           || 'ubuntu-latest' }}
+    name: bench (${{ matrix.benchmark.id }}${{ matrix.benchmark.variant_id }})
     strategy:
       fail-fast: false
       matrix:
@@ -45,6 +46,14 @@ jobs:
             name: Random Access
             build_args: "--features lance"
             v4_ingest: true
+          # Same benchmark, reading the data from S3 instead of local NVMe.
+          - id: random-access-bench
+            variant_id: "-s3"
+            name: Random Access (S3)
+            build_args: "--features lance"
+            v4_ingest: true
+            remote_data_dir: >-
+              s3://vortex-ci-benchmark-datasets/develop/random-access/
           - id: compress-bench
             name: Compression
             build_args: "--features lance"
@@ -103,6 +112,25 @@ jobs:
           extra_args: "--debuginfo-strip=false"
           parca_agent_version: "0.49.0"
 
+      - name: Setup AWS CLI
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
+        with:
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
+          aws-region: us-east-1
+
+      - name: Upload benchmark data to S3
+        if: matrix.benchmark.remote_data_dir != null
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: |
+          set -Eeu -o pipefail -x
+
+          target/release_debug/${{ matrix.benchmark.id }} --prepare-data \
+            --formats parquet,vortex,lance
+          aws s3 rm --recursive "${{ matrix.benchmark.remote_data_dir }}"
+          aws s3 cp --recursive vortex-bench/data "${{ matrix.benchmark.remote_data_dir }}"
+
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
 
@@ -113,8 +141,10 @@ jobs:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+          AWS_REGION: "us-east-1"
         run: |
-          python3 scripts/random-access-split.py --emit-ingest-records
+          python3 scripts/random-access-split.py --emit-ingest-records \
+            ${{ matrix.benchmark.remote_data_dir && format('--remote-data-dir {0}', matrix.benchmark.remote_data_dir) || '' }}
 
       - name: Run ${{ matrix.benchmark.name }} benchmark
         if: matrix.benchmark.id != 'random-access-bench'
@@ -127,12 +157,6 @@ jobs:
           bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
             ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
-        with:
-          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-
       - name: Upload Benchmark Results
         shell: bash
         run: |

.github/workflows/pr-bench-dispatch.yml

@@ -35,6 +35,11 @@ jobs:
     uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
+  all-random-access-s3-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-random-access-s3.yml
+    secrets: inherit
+
   all-compression-bench:
     needs: remove-all-label
     uses: ./.github/workflows/pr-bench-compress.yml
@@ -68,6 +73,22 @@ jobs:
     uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
+  remove-random-access-s3-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-random-access-s3'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-random-access-s3
+          fail_on_error: true
+
+  random-access-s3-bench:
+    needs: remove-random-access-s3-label
+    uses: ./.github/workflows/pr-bench-random-access-s3.yml
+    secrets: inherit
+
   remove-compress-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10

.github/workflows/pr-bench-random-access-s3.yml

@@ -0,0 +1,24 @@
+# Runs the random-access benchmark for a pull request, reading the data from S3.
+
+name: PR Random Access S3 Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: random-access-bench
+      benchmark_name: Random Access (S3)
+      with_lance: true
+      variant_id: "-s3"
+      remote_data_dir: >-
+        s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/random-access/

.github/workflows/pr-bench-runner.yml

@@ -4,7 +4,7 @@ name: PR Benchmark Runner
 
 concurrency:
   # The group causes runs to queue instead of running in parallel.
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
   # Don't cancel benchmarks that are already running, instead just queue them up.
   cancel-in-progress: false
 
@@ -21,6 +21,20 @@ on:
         required: false
         type: boolean
         default: false
+      variant_id:
+        description: >-
+          Suffix distinguishing runs of the same benchmark, e.g. "-s3". Keeps the PR comment
+          tag and the concurrency group of a variant separate from the default run.
+        required: false
+        type: string
+        default: ""
+      remote_data_dir:
+        description: >-
+          When set, the benchmark data is uploaded to this S3 prefix and read back from there
+          instead of local disk. Only supported by random-access-bench.
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -88,6 +102,26 @@ jobs:
           extra_args: "--debuginfo-strip=false"
           parca_agent_version: "0.49.0"
 
+      - name: Setup AWS CLI
+        if: github.event.pull_request.head.repo.fork == false
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
+        with:
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
+          aws-region: us-east-1
+
+      - name: Upload benchmark data to S3
+        if: inputs.remote_data_dir != '' && github.event.pull_request.head.repo.fork == false
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: |
+          set -Eeu -o pipefail -x
+
+          target/release_debug/${{ inputs.benchmark_id }} --prepare-data \
+            --formats ${{ inputs.with_lance && 'parquet,vortex,lance' || 'parquet,vortex' }}
+          aws s3 rm --recursive "${{ inputs.remote_data_dir }}"
+          aws s3 cp --recursive vortex-bench/data "${{ inputs.remote_data_dir }}"
+
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
 
@@ -98,8 +132,10 @@ jobs:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+          AWS_REGION: "us-east-1"
         run: |
-          python3 scripts/random-access-split.py
+          python3 scripts/random-access-split.py \
+            ${{ inputs.remote_data_dir != '' && format('--remote-data-dir {0}', inputs.remote_data_dir) || '' }}
 
       - name: Run ${{ inputs.benchmark_name }} benchmark
         if: inputs.benchmark_id != 'random-access-bench'
@@ -111,13 +147,6 @@ jobs:
         run: |
           bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json
 
-      - name: Setup AWS CLI
-        if: github.event.pull_request.head.repo.fork == false
-        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
-        with:
-          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-
       - name: Install uv
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
@@ -140,7 +169,7 @@ jobs:
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
 
       - name: Comment PR on failure
         if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
@@ -150,4 +179,12 @@ jobs:
             # BENCHMARK FAILED
 
             Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
+
+
+      - name: Delete benchmark data from S3
+        if: always() && inputs.remote_data_dir != '' && github.event.pull_request.head.repo.fork == false
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: aws s3 rm --recursive "${{ inputs.remote_data_dir }}"