From 0e297f8acf4c57366ce09702f1d8f69e0f65ae31 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 14 Aug 2026 13:49:19 +0000
Subject: [PATCH 1/4] feat(bench): run the random access benchmark against S3

Adds an S3 variant of the random-access benchmark. It is the same benchmark
and the same code paths, only the data is read from an object store instead
of local NVMe:

- `--remote-data-dir s3://bucket/prefix/` opens the Vortex, Parquet and Lance
  files from S3, mirroring the local data directory layout;
- `--prepare-data` materializes the local files (and prints their paths) so
  CI can upload them before the run;
- remote measurements are suffixed `-tokio-s3` and reported with `s3` storage
  so they form a separate series from the local-disk numbers.

CI runs it from a new `pr-bench-random-access-s3.yml` workflow (label
`action/bench-random-access-s3`, also covered by `action/bench-all`) and from
a new `Random Access (S3)` entry in the develop benchmark matrix. The shared
PR benchmark runner gained `remote_data_dir` and `variant_id` inputs; it
uploads the data to a per-run S3 prefix, runs the benchmark against it, and
deletes the prefix afterwards.

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
---
 .github/workflows/develop-bench.yml           |  40 ++++-
 .github/workflows/pr-bench-dispatch.yml       |  21 +++
 .../workflows/pr-bench-random-access-s3.yml   |  24 +++
 .github/workflows/pr-bench-runner.yml         |  59 +++++--
 Cargo.lock                                    |   2 +
 benchmarks/lance-bench/src/random_access.rs   |  10 +-
 benchmarks/random-access-bench/Cargo.toml     |   1 +
 benchmarks/random-access-bench/README.md      |  24 +++
 benchmarks/random-access-bench/src/lib.rs     | 132 +++++++++++----
 benchmarks/random-access-bench/src/main.rs    |  31 +++-
 scripts/random-access-split.py                |  10 +-
 vortex-bench/Cargo.toml                       |   3 +-
 vortex-bench/src/random_access/mod.rs         | 120 +++++++++++++
 vortex-bench/src/random_access/take.rs        | 159 ++++++++++++++----
 14 files changed, 543 insertions(+), 93 deletions(-)
 create mode 100644 .github/workflows/pr-bench-random-access-s3.yml

diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml
index 2066d80257c..a15eb1cacba 100644
--- a/.github/workflows/develop-bench.yml
+++ b/.github/workflows/develop-bench.yml
@@ -35,8 +35,9 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
+          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}{2}', github.run_id, matrix.benchmark.id, matrix.benchmark.variant_id)
           || 'ubuntu-latest' }}
+    name: bench (${{ matrix.benchmark.id }}${{ matrix.benchmark.variant_id }})
     strategy:
       fail-fast: false
       matrix:
@@ -45,6 +46,14 @@ jobs:
             name: Random Access
             build_args: "--features lance"
             v4_ingest: true
+          # Same benchmark, reading the data from S3 instead of local NVMe.
+          - id: random-access-bench
+            variant_id: "-s3"
+            name: Random Access (S3)
+            build_args: "--features lance"
+            v4_ingest: true
+            remote_data_dir: >-
+              s3://vortex-ci-benchmark-datasets/develop/random-access/
           - id: compress-bench
             name: Compression
             build_args: "--features lance"
@@ -103,6 +112,25 @@ jobs:
           extra_args: "--debuginfo-strip=false"
           parca_agent_version: "0.49.0"
 
+      - name: Setup AWS CLI
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
+        with:
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
+          aws-region: us-east-1
+
+      - name: Upload benchmark data to S3
+        if: matrix.benchmark.remote_data_dir != null
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: |
+          set -Eeu -o pipefail -x
+
+          target/release_debug/${{ matrix.benchmark.id }} --prepare-data \
+            --formats parquet,vortex,lance
+          aws s3 rm --recursive "${{ matrix.benchmark.remote_data_dir }}"
+          aws s3 cp --recursive vortex-bench/data "${{ matrix.benchmark.remote_data_dir }}"
+
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
 
@@ -113,8 +141,10 @@ jobs:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+          AWS_REGION: "us-east-1"
         run: |
-          python3 scripts/random-access-split.py --emit-ingest-records
+          python3 scripts/random-access-split.py --emit-ingest-records \
+            ${{ matrix.benchmark.remote_data_dir && format('--remote-data-dir {0}', matrix.benchmark.remote_data_dir) || '' }}
 
       - name: Run ${{ matrix.benchmark.name }} benchmark
         if: matrix.benchmark.id != 'random-access-bench'
@@ -127,12 +157,6 @@ jobs:
           bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
             ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
-        with:
-          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-
       - name: Upload Benchmark Results
         shell: bash
         run: |
diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
index 3cb60c619bf..fa9a701d758 100644
--- a/.github/workflows/pr-bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -35,6 +35,11 @@ jobs:
     uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
+  all-random-access-s3-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-random-access-s3.yml
+    secrets: inherit
+
   all-compression-bench:
     needs: remove-all-label
     uses: ./.github/workflows/pr-bench-compress.yml
@@ -68,6 +73,22 @@ jobs:
     uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
+  remove-random-access-s3-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-random-access-s3'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-random-access-s3
+          fail_on_error: true
+
+  random-access-s3-bench:
+    needs: remove-random-access-s3-label
+    uses: ./.github/workflows/pr-bench-random-access-s3.yml
+    secrets: inherit
+
   remove-compress-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
diff --git a/.github/workflows/pr-bench-random-access-s3.yml b/.github/workflows/pr-bench-random-access-s3.yml
new file mode 100644
index 00000000000..d5d77cab2d3
--- /dev/null
+++ b/.github/workflows/pr-bench-random-access-s3.yml
@@ -0,0 +1,24 @@
+# Runs the random-access benchmark for a pull request, reading the data from S3.
+
+name: PR Random Access S3 Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: random-access-bench
+      benchmark_name: Random Access (S3)
+      with_lance: true
+      variant_id: "-s3"
+      remote_data_dir: >-
+        s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/random-access/
diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml
index 22c2c693fd3..ef1042eb257 100644
--- a/.github/workflows/pr-bench-runner.yml
+++ b/.github/workflows/pr-bench-runner.yml
@@ -4,7 +4,7 @@ name: PR Benchmark Runner
 
 concurrency:
   # The group causes runs to queue instead of running in parallel.
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
   # Don't cancel benchmarks that are already running, instead just queue them up.
   cancel-in-progress: false
 
@@ -21,6 +21,20 @@ on:
         required: false
         type: boolean
         default: false
+      variant_id:
+        description: >-
+          Suffix distinguishing runs of the same benchmark, e.g. "-s3". Keeps the PR comment
+          tag and the concurrency group of a variant separate from the default run.
+        required: false
+        type: string
+        default: ""
+      remote_data_dir:
+        description: >-
+          When set, the benchmark data is uploaded to this S3 prefix and read back from there
+          instead of local disk. Only supported by random-access-bench.
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -88,6 +102,26 @@ jobs:
           extra_args: "--debuginfo-strip=false"
           parca_agent_version: "0.49.0"
 
+      - name: Setup AWS CLI
+        if: github.event.pull_request.head.repo.fork == false
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
+        with:
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
+          aws-region: us-east-1
+
+      - name: Upload benchmark data to S3
+        if: inputs.remote_data_dir != '' && github.event.pull_request.head.repo.fork == false
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: |
+          set -Eeu -o pipefail -x
+
+          target/release_debug/${{ inputs.benchmark_id }} --prepare-data \
+            --formats ${{ inputs.with_lance && 'parquet,vortex,lance' || 'parquet,vortex' }}
+          aws s3 rm --recursive "${{ inputs.remote_data_dir }}"
+          aws s3 cp --recursive vortex-bench/data "${{ inputs.remote_data_dir }}"
+
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
 
@@ -98,8 +132,10 @@ jobs:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+          AWS_REGION: "us-east-1"
         run: |
-          python3 scripts/random-access-split.py
+          python3 scripts/random-access-split.py \
+            ${{ inputs.remote_data_dir != '' && format('--remote-data-dir {0}', inputs.remote_data_dir) || '' }}
 
       - name: Run ${{ inputs.benchmark_name }} benchmark
         if: inputs.benchmark_id != 'random-access-bench'
@@ -111,13 +147,6 @@ jobs:
         run: |
           bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json
 
-      - name: Setup AWS CLI
-        if: github.event.pull_request.head.repo.fork == false
-        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
-        with:
-          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-
       - name: Install uv
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
@@ -140,7 +169,7 @@ jobs:
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
 
       - name: Comment PR on failure
         if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
@@ -150,4 +179,12 @@ jobs:
             # BENCHMARK FAILED
 
             Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}${{ inputs.variant_id }}
+
+
+      - name: Delete benchmark data from S3
+        if: always() && inputs.remote_data_dir != '' && github.event.pull_request.head.repo.fork == false
+        shell: bash
+        env:
+          AWS_REGION: "us-east-1"
+        run: aws s3 rm --recursive "${{ inputs.remote_data_dir }}"
diff --git a/Cargo.lock b/Cargo.lock
index b0f98336b0b..0b2d77515d7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7298,6 +7298,7 @@ dependencies = [
  "rand_distr 0.6.0",
  "tabled",
  "tokio",
+ "url",
  "vortex-bench",
 ]
 
@@ -9706,6 +9707,7 @@ dependencies = [
  "mimalloc",
  "noodles-bgzf",
  "noodles-vcf",
+ "object_store",
  "parking_lot",
  "parquet 56.2.1",
  "parquet 58.4.0",
diff --git a/benchmarks/lance-bench/src/random_access.rs b/benchmarks/lance-bench/src/random_access.rs
index f38bfca2d5f..64be46c00ff 100644
--- a/benchmarks/lance-bench/src/random_access.rs
+++ b/benchmarks/lance-bench/src/random_access.rs
@@ -89,11 +89,17 @@ pub struct LanceRandomAccessor {
 impl LanceRandomAccessor {
     /// Open a Lance dataset and return a ready-to-use accessor.
     pub async fn open(path: PathBuf, name: impl Into<String>) -> anyhow::Result<Self> {
-        let dataset = Dataset::open(
+        Self::open_uri(
             path.to_str()
                 .ok_or_else(|| anyhow!("Invalid dataset path"))?,
+            name,
         )
-        .await?;
+        .await
+    }
+
+    /// Open a Lance dataset from any URI (local path, `s3://...`, ...).
+    pub async fn open_uri(uri: &str, name: impl Into<String>) -> anyhow::Result<Self> {
+        let dataset = Dataset::open(uri).await?;
         let projection = ProjectionRequest::from_schema(dataset.schema().clone());
         Ok(Self {
             name: name.into(),
diff --git a/benchmarks/random-access-bench/Cargo.toml b/benchmarks/random-access-bench/Cargo.toml
index c45c1ac2bff..001a6acedc6 100644
--- a/benchmarks/random-access-bench/Cargo.toml
+++ b/benchmarks/random-access-bench/Cargo.toml
@@ -23,6 +23,7 @@ rand = { workspace = true }
 rand_distr = { workspace = true }
 tabled = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
+url = { workspace = true }
 vortex-bench = { workspace = true }
 
 [features]
diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md
index 14949c85fcb..d9a9926635f 100644
--- a/benchmarks/random-access-bench/README.md
+++ b/benchmarks/random-access-bench/README.md
@@ -20,3 +20,27 @@ reopening the file per lookup. CI drives the full matrix via
 ```bash
 cargo run -p random-access-bench --profile release_debug --features lance
 ```
+
+## Running against S3
+
+The same benchmark can read its data from an object store instead of local disk. The remote
+directory must mirror the layout of the local data directory (`vortex-bench/data/`), so the
+files are materialized locally first and then uploaded verbatim:
+
+```bash
+cargo run -p random-access-bench --profile release_debug --features lance -- \
+  --prepare-data --formats parquet,vortex,lance
+aws s3 cp --recursive vortex-bench/data s3://my-bucket/my-prefix/
+
+cargo run -p random-access-bench --profile release_debug --features lance -- \
+  --remote-data-dir s3://my-bucket/my-prefix/
+```
+
+Credentials and region come from the environment (`AWS_REGION`, `AWS_PROFILE`, ...).
+
+Remote measurements are named `...-tokio-s3` instead of `...-tokio-local-disk` and are
+reported with `s3` storage, so they form a series separate from the local-disk numbers. In CI
+the variant runs from
+[`pr-bench-random-access-s3.yml`](../../.github/workflows/pr-bench-random-access-s3.yml)
+(label `action/bench-random-access-s3`) and from the `Random Access (S3)` matrix entry in
+`develop-bench.yml`.
diff --git a/benchmarks/random-access-bench/src/lib.rs b/benchmarks/random-access-bench/src/lib.rs
index 11888d34c43..bc43f940e21 100644
--- a/benchmarks/random-access-bench/src/lib.rs
+++ b/benchmarks/random-access-bench/src/lib.rs
@@ -23,8 +23,10 @@ use vortex_bench::measurements::TimingMeasurement;
 use vortex_bench::random_access::BenchDataset;
 use vortex_bench::random_access::ParquetRandomAccessor;
 use vortex_bench::random_access::RandomAccessor;
+use vortex_bench::random_access::RemoteDataDir;
 use vortex_bench::random_access::VortexRandomAccessor;
 use vortex_bench::utils::constants::STORAGE_NVME;
+use vortex_bench::utils::constants::STORAGE_S3;
 use vortex_bench::v3;
 
 use crate::render::RandomAccessRun;
@@ -145,11 +147,12 @@ async fn benchmark_random_access(
     time_limit_secs: u64,
     storage: &str,
     reopen: bool,
+    remote: Option<&RemoteDataDir>,
 ) -> Result<RandomAccessRun> {
     let time_limit = Duration::from_secs(time_limit_secs);
     let overall_start = Instant::now();
     let mut runs = Vec::new();
-    let mut accessor = open_accessor(dataset, format).await?;
+    let mut accessor = open_accessor(dataset, format, remote).await?;
 
     loop {
         let start = Instant::now();
@@ -162,7 +165,7 @@ async fn benchmark_random_access(
         }
 
         if reopen {
-            accessor = open_accessor(dataset, format).await?;
+            accessor = open_accessor(dataset, format, remote).await?;
         }
     }
 
@@ -197,16 +200,29 @@ fn display_name(dataset: &str, pattern: Option<AccessPattern>) -> String {
 /// historical continuity with existing benchmark data.
 /// For other datasets, includes dataset and pattern:
 /// `random-access/{dataset}/{pattern}/{format}-tokio-local-disk`.
-fn measurement_name(dataset: &str, pattern: Option<AccessPattern>, format: Format) -> String {
+///
+/// Remote runs use a `-tokio-s3` suffix so their results form a separate series from the
+/// local-disk ones.
+fn measurement_name(
+    dataset: &str,
+    pattern: Option<AccessPattern>,
+    format: Format,
+    remote: bool,
+) -> String {
     let fmt = format.ext();
+    let suffix = source_suffix(remote);
     match pattern {
-        Some(p) => format!(
-            "random-access/{}/{}/{}-tokio-local-disk",
-            dataset,
-            p.name(),
-            fmt
-        ),
-        None => format!("random-access/{}-tokio-local-disk", fmt),
+        Some(p) => format!("random-access/{}/{}/{}-{}", dataset, p.name(), fmt, suffix),
+        None => format!("random-access/{}-{}", fmt, suffix),
+    }
+}
+
+/// Name suffix identifying where the benchmarked files are read from.
+fn source_suffix(remote: bool) -> &'static str {
+    if remote {
+        "tokio-s3"
+    } else {
+        "tokio-local-disk"
     }
 }
 
@@ -226,6 +242,38 @@ fn push_v3_random_access_record(records: &mut Vec<v3::V3Record>, run: &RandomAcc
     records.push(v3::random_access_record(&run.timing, &dataset));
 }
 
+/// Materialize the local data file for `dataset` in `format`, writing it if it is missing.
+async fn dataset_path(dataset: &dyn BenchDataset, format: Format) -> Result<PathBuf> {
+    match format {
+        #[cfg(feature = "lance")]
+        Format::Lance => {
+            use lance_bench::random_access;
+            match dataset.name() {
+                "taxi" => random_access::taxi_data_lance().await,
+                "feature-vectors" => random_access::feature_vectors_lance().await,
+                "nested-lists" => random_access::nested_lists_lance().await,
+                "nested-structs" => random_access::nested_structs_lance().await,
+                other => anyhow::bail!("Unknown dataset for Lance: {other}"),
+            }
+        }
+        format => dataset.path(format).await,
+    }
+}
+
+/// Materialize every local data file needed for `datasets` and `formats`.
+///
+/// Remote runs read data that must already exist in the remote directory, so this is run first
+/// (locally) and the resulting files uploaded verbatim.
+pub async fn prepare_data(datasets: &[Box<dyn BenchDataset>], formats: &[Format]) -> Result<()> {
+    for dataset in datasets {
+        for format in formats {
+            let path = dataset_path(dataset.as_ref(), *format).await?;
+            println!("{}", path.display());
+        }
+    }
+    Ok(())
+}
+
 /// Open a random accessor for any supported format.
 ///
 /// For Vortex and Parquet, the path comes from [`BenchDataset::path`].
@@ -233,36 +281,43 @@ fn push_v3_random_access_record(records: &mut Vec<v3::V3Record>, run: &RandomAcc
 async fn open_accessor(
     dataset: &dyn BenchDataset,
     format: Format,
+    remote: Option<&RemoteDataDir>,
 ) -> Result<Box<dyn RandomAccessor>> {
     let name = format!(
-        "random-access/{}/{}-tokio-local-disk",
+        "random-access/{}/{}-{}",
         dataset.name(),
-        format.ext()
+        format.ext(),
+        source_suffix(remote.is_some())
     );
     match format {
         Format::OnDiskVortex | Format::VortexCompact => {
-            let path = dataset.path(format).await?;
-            Ok(Box::new(
-                VortexRandomAccessor::open(path, name, format).await?,
-            ))
+            let path = dataset_path(dataset, format).await?;
+            Ok(match remote {
+                Some(remote) => Box::new(
+                    VortexRandomAccessor::open_object_store(remote, &path, name, format).await?,
+                ),
+                None => Box::new(VortexRandomAccessor::open(path, name, format).await?),
+            })
         }
         Format::Parquet => {
-            let path = dataset.path(format).await?;
-            Ok(Box::new(ParquetRandomAccessor::open(path, name).await?))
+            let path = dataset_path(dataset, format).await?;
+            Ok(match remote {
+                Some(remote) => {
+                    Box::new(ParquetRandomAccessor::open_object_store(remote, &path, name).await?)
+                }
+                None => Box::new(ParquetRandomAccessor::open(path, name).await?),
+            })
         }
         #[cfg(feature = "lance")]
         Format::Lance => {
             use lance_bench::random_access;
-            let path = match dataset.name() {
-                "taxi" => random_access::taxi_data_lance().await?,
-                "feature-vectors" => random_access::feature_vectors_lance().await?,
-                "nested-lists" => random_access::nested_lists_lance().await?,
-                "nested-structs" => random_access::nested_structs_lance().await?,
-                other => anyhow::bail!("Unknown dataset for Lance: {other}"),
-            };
-            Ok(Box::new(
-                random_access::LanceRandomAccessor::open(path, name).await?,
-            ))
+            let path = dataset_path(dataset, format).await?;
+            Ok(match remote {
+                Some(remote) => Box::new(
+                    random_access::LanceRandomAccessor::open_uri(&remote.uri(&path)?, name).await?,
+                ),
+                None => Box::new(random_access::LanceRandomAccessor::open(path, name).await?),
+            })
         }
         other => unimplemented!("open_accessor not implemented for {other}"),
     }
@@ -295,6 +350,8 @@ pub struct RunConfig {
     pub output_path: Option<PathBuf>,
     /// Optional path for benchmark ingest JSONL records.
     pub ingest_output: Option<PathBuf>,
+    /// When set, read the data files from this remote directory instead of local disk.
+    pub remote_data_dir: Option<RemoteDataDir>,
 }
 
 /// Run random-access benchmarks with `config`.
@@ -308,8 +365,16 @@ pub async fn run(config: RunConfig) -> Result<()> {
         display_format,
         output_path,
         ingest_output,
+        remote_data_dir,
     } = config;
 
+    let remote = remote_data_dir.as_ref();
+    let storage = if remote.is_some() {
+        STORAGE_S3
+    } else {
+        STORAGE_NVME
+    };
+
     let reopen_variants: &[bool] = match open_mode {
         OpenMode::Cached => &[false],
         OpenMode::Reopen => &[true],
@@ -333,7 +398,7 @@ pub async fn run(config: RunConfig) -> Result<()> {
     for dataset in &datasets {
         for format in &formats {
             if dataset.name() == "taxi" {
-                let name = measurement_name(dataset.name(), None, *format);
+                let name = measurement_name(dataset.name(), None, *format, remote.is_some());
                 for &reopen in reopen_variants {
                     let bench_name = if reopen {
                         format!("{name}-footer")
@@ -347,8 +412,9 @@ pub async fn run(config: RunConfig) -> Result<()> {
                         None,
                         &FIXED_TAXI_INDICES,
                         time_limit,
-                        STORAGE_NVME,
+                        storage,
                         reopen,
+                        remote,
                     )
                     .await?;
 
@@ -360,7 +426,8 @@ pub async fn run(config: RunConfig) -> Result<()> {
 
             for pattern in &patterns {
                 let indices = generate_indices(dataset.as_ref(), *pattern);
-                let name = measurement_name(dataset.name(), Some(*pattern), *format);
+                let name =
+                    measurement_name(dataset.name(), Some(*pattern), *format, remote.is_some());
                 for &reopen in reopen_variants {
                     let bench_name = if reopen {
                         format!("{name}-footer")
@@ -374,8 +441,9 @@ pub async fn run(config: RunConfig) -> Result<()> {
                         Some(*pattern),
                         &indices,
                         time_limit,
-                        STORAGE_NVME,
+                        storage,
                         reopen,
+                        remote,
                     )
                     .await?;
 
diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs
index 11ef81edd3c..678a5787849 100644
--- a/benchmarks/random-access-bench/src/main.rs
+++ b/benchmarks/random-access-bench/src/main.rs
@@ -9,6 +9,7 @@ use clap::ValueEnum;
 use random_access_bench::AccessPattern;
 use random_access_bench::OpenMode;
 use random_access_bench::RunConfig;
+use url::Url;
 use vortex_bench::Format;
 use vortex_bench::datasets::feature_vectors::FeatureVectorsData;
 use vortex_bench::datasets::nested_lists::NestedListsData;
@@ -16,6 +17,7 @@ use vortex_bench::datasets::nested_structs::NestedStructsData;
 use vortex_bench::datasets::taxi_data::TaxiData;
 use vortex_bench::display::DisplayFormat;
 use vortex_bench::random_access::BenchDataset;
+use vortex_bench::random_access::RemoteDataDir;
 use vortex_bench::setup_logging_and_tracing;
 
 /// Which synthetic dataset to benchmark.
@@ -85,6 +87,15 @@ struct Args {
     /// Whether to reopen the file on each iteration, use a cached handle, or run both.
     #[arg(long, value_enum, default_value_t = OpenMode::Both)]
     open_mode: OpenMode,
+    /// Read the data files from this remote directory (e.g. `s3://bucket/prefix/`) instead of
+    /// local disk. The directory must mirror the layout of the local benchmark data directory,
+    /// as produced by `--prepare-data`.
+    #[arg(long)]
+    remote_data_dir: Option<Url>,
+    /// Materialize the local data files for the selected datasets and formats, print their
+    /// paths, and exit without benchmarking.
+    #[arg(long)]
+    prepare_data: bool,
 }
 
 #[tokio::main]
@@ -92,12 +103,18 @@ async fn main() -> Result<()> {
     let args = Args::parse();
     setup_logging_and_tracing(args.verbose, args.tracing)?;
 
+    let datasets: Vec<Box<dyn BenchDataset>> = args
+        .datasets
+        .into_iter()
+        .map(DatasetArg::into_dataset)
+        .collect();
+
+    if args.prepare_data {
+        return random_access_bench::prepare_data(&datasets, &args.formats).await;
+    }
+
     let run_config = RunConfig {
-        datasets: args
-            .datasets
-            .into_iter()
-            .map(DatasetArg::into_dataset)
-            .collect(),
+        datasets,
         formats: args.formats,
         patterns: args.patterns,
         time_limit: args.time_limit,
@@ -105,6 +122,10 @@ async fn main() -> Result<()> {
         display_format: args.display_format,
         output_path: args.output_path,
         ingest_output: args.ingest_output,
+        remote_data_dir: args
+            .remote_data_dir
+            .map(RemoteDataDir::try_new)
+            .transpose()?,
     };
 
     random_access_bench::run(run_config).await
diff --git a/scripts/random-access-split.py b/scripts/random-access-split.py
index 698aa28f506..ac08459aed1 100755
--- a/scripts/random-access-split.py
+++ b/scripts/random-access-split.py
@@ -23,7 +23,7 @@
 OPEN_MODES = ["cached", "reopen"]
 
 
-def run_combinations(emit_ingest_records: bool) -> None:
+def run_combinations(emit_ingest_records: bool, remote_data_dir: str | None) -> None:
     PARTS_DIR.mkdir(parents=True, exist_ok=True)
     i = 0
     for dataset in DATASETS:
@@ -47,6 +47,8 @@ def run_combinations(emit_ingest_records: bool) -> None:
                         "-o",
                         str(PARTS_DIR / f"{i}.gh.json"),
                     ]
+                    if remote_data_dir:
+                        args += ["--remote-data-dir", remote_data_dir]
                     if emit_ingest_records:
                         args += ["--ingest-jsonl", str(PARTS_DIR / f"{i}.ingest.jsonl")]
                     print("+", " ".join(args), flush=True)
@@ -86,9 +88,13 @@ def main() -> None:
         action="store_true",
         help="merge --ingest-jsonl records into results.ingest.jsonl",
     )
+    parser.add_argument(
+        "--remote-data-dir",
+        help="read the benchmark data from this remote directory (e.g. s3://bucket/prefix/)",
+    )
     args = parser.parse_args()
 
-    run_combinations(args.emit_ingest_records)
+    run_combinations(args.emit_ingest_records, args.remote_data_dir)
     merge(f"{PARTS_DIR}/*.gh.json", lambda record: record["name"], "results.json")
     if args.emit_ingest_records:
         merge(
diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml
index 965a80c6259..14074792733 100644
--- a/vortex-bench/Cargo.toml
+++ b/vortex-bench/Cargo.toml
@@ -51,8 +51,9 @@ itertools = { workspace = true }
 mimalloc = { workspace = true }
 noodles-bgzf = { workspace = true, features = ["async"] }
 noodles-vcf = { workspace = true, features = ["async"] }
+object_store = { workspace = true, features = ["aws"] }
 parking_lot = { workspace = true }
-parquet = { workspace = true, features = ["async"] }
+parquet = { workspace = true, features = ["async", "object_store"] }
 rand = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true, features = ["stream"] }
diff --git a/vortex-bench/src/random_access/mod.rs b/vortex-bench/src/random_access/mod.rs
index 3b9f95f5eaf..f809d0aba7a 100644
--- a/vortex-bench/src/random_access/mod.rs
+++ b/vortex-bench/src/random_access/mod.rs
@@ -1,14 +1,21 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::path::Path;
 use std::path::PathBuf;
+use std::sync::Arc;
 
 use anyhow::Result;
+use anyhow::anyhow;
 use arrow_array::RecordBatch;
 use async_trait::async_trait;
+use object_store::ObjectStore;
+use object_store::aws::AmazonS3Builder;
+use url::Url;
 use vortex::array::ArrayRef;
 
 use crate::Format;
+use crate::data_dir;
 
 pub mod take;
 
@@ -28,6 +35,74 @@ pub fn data_path(dataset: &str, format: Format) -> String {
     }
 }
 
+/// A remote directory holding the same layout as the local benchmark data directory.
+///
+/// Random access datasets are always materialized locally first, then uploaded verbatim, so a
+/// remote object key is just the local path relative to [`data_dir`] appended to the URL path.
+#[derive(Clone, Debug)]
+pub struct RemoteDataDir {
+    url: Url,
+    store: Arc<dyn ObjectStore>,
+}
+
+impl RemoteDataDir {
+    /// Build an object store for `url` (e.g. `s3://bucket/prefix/`) from the ambient environment.
+    pub fn try_new(url: Url) -> Result<Self> {
+        let store: Arc<dyn ObjectStore> = match url.scheme() {
+            "s3" => {
+                let bucket = url
+                    .host_str()
+                    .ok_or_else(|| anyhow!("remote data dir has no bucket: {url}"))?;
+                Arc::new(
+                    AmazonS3Builder::from_env()
+                        .with_bucket_name(bucket)
+                        .build()?,
+                )
+            }
+            other => return Err(anyhow!("unsupported remote data dir scheme: {other}")),
+        };
+        Ok(Self { url, store })
+    }
+
+    /// The object store backing this directory.
+    pub fn store(&self) -> &Arc<dyn ObjectStore> {
+        &self.store
+    }
+
+    /// The object key of `local_path`, mirroring its location under the local data directory.
+    pub fn key(&self, local_path: &Path) -> Result<String> {
+        let relative = local_path.strip_prefix(data_dir()).map_err(|_| {
+            anyhow!(
+                "{} is not inside the benchmark data directory",
+                local_path.display()
+            )
+        })?;
+        let relative = relative
+            .to_str()
+            .ok_or_else(|| anyhow!("non-UTF-8 data path: {}", local_path.display()))?;
+        let prefix = self
+            .url
+            .path()
+            .trim_start_matches('/')
+            .trim_end_matches('/');
+        Ok(if prefix.is_empty() {
+            relative.to_string()
+        } else {
+            format!("{prefix}/{relative}")
+        })
+    }
+
+    /// The fully qualified URL of `local_path` in this remote directory.
+    pub fn uri(&self, local_path: &Path) -> Result<String> {
+        let scheme = self.url.scheme();
+        let host = self
+            .url
+            .host_str()
+            .ok_or_else(|| anyhow!("remote data dir has no bucket: {}", self.url))?;
+        Ok(format!("{scheme}://{host}/{}", self.key(local_path)?))
+    }
+}
+
 /// Trait for a benchmark dataset that knows how to prepare data files.
 #[async_trait]
 pub trait BenchDataset: Send + Sync {
@@ -63,3 +138,48 @@ pub trait RandomAccessor: Send + Sync {
     /// Take rows at the given indices, returning the handle.
     async fn take(&self, indices: &[u64]) -> Result<RandomAccessorRet>;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn remote(url: &str) -> Result<RemoteDataDir> {
+        // `from_env` needs no credentials to construct the client.
+        RemoteDataDir::try_new(Url::parse(url)?)
+    }
+
+    #[test]
+    fn key_mirrors_the_local_data_dir_layout() -> Result<()> {
+        let local = data_dir().join("random_access/taxi/taxi.vortex");
+
+        assert_eq!(
+            remote("s3://bucket/prefix/")?.key(&local)?,
+            "prefix/random_access/taxi/taxi.vortex"
+        );
+        assert_eq!(
+            remote("s3://bucket/")?.key(&local)?,
+            "random_access/taxi/taxi.vortex"
+        );
+        assert_eq!(
+            remote("s3://bucket/prefix/")?.uri(&local)?,
+            "s3://bucket/prefix/random_access/taxi/taxi.vortex"
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn key_rejects_paths_outside_the_data_dir() -> Result<()> {
+        assert!(
+            remote("s3://bucket/prefix/")?
+                .key(Path::new("/tmp/taxi.vortex"))
+                .is_err()
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn unsupported_scheme_is_rejected() -> Result<()> {
+        assert!(RemoteDataDir::try_new(Url::parse("gs://bucket/prefix/")?).is_err());
+        Ok(())
+    }
+}
diff --git a/vortex-bench/src/random_access/take.rs b/vortex-bench/src/random_access/take.rs
index fa941d07362..6e6f0328739 100644
--- a/vortex-bench/src/random_access/take.rs
+++ b/vortex-bench/src/random_access/take.rs
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::iter::once;
+use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -12,9 +13,12 @@ use arrow_select::take::take_record_batch;
 use async_trait::async_trait;
 use futures::stream;
 use itertools::Itertools;
+use object_store::path::Path as ObjectStorePath;
 use parquet::arrow::ParquetRecordBatchStreamBuilder;
 use parquet::arrow::arrow_reader::ArrowReaderMetadata;
 use parquet::arrow::arrow_reader::ArrowReaderOptions;
+use parquet::arrow::async_reader::AsyncFileReader;
+use parquet::arrow::async_reader::ParquetObjectReader;
 use parquet::file::metadata::PageIndexPolicy;
 use stream::StreamExt;
 use tokio::fs::File;
@@ -32,6 +36,7 @@ use crate::Format;
 use crate::SESSION;
 use crate::random_access::RandomAccessor;
 use crate::random_access::RandomAccessorRet;
+use crate::random_access::RemoteDataDir;
 
 /// Random accessor for Vortex format files.
 ///
@@ -45,7 +50,7 @@ pub struct VortexRandomAccessor {
 impl VortexRandomAccessor {
     /// Open a Vortex file and return a ready-to-use accessor.
     pub async fn open(
-        path: impl AsRef<std::path::Path>,
+        path: impl AsRef<Path>,
         name: impl Into<String>,
         format: Format,
     ) -> anyhow::Result<Self> {
@@ -60,6 +65,25 @@ impl VortexRandomAccessor {
             file,
         })
     }
+
+    /// Open a Vortex file stored in an object store and return a ready-to-use accessor.
+    pub async fn open_object_store(
+        remote: &RemoteDataDir,
+        path: &Path,
+        name: impl Into<String>,
+        format: Format,
+    ) -> anyhow::Result<Self> {
+        let file = SESSION
+            .open_options()
+            .with_layout_reader_cache()
+            .open_object_store(remote.store(), &remote.key(path)?)
+            .await?;
+        Ok(Self {
+            name: name.into(),
+            format,
+            file,
+        })
+    }
 }
 
 #[async_trait]
@@ -99,17 +123,49 @@ pub struct ParquetRandomAccessor {
     row_group_offsets: Vec<i64>,
     /// Cached Arrow reader metadata (footer) to avoid re-parsing on each take.
     arrow_metadata: ArrowReaderMetadata,
-    /// Path to the Parquet file (for re-opening on each take).
-    path: PathBuf,
+    /// Where to re-open the file from on each take.
+    source: ParquetSource,
+}
+
+/// Backing store of a [`ParquetRandomAccessor`].
+enum ParquetSource {
+    /// Path to a local Parquet file.
+    Local(PathBuf),
+    /// Reader for a Parquet file held in an object store.
+    Object(ParquetObjectReader),
 }
 
 impl ParquetRandomAccessor {
     /// Open a Parquet file, parse the footer, and return a ready-to-use accessor.
     pub async fn open(path: PathBuf, name: impl Into<String>) -> anyhow::Result<Self> {
         let mut file = File::open(&path).await?;
-        let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required);
-        let arrow_metadata = ArrowReaderMetadata::load_async(&mut file, options).await?;
+        let arrow_metadata = load_metadata(&mut file).await?;
+        Ok(Self::new(name, arrow_metadata, ParquetSource::Local(path)))
+    }
+
+    /// Open a Parquet file stored in an object store and return a ready-to-use accessor.
+    pub async fn open_object_store(
+        remote: &RemoteDataDir,
+        path: &Path,
+        name: impl Into<String>,
+    ) -> anyhow::Result<Self> {
+        let mut reader = ParquetObjectReader::new(
+            Arc::clone(remote.store()),
+            ObjectStorePath::from(remote.key(path)?),
+        );
+        let arrow_metadata = load_metadata(&mut reader).await?;
+        Ok(Self::new(
+            name,
+            arrow_metadata,
+            ParquetSource::Object(reader),
+        ))
+    }
 
+    fn new(
+        name: impl Into<String>,
+        arrow_metadata: ArrowReaderMetadata,
+        source: ParquetSource,
+    ) -> Self {
         let row_group_offsets = once(0)
             .chain(
                 arrow_metadata
@@ -124,15 +180,23 @@ impl ParquetRandomAccessor {
             })
             .collect::<Vec<_>>();
 
-        Ok(Self {
+        Self {
             name: name.into(),
             row_group_offsets,
             arrow_metadata,
-            path,
-        })
+            source,
+        }
     }
 }
 
+/// Parse the Parquet footer, including the page index, from any async reader.
+async fn load_metadata<T: AsyncFileReader + Send>(
+    reader: &mut T,
+) -> anyhow::Result<ArrowReaderMetadata> {
+    let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required);
+    Ok(ArrowReaderMetadata::load_async(reader, options).await?)
+}
+
 #[async_trait]
 impl RandomAccessor for ParquetRandomAccessor {
     fn format(&self) -> Format {
@@ -164,29 +228,60 @@ impl RandomAccessor for ParquetRandomAccessor {
             .collect_vec();
 
         // Re-open the file but reuse cached metadata (avoids re-parsing the footer).
-        let file = File::open(&self.path).await?;
-        let builder =
-            ParquetRecordBatchStreamBuilder::new_with_metadata(file, self.arrow_metadata.clone());
-
-        let reader = builder
-            .with_row_groups(sorted_row_group_keys)
-            // FIXME(ngates): our indices code assumes the batch size == the row group sizes
-            .with_batch_size(10_000_000)
-            .build()?;
-
-        let schema = Arc::clone(reader.schema());
-
-        let batches = reader
-            .enumerate()
-            .map(|(idx, batch)| {
-                let batch = batch.unwrap();
-                let indices = PrimitiveArray::<Int64Type>::from(row_group_indices[idx].clone());
-                take_record_batch(&batch, &indices).unwrap()
-            })
-            .collect::<Vec<_>>()
-            .await;
-
-        let result = concat_batches(&schema, &batches)?;
-        Ok(RandomAccessorRet::RecordBatch(result))
+        match &self.source {
+            ParquetSource::Local(path) => {
+                let file = File::open(path).await?;
+                take_row_groups(
+                    file,
+                    self.arrow_metadata.clone(),
+                    sorted_row_group_keys,
+                    &row_group_indices,
+                )
+                .await
+            }
+            ParquetSource::Object(reader) => {
+                take_row_groups(
+                    reader.clone(),
+                    self.arrow_metadata.clone(),
+                    sorted_row_group_keys,
+                    &row_group_indices,
+                )
+                .await
+            }
+        }
     }
 }
+
+/// Read `row_groups` from `reader` and take `row_group_indices` within each of them.
+async fn take_row_groups<T>(
+    reader: T,
+    metadata: ArrowReaderMetadata,
+    row_groups: Vec<usize>,
+    row_group_indices: &[Vec<i64>],
+) -> anyhow::Result<RandomAccessorRet>
+where
+    T: AsyncFileReader + Unpin + Send + 'static,
+{
+    let builder = ParquetRecordBatchStreamBuilder::new_with_metadata(reader, metadata);
+
+    let reader = builder
+        .with_row_groups(row_groups)
+        // FIXME(ngates): our indices code assumes the batch size == the row group sizes
+        .with_batch_size(10_000_000)
+        .build()?;
+
+    let schema = Arc::clone(reader.schema());
+
+    let batches = reader
+        .enumerate()
+        .map(|(idx, batch)| {
+            let batch = batch.unwrap();
+            let indices = PrimitiveArray::<Int64Type>::from(row_group_indices[idx].clone());
+            take_record_batch(&batch, &indices).unwrap()
+        })
+        .collect::<Vec<_>>()
+        .await;
+
+    let result = concat_batches(&schema, &batches)?;
+    Ok(RandomAccessorRet::RecordBatch(result))
+}

From 552c49afc8e550e02a660d58b542435494e482bc Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 14 Aug 2026 14:42:14 +0000
Subject: [PATCH 2/4] fix(bench): enable Lance's aws feature for S3 random
 access

`lance` is pinned with `default-features = false`, which drops the `aws`
feature it enables by default. Without it Lance has no `s3://` object store
provider registered and opening a remote dataset fails with:

    Invalid user input: No object store provider found for scheme: 's3'

Parquet and Vortex already read from S3 fine; only Lance was affected.

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
---
 Cargo.lock                        | 759 ++++++++++++++++++++++++++++--
 benchmarks/lance-bench/Cargo.toml |   3 +-
 2 files changed, 717 insertions(+), 45 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0b2d77515d7..ebd7271de7a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -16,7 +16,7 @@ checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58"
 dependencies = [
  "cipher",
  "cpubits",
- "cpufeatures",
+ "cpufeatures 0.3.0",
 ]
 
 [[package]]
@@ -830,6 +830,49 @@ version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
+[[package]]
+name = "aws-config"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-sdk-sso",
+ "aws-sdk-ssooidc",
+ "aws-sdk-sts",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "hex",
+ "http 1.5.0",
+ "sha1 0.10.7",
+ "time",
+ "tokio",
+ "tracing",
+ "url",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-credential-types"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "zeroize",
+]
+
 [[package]]
 name = "aws-lc-rs"
 version = "1.17.3"
@@ -853,6 +896,348 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "aws-runtime"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb"
+dependencies = [
+ "aws-credential-types",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "bytes-utils",
+ "fastrand",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "percent-encoding",
+ "pin-project-lite",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "aws-sdk-sso"
+version = "1.105.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-observability",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.5.0",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-ssooidc"
+version = "1.107.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-observability",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.5.0",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-sts"
+version = "1.110.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-observability",
+ "aws-smithy-query",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.5.0",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sigv4"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "form_urlencoded",
+ "hex",
+ "hmac",
+ "http 0.2.12",
+ "http 1.5.0",
+ "percent-encoding",
+ "sha2",
+ "time",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-async"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316"
+dependencies = [
+ "futures-util",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "aws-smithy-http"
+version = "0.64.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "bytes-utils",
+ "futures-core",
+ "futures-util",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "http-body-util",
+ "percent-encoding",
+ "pin-project-lite",
+ "pin-utils",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-http-client"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c1c8a04cb31ba74d0115af5a890bb8c0d48fba64b52812fa13929a6ef0cc83c"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "h2",
+ "http 1.5.0",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "pin-project-lite",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-json"
+version = "0.63.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+]
+
+[[package]]
+name = "aws-smithy-observability"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c"
+dependencies = [
+ "aws-smithy-runtime-api",
+]
+
+[[package]]
+name = "aws-smithy-query"
+version = "0.62.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "urlencoding",
+]
+
+[[package]]
+name = "aws-smithy-runtime"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "483b858ff67522011c4786310c5cd8fd88d0be7ea3d5f1a48328446300c4269e"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-http-client",
+ "aws-smithy-observability",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "bytes",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.5.0",
+ "http-body 0.4.6",
+ "http-body 1.1.0",
+ "http-body-util",
+ "pin-project-lite",
+ "pin-utils",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-runtime-api"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api-macros",
+ "aws-smithy-types",
+ "bytes",
+ "http 0.2.12",
+ "http 1.5.0",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-runtime-api-macros"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "aws-smithy-schema"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "http 1.5.0",
+]
+
+[[package]]
+name = "aws-smithy-types"
+version = "1.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
+dependencies = [
+ "base64-simd",
+ "bytes",
+ "bytes-utils",
+ "http 0.2.12",
+ "http 1.5.0",
+ "http-body 0.4.6",
+ "http-body 1.1.0",
+ "http-body-util",
+ "itoa",
+ "num-integer",
+ "pin-project-lite",
+ "pin-utils",
+ "ryu",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "aws-smithy-xml"
+version = "0.62.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "xmlparser",
+]
+
+[[package]]
+name = "aws-types"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "rustc_version",
+ "tracing",
+]
+
+[[package]]
+name = "backon"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
+dependencies = [
+ "fastrand",
+ "gloo-timers",
+ "tokio",
+]
+
 [[package]]
 name = "base16ct"
 version = "1.0.0"
@@ -871,6 +1256,16 @@ version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
 
+[[package]]
+name = "base64-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
 [[package]]
 name = "beamterm-core"
 version = "1.0.0"
@@ -1020,7 +1415,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures",
+ "cpufeatures 0.3.0",
 ]
 
 [[package]]
@@ -1210,6 +1605,16 @@ version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
+[[package]]
+name = "bytes-utils"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
+dependencies = [
+ "bytes",
+ "either",
+]
+
 [[package]]
 name = "bzip2"
 version = "0.6.1"
@@ -1293,7 +1698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.3.0",
  "rand_core 0.10.1",
 ]
 
@@ -1757,6 +2162,15 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
 
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.3.0"
@@ -1781,6 +2195,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
 
+[[package]]
+name = "crc32c"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
+dependencies = [
+ "rustc_version",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -1933,6 +2356,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ctor"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
+dependencies = [
+ "link-section",
+ "linktime-proc-macro",
+]
+
 [[package]]
 name = "ctutils"
 version = "0.4.2"
@@ -2755,7 +3188,7 @@ dependencies = [
  "percent-encoding",
  "rand 0.9.5",
  "serde_json",
- "sha1",
+ "sha1 0.11.0",
  "sha2",
  "twox-hash",
  "url",
@@ -3657,6 +4090,18 @@ version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
 
+[[package]]
+name = "gloo-timers"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "glow"
 version = "0.17.0"
@@ -3748,7 +4193,7 @@ dependencies = [
  "fnv",
  "futures-core",
  "futures-sink",
- "http",
+ "http 1.5.0",
  "indexmap 2.14.0",
  "slab",
  "tokio",
@@ -3878,6 +4323,17 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "http"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
 [[package]]
 name = "http"
 version = "1.5.0"
@@ -3888,6 +4344,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http-body"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
+dependencies = [
+ "bytes",
+ "http 0.2.12",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "http-body"
 version = "1.1.0"
@@ -3895,7 +4362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
 dependencies = [
  "bytes",
- "http",
+ "http 1.5.0",
 ]
 
 [[package]]
@@ -3906,8 +4373,8 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
 dependencies = [
  "bytes",
  "futures-core",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "pin-project-lite",
 ]
 
@@ -3958,8 +4425,8 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "h2",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "httparse",
  "httpdate",
  "itoa",
@@ -3975,7 +4442,7 @@ version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
- "http",
+ "http 1.5.0",
  "hyper",
  "hyper-util",
  "rustls",
@@ -4009,8 +4476,8 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "hyper",
  "ipnet",
  "libc",
@@ -4675,6 +5142,7 @@ dependencies = [
  "async-recursion",
  "async-trait",
  "async_cell",
+ "aws-credential-types",
  "byteorder",
  "bytes",
  "chrono",
@@ -5086,11 +5554,13 @@ dependencies = [
  "arrow-array 58.4.0",
  "arrow-schema 58.4.0",
  "async-trait",
+ "aws-config",
+ "aws-credential-types",
  "byteorder",
  "bytes",
  "chrono",
  "futures",
- "http",
+ "http 1.5.0",
  "io-uring",
  "lance-arrow",
  "lance-core",
@@ -5098,6 +5568,8 @@ dependencies = [
  "log",
  "moka",
  "object_store",
+ "object_store_opendal 0.57.0",
+ "opendal 0.57.0",
  "path_abs",
  "pin-project",
  "prost 0.14.4",
@@ -5438,6 +5910,18 @@ dependencies = [
  "bitflags 2.13.1",
 ]
 
+[[package]]
+name = "link-section"
+version = "0.19.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9"
+
+[[package]]
+name = "linktime-proc-macro"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -6049,7 +6533,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "http",
+ "http 1.5.0",
  "http-body-util",
  "httparse",
  "humantime",
@@ -6075,6 +6559,23 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "object_store_opendal"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "chrono",
+ "futures",
+ "mea",
+ "object_store",
+ "opendal 0.57.0",
+ "pin-project",
+ "tokio",
+]
+
 [[package]]
 name = "object_store_opendal"
 version = "0.58.0"
@@ -6087,7 +6588,7 @@ dependencies = [
  "futures",
  "mea",
  "object_store",
- "opendal",
+ "opendal 0.58.1",
  "pin-project",
  "tokio",
 ]
@@ -6129,18 +6630,61 @@ version = "11.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
 
+[[package]]
+name = "opendal"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
+dependencies = [
+ "ctor",
+ "opendal-core 0.57.0",
+ "opendal-layer-concurrent-limit",
+ "opendal-layer-logging",
+ "opendal-layer-retry",
+ "opendal-layer-timeout",
+ "opendal-service-s3",
+]
+
 [[package]]
 name = "opendal"
 version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
 dependencies = [
- "opendal-core",
+ "opendal-core 0.58.1",
  "opendal-service-cos",
  "opendal-service-goosefs",
  "opendal-service-oss",
 ]
 
+[[package]]
+name = "opendal-core"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bytes",
+ "futures",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "jiff",
+ "log",
+ "md-5 0.11.0",
+ "mea",
+ "percent-encoding",
+ "quick-xml 0.39.4",
+ "reqsign-core",
+ "reqwest 0.13.4",
+ "serde",
+ "serde_json",
+ "tokio",
+ "url",
+ "uuid",
+ "web-time",
+]
+
 [[package]]
 name = "opendal-core"
 version = "0.58.1"
@@ -6151,7 +6695,7 @@ dependencies = [
  "base64 0.23.0",
  "bytes",
  "futures",
- "http",
+ "http 1.5.0",
  "jiff",
  "log",
  "md-5 0.11.0",
@@ -6167,6 +6711,49 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "opendal-layer-concurrent-limit"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
+dependencies = [
+ "futures",
+ "http 1.5.0",
+ "mea",
+ "opendal-core 0.57.0",
+]
+
+[[package]]
+name = "opendal-layer-logging"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
+dependencies = [
+ "log",
+ "opendal-core 0.57.0",
+]
+
+[[package]]
+name = "opendal-layer-retry"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
+dependencies = [
+ "backon",
+ "log",
+ "opendal-core 0.57.0",
+]
+
+[[package]]
+name = "opendal-layer-timeout"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
+dependencies = [
+ "opendal-core 0.57.0",
+ "tokio",
+]
+
 [[package]]
 name = "opendal-service-cos"
 version = "0.58.1"
@@ -6174,9 +6761,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0"
 dependencies = [
  "bytes",
- "http",
+ "http 1.5.0",
  "log",
- "opendal-core",
+ "opendal-core 0.58.1",
  "quick-xml 0.41.0",
  "reqsign-core",
  "reqsign-file-read-tokio",
@@ -6193,7 +6780,7 @@ dependencies = [
  "bytes",
  "goosefs-sdk",
  "log",
- "opendal-core",
+ "opendal-core 0.58.1",
  "serde",
  "tokio",
 ]
@@ -6205,9 +6792,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b"
 dependencies = [
  "bytes",
- "http",
+ "http 1.5.0",
  "log",
- "opendal-core",
+ "opendal-core 0.58.1",
  "quick-xml 0.41.0",
  "reqsign-aliyun-oss",
  "reqsign-core",
@@ -6215,6 +6802,27 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "opendal-service-s3"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "crc32c",
+ "http 1.5.0",
+ "log",
+ "md-5 0.11.0",
+ "opendal-core 0.57.0",
+ "quick-xml 0.39.4",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+ "url",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -6243,7 +6851,7 @@ checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
 dependencies = [
  "async-trait",
  "bytes",
- "http",
+ "http 1.5.0",
  "opentelemetry",
  "reqwest 0.13.4",
 ]
@@ -6254,7 +6862,7 @@ version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
 dependencies = [
- "http",
+ "http 1.5.0",
  "opentelemetry",
  "opentelemetry-http",
  "opentelemetry-proto",
@@ -6325,6 +6933,12 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
+[[package]]
+name = "outref"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
+
 [[package]]
 name = "owo-colors"
 version = "4.3.0"
@@ -6664,6 +7278,12 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
 [[package]]
 name = "ping"
 version = "0.7.1"
@@ -7034,7 +7654,7 @@ dependencies = [
  "bytes",
  "chrono",
  "futures",
- "http",
+ "http 1.5.0",
  "humantime",
  "itertools 0.14.0",
  "object_store",
@@ -7535,13 +8155,35 @@ checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f"
 dependencies = [
  "anyhow",
  "form_urlencoded",
- "http",
+ "http 1.5.0",
+ "log",
+ "percent-encoding",
+ "reqsign-core",
+ "rust-ini",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "reqsign-aws-v4"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "form_urlencoded",
+ "hex",
+ "http 1.5.0",
  "log",
  "percent-encoding",
+ "quick-xml 0.41.0",
  "reqsign-core",
  "rust-ini",
  "serde",
  "serde_json",
+ "serde_urlencoded",
+ "sha1 0.11.0",
 ]
 
 [[package]]
@@ -7556,11 +8198,11 @@ dependencies = [
  "futures",
  "hex",
  "hmac",
- "http",
+ "http 1.5.0",
  "jiff",
  "log",
  "percent-encoding",
- "sha1",
+ "sha1 0.11.0",
  "sha2",
  "windows-sys 0.61.2",
 ]
@@ -7583,7 +8225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644"
 dependencies = [
  "anyhow",
- "http",
+ "http 1.5.0",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -7603,8 +8245,8 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "http-body-util",
  "hyper",
  "hyper-rustls",
@@ -7650,8 +8292,8 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "http-body-util",
  "hyper",
  "hyper-rustls",
@@ -8196,6 +8838,17 @@ dependencies = [
  "syn 3.0.3",
 ]
 
+[[package]]
+name = "sha1"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.2.17",
+ "digest 0.10.7",
+]
+
 [[package]]
 name = "sha1"
 version = "0.11.0"
@@ -8203,7 +8856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.3.0",
  "digest 0.11.3",
 ]
 
@@ -8214,7 +8867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.3.0",
  "digest 0.11.3",
 ]
 
@@ -9148,8 +9801,8 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "h2",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "http-body-util",
  "hyper",
  "hyper-timeout",
@@ -9207,8 +9860,8 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http",
- "http-body",
+ "http 1.5.0",
+ "http-body 1.1.0",
  "http-body-util",
  "pin-project-lite",
  "tokio",
@@ -9459,6 +10112,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "urlencoding"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
+
 [[package]]
 name = "utf8-ranges"
 version = "1.0.5"
@@ -9820,10 +10479,10 @@ dependencies = [
 name = "vortex-cloud"
 version = "0.1.0"
 dependencies = [
- "http",
+ "http 1.5.0",
  "object_store",
- "object_store_opendal",
- "opendal",
+ "object_store_opendal 0.58.0",
+ "opendal 0.58.1",
  "parking_lot",
  "percent-encoding",
  "rstest",
@@ -10774,6 +11433,12 @@ dependencies = [
  "zstd",
 ]
 
+[[package]]
+name = "vsimd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"
@@ -11262,6 +11927,12 @@ dependencies = [
  "rustix",
 ]
 
+[[package]]
+name = "xmlparser"
+version = "0.13.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
+
 [[package]]
 name = "xshell"
 version = "0.2.7"
@@ -11432,7 +12103,7 @@ dependencies = [
  "memchr",
  "pbkdf2",
  "ppmd-rust",
- "sha1",
+ "sha1 0.11.0",
  "time",
  "typed-path",
  "zeroize",
diff --git a/benchmarks/lance-bench/Cargo.toml b/benchmarks/lance-bench/Cargo.toml
index 37222e74452..1ab826346d3 100644
--- a/benchmarks/lance-bench/Cargo.toml
+++ b/benchmarks/lance-bench/Cargo.toml
@@ -15,7 +15,8 @@ version.workspace = true
 publish = false
 
 [dependencies]
-lance = { version = "10", default-features = false }
+# `aws` registers Lance's s3:// object store provider, needed to benchmark against remote data.
+lance = { version = "10", default-features = false, features = ["aws"] }
 lance-encoding = { version = "10" }
 
 anyhow = { workspace = true }

From c8e7846c886fea430fb385e3bbad1cc15530179e Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 14 Aug 2026 17:59:17 +0000
Subject: [PATCH 3/4] fix(bench): size random-access row groups so lookups
 don't read whole files

Every synthetic random-access dataset holds 1,000,000 rows and was written
with default writer properties, whose `max_row_group_size` is 1Mi rows. A
million rows never crosses that threshold, so each file is a single row
group. Readers select row groups before rows, so a point lookup fetches and
decodes the entire file: masked by page cache locally, ruinous over an object
store, where one `take` on feature-vectors spends ~87s moving ~4GB.

The tell is in the measurements: nested-lists and nested-structs take the
same time under both access patterns to within 0.04%, because the indices
never change what is read.

Size row groups from the row width instead, targeting 128MiB and clamping to
8-64 batches of 1024 rows. The clamp is what fixes narrow rows: a pure byte
budget would still leave nested-lists in one group. Sizes stay whole
multiples of the 1024-row Arrow batches that `parquet_to_vortex_chunks`
streams, so row group boundaries never split a batch and the derived Vortex
files are unchanged -- only Parquet layout moves. Data pages also drop from
20k to 1024 rows, giving the page index resolution worth having here.

Scan-oriented generators (TPC-H, SpatialBench, PolarSignals, ...) keep large
row groups, which is right for full scans.

This shifts Parquet random-access baselines once, most visibly on the
correlated pattern. Uniform lookups still touch most row groups; only
page-level row selection in the reader addresses those.

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
---
 vortex-bench/src/datasets/feature_vectors.rs | 10 ++-
 vortex-bench/src/datasets/nested_lists.rs    | 10 ++-
 vortex-bench/src/datasets/nested_structs.rs  | 10 ++-
 vortex-bench/src/random_access/mod.rs        | 76 ++++++++++++++++++++
 4 files changed, 103 insertions(+), 3 deletions(-)

diff --git a/vortex-bench/src/datasets/feature_vectors.rs b/vortex-bench/src/datasets/feature_vectors.rs
index e09f29f49ec..3ffc2709cc8 100644
--- a/vortex-bench/src/datasets/feature_vectors.rs
+++ b/vortex-bench/src/datasets/feature_vectors.rs
@@ -25,6 +25,7 @@ use crate::conversions::write_parquet_as_vortex;
 use crate::idempotent_async;
 use crate::random_access::BenchDataset;
 use crate::random_access::data_path;
+use crate::random_access::random_access_writer_properties;
 
 /// Dataset identifier used for data path generation.
 pub const DATASET: &str = "feature_vectors";
@@ -60,6 +61,9 @@ const VECTOR_DIM: i32 = 1024;
 /// Batch size for data generation.
 const BATCH_SIZE: usize = 100_000;
 
+/// Approximate encoded size of one row: `id` plus `VECTOR_DIM` uncompressed `f32` values.
+const APPROX_ROW_BYTES: usize = 8 + (VECTOR_DIM as usize) * 4;
+
 /// Generate a synthetic feature vectors parquet file.
 ///
 /// Schema: `id: Int64, embedding: FixedSizeList<Float32, VECTOR_DIM>`.
@@ -81,7 +85,11 @@ pub async fn feature_vectors_parquet() -> Result<PathBuf> {
             ]));
 
             let file = File::create(&temp_path)?;
-            let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None)?;
+            let mut writer = ArrowWriter::try_new(
+                file,
+                Arc::clone(&schema),
+                Some(random_access_writer_properties(APPROX_ROW_BYTES)),
+            )?;
             let mut rng = StdRng::seed_from_u64(42);
 
             for batch_start in (0..ROW_COUNT).step_by(BATCH_SIZE) {
diff --git a/vortex-bench/src/datasets/nested_lists.rs b/vortex-bench/src/datasets/nested_lists.rs
index 3c89ed8eb2b..c45de2ecee7 100644
--- a/vortex-bench/src/datasets/nested_lists.rs
+++ b/vortex-bench/src/datasets/nested_lists.rs
@@ -24,6 +24,7 @@ use crate::conversions::write_parquet_as_vortex;
 use crate::idempotent_async;
 use crate::random_access::BenchDataset;
 use crate::random_access::data_path;
+use crate::random_access::random_access_writer_properties;
 
 /// Dataset identifier used for data path generation.
 pub const DATASET: &str = "nested_lists";
@@ -59,6 +60,9 @@ const MAX_LIST_LEN: usize = 20;
 /// Batch size for data generation.
 const BATCH_SIZE: usize = 100_000;
 
+/// Approximate encoded size of one row: `id` plus an average-length list of `i64` values.
+const APPROX_ROW_BYTES: usize = 8 + (MAX_LIST_LEN / 2) * 8;
+
 /// Generate a synthetic nested lists parquet file.
 ///
 /// Schema: `id: Int64, values: List<Int64>`.
@@ -77,7 +81,11 @@ pub async fn nested_lists_parquet() -> Result<PathBuf> {
             ]));
 
             let file = std::fs::File::create(&temp_path)?;
-            let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None)?;
+            let mut writer = ArrowWriter::try_new(
+                file,
+                Arc::clone(&schema),
+                Some(random_access_writer_properties(APPROX_ROW_BYTES)),
+            )?;
             let mut rng = StdRng::seed_from_u64(42);
 
             for batch_start in (0..ROW_COUNT).step_by(BATCH_SIZE) {
diff --git a/vortex-bench/src/datasets/nested_structs.rs b/vortex-bench/src/datasets/nested_structs.rs
index c4a522184d3..76c9b9575b1 100644
--- a/vortex-bench/src/datasets/nested_structs.rs
+++ b/vortex-bench/src/datasets/nested_structs.rs
@@ -25,6 +25,7 @@ use crate::conversions::write_parquet_as_vortex;
 use crate::idempotent_async;
 use crate::random_access::BenchDataset;
 use crate::random_access::data_path;
+use crate::random_access::random_access_writer_properties;
 
 /// Dataset identifier used for data path generation.
 pub const DATASET: &str = "nested_structs";
@@ -57,6 +58,9 @@ impl BenchDataset for NestedStructsData {
 /// Batch size for data generation.
 const BATCH_SIZE: usize = 100_000;
 
+/// Approximate encoded size of one row: `id`, `a`, `b`, and the three inner `f64` fields.
+const APPROX_ROW_BYTES: usize = 8 * 6;
+
 /// Generate a synthetic nested structs parquet file.
 ///
 /// Schema:
@@ -92,7 +96,11 @@ pub async fn nested_structs_parquet() -> Result<PathBuf> {
             ]));
 
             let file = std::fs::File::create(&temp_path)?;
-            let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None)?;
+            let mut writer = ArrowWriter::try_new(
+                file,
+                Arc::clone(&schema),
+                Some(random_access_writer_properties(APPROX_ROW_BYTES)),
+            )?;
             let mut rng = StdRng::seed_from_u64(42);
 
             for batch_start in (0..ROW_COUNT).step_by(BATCH_SIZE) {
diff --git a/vortex-bench/src/random_access/mod.rs b/vortex-bench/src/random_access/mod.rs
index f809d0aba7a..8fbb5c6e874 100644
--- a/vortex-bench/src/random_access/mod.rs
+++ b/vortex-bench/src/random_access/mod.rs
@@ -11,6 +11,7 @@ use arrow_array::RecordBatch;
 use async_trait::async_trait;
 use object_store::ObjectStore;
 use object_store::aws::AmazonS3Builder;
+use parquet::file::properties::WriterProperties;
 use url::Url;
 use vortex::array::ArrayRef;
 
@@ -35,6 +36,45 @@ pub fn data_path(dataset: &str, format: Format) -> String {
     }
 }
 
+/// Approximate byte budget for one row group in the synthetic random-access datasets.
+const TARGET_ROW_GROUP_BYTES: usize = 128 * 1024 * 1024;
+
+/// Row groups are sized in whole multiples of this many rows.
+///
+/// Parquet is converted to Vortex by streaming Arrow batches of [`PARQUET_READ_BATCH_SIZE`] rows,
+/// so keeping row groups a whole multiple of it leaves the derived Vortex files byte-identical:
+/// only the Parquet layout changes.
+const PARQUET_READ_BATCH_SIZE: usize = 1024;
+
+/// Bounds on the row group size, in units of [`PARQUET_READ_BATCH_SIZE`] rows.
+///
+/// The upper bound matters more than the byte budget for narrow rows: without it a million-row
+/// dataset lands in a single row group, and a point lookup then has to read the entire file.
+const MIN_BATCHES_PER_ROW_GROUP: usize = 8;
+const MAX_BATCHES_PER_ROW_GROUP: usize = 64;
+
+/// Rows per data page.
+///
+/// Finer pages than the 20k-row default give the page index enough resolution to be useful for
+/// point lookups, at the cost of a slightly larger index.
+const DATA_PAGE_ROWS: usize = PARQUET_READ_BATCH_SIZE;
+
+/// Parquet writer properties for a synthetic random-access dataset of `approx_row_bytes` per row.
+///
+/// The defaults are wrong for this suite: `max_row_group_size` defaults to 1Mi rows, which is more
+/// than every dataset here holds, so each file ends up as one row group. Readers select row groups
+/// before rows, so a single-row-group file forces a point lookup to fetch and decode the whole
+/// file — cheap from page cache, ruinous over an object store.
+pub fn random_access_writer_properties(approx_row_bytes: usize) -> WriterProperties {
+    let batches = (TARGET_ROW_GROUP_BYTES / approx_row_bytes / PARQUET_READ_BATCH_SIZE)
+        .clamp(MIN_BATCHES_PER_ROW_GROUP, MAX_BATCHES_PER_ROW_GROUP);
+
+    WriterProperties::builder()
+        .set_max_row_group_size(batches * PARQUET_READ_BATCH_SIZE)
+        .set_data_page_row_count_limit(DATA_PAGE_ROWS)
+        .build()
+}
+
 /// A remote directory holding the same layout as the local benchmark data directory.
 ///
 /// Random access datasets are always materialized locally first, then uploaded verbatim, so a
@@ -141,6 +181,8 @@ pub trait RandomAccessor: Send + Sync {
 
 #[cfg(test)]
 mod tests {
+    use rstest::rstest;
+
     use super::*;
 
     fn remote(url: &str) -> Result<RemoteDataDir> {
@@ -177,6 +219,40 @@ mod tests {
         Ok(())
     }
 
+    /// Row widths of the three synthetic random-access datasets.
+    const FEATURE_VECTORS_ROW_BYTES: usize = 8 + 1024 * 4;
+    const NESTED_LISTS_ROW_BYTES: usize = 8 + 10 * 8;
+    const NESTED_STRUCTS_ROW_BYTES: usize = 8 * 6;
+
+    #[rstest]
+    #[case::feature_vectors(FEATURE_VECTORS_ROW_BYTES)]
+    #[case::nested_lists(NESTED_LISTS_ROW_BYTES)]
+    #[case::nested_structs(NESTED_STRUCTS_ROW_BYTES)]
+    fn row_groups_split_a_million_row_dataset(#[case] approx_row_bytes: usize) {
+        let props = random_access_writer_properties(approx_row_bytes);
+        let rows_per_group = props.max_row_group_size();
+
+        // The whole point: a million-row dataset must not land in a single row group.
+        assert!(
+            rows_per_group < 1_000_000,
+            "{approx_row_bytes} byte rows produced one row group of {rows_per_group}"
+        );
+        // Row group boundaries stay aligned to the Arrow batches the Vortex conversion reads,
+        // so the derived Vortex files are unaffected by this layout.
+        assert_eq!(rows_per_group % PARQUET_READ_BATCH_SIZE, 0);
+        assert!(rows_per_group * approx_row_bytes <= TARGET_ROW_GROUP_BYTES);
+    }
+
+    #[test]
+    fn wide_rows_get_smaller_row_groups_than_narrow_rows() {
+        let wide = random_access_writer_properties(FEATURE_VECTORS_ROW_BYTES).max_row_group_size();
+        let narrow = random_access_writer_properties(NESTED_STRUCTS_ROW_BYTES).max_row_group_size();
+        assert!(
+            wide < narrow,
+            "wide {wide} should be smaller than narrow {narrow}"
+        );
+    }
+
     #[test]
     fn unsupported_scheme_is_rejected() -> Result<()> {
         assert!(RemoteDataDir::try_new(Url::parse("gs://bucket/prefix/")?).is_err());

From 8692404c873d8356face18a723bfa0899f9749bd Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 14 Aug 2026 18:06:17 +0000
Subject: [PATCH 4/4] fix(bench): use the non-deprecated row group row count
 setter

`set_max_row_group_size` is deprecated in favour of
`set_max_row_group_row_count`, which takes an `Option<usize>` where `None`
means unlimited. The deprecation warning fails the lint job.

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
---
 vortex-bench/src/random_access/mod.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/vortex-bench/src/random_access/mod.rs b/vortex-bench/src/random_access/mod.rs
index 8fbb5c6e874..7499ae8c14d 100644
--- a/vortex-bench/src/random_access/mod.rs
+++ b/vortex-bench/src/random_access/mod.rs
@@ -61,7 +61,7 @@ const DATA_PAGE_ROWS: usize = PARQUET_READ_BATCH_SIZE;
 
 /// Parquet writer properties for a synthetic random-access dataset of `approx_row_bytes` per row.
 ///
-/// The defaults are wrong for this suite: `max_row_group_size` defaults to 1Mi rows, which is more
+/// The defaults are wrong for this suite: the max row group row count defaults to 1Mi rows, which is more
 /// than every dataset here holds, so each file ends up as one row group. Readers select row groups
 /// before rows, so a single-row-group file forces a point lookup to fetch and decode the whole
 /// file — cheap from page cache, ruinous over an object store.
@@ -70,7 +70,7 @@ pub fn random_access_writer_properties(approx_row_bytes: usize) -> WriterPropert
         .clamp(MIN_BATCHES_PER_ROW_GROUP, MAX_BATCHES_PER_ROW_GROUP);
 
     WriterProperties::builder()
-        .set_max_row_group_size(batches * PARQUET_READ_BATCH_SIZE)
+        .set_max_row_group_row_count(Some(batches * PARQUET_READ_BATCH_SIZE))
         .set_data_page_row_count_limit(DATA_PAGE_ROWS)
         .build()
 }
@@ -219,6 +219,11 @@ mod tests {
         Ok(())
     }
 
+    /// Rows per row group, treating "unlimited" as the whole file.
+    fn rows_per_row_group(props: &WriterProperties) -> usize {
+        props.max_row_group_row_count().unwrap_or(usize::MAX)
+    }
+
     /// Row widths of the three synthetic random-access datasets.
     const FEATURE_VECTORS_ROW_BYTES: usize = 8 + 1024 * 4;
     const NESTED_LISTS_ROW_BYTES: usize = 8 + 10 * 8;
@@ -230,7 +235,7 @@ mod tests {
     #[case::nested_structs(NESTED_STRUCTS_ROW_BYTES)]
     fn row_groups_split_a_million_row_dataset(#[case] approx_row_bytes: usize) {
         let props = random_access_writer_properties(approx_row_bytes);
-        let rows_per_group = props.max_row_group_size();
+        let rows_per_group = rows_per_row_group(&props);
 
         // The whole point: a million-row dataset must not land in a single row group.
         assert!(
@@ -245,8 +250,8 @@ mod tests {
 
     #[test]
     fn wide_rows_get_smaller_row_groups_than_narrow_rows() {
-        let wide = random_access_writer_properties(FEATURE_VECTORS_ROW_BYTES).max_row_group_size();
-        let narrow = random_access_writer_properties(NESTED_STRUCTS_ROW_BYTES).max_row_group_size();
+        let wide = rows_per_row_group(&random_access_writer_properties(FEATURE_VECTORS_ROW_BYTES));
+        let narrow = rows_per_row_group(&random_access_writer_properties(NESTED_STRUCTS_ROW_BYTES));
         assert!(
             wide < narrow,
             "wide {wide} should be smaller than narrow {narrow}"