From cefe3c9331fbb732f1fef7c758430fb6e20004f1 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 25 Sep 2026 10:09:14 -0400 Subject: [PATCH 1/5] Add list_contains membership benchmarks Benchmark constant-set membership before introducing prepared-set probes. Cover dense and random integers, UTF-8, decimals, nested lists, large sets, and flat and chunked needles using the same workloads for comparisons. Signed-off-by: Robert Kruszewski --- vortex-array/Cargo.toml | 4 + vortex-array/benches/list_contains_set.rs | 224 ++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 vortex-array/benches/list_contains_set.rs diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 319fc21e049..3a0bc8419bc 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -156,6 +156,10 @@ harness = false name = "like" harness = false +[[bench]] +name = "list_contains_set" +harness = false + [[bench]] name = "interleave" harness = false diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-array/benches/list_contains_set.rs new file mode 100644 index 00000000000..94d3a11c3c6 --- /dev/null +++ b/vortex-array/benches/list_contains_set.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! `list_contains(lit([...]), column)`, the `IN` shape: a constant set probed by a column of +//! needles, flat and split into chunks so that the cost of preparing the set shows next to the +//! cost of probing it. + +#![expect(clippy::unwrap_used)] + +use std::sync::Arc; + +use divan::Bencher; +use rand::RngExt; +use rand::SeedableRng; +use rand::prelude::StdRng; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::expr::list_contains; +use vortex_array::expr::lit; +use vortex_array::expr::root; +use vortex_array::scalar::Scalar; + +fn main() { + divan::main(); +} + +// Sized to keep CodSpeed simulation under 1ms per benchmark. +const ROWS: usize = 4_096; +const CHUNKS: usize = 16; +const SET_LENS: &[usize] = &[4, 64, 1_024]; + +/// A random set of `len` values, and needles of which about half are members. +fn random_i64(len: usize) -> (Vec, Vec) { + let mut rng = StdRng::seed_from_u64(0); + let set: Vec = (0..len).map(|_| rng.random_range(0..1 << 40)).collect(); + let needles = (0..ROWS) + .map(|_| { + if rng.random_bool(0.5) { + set[rng.random_range(0..len)] + } else { + rng.random_range(0..1 << 40) + } + }) + .collect(); + (set, needles) +} + +/// The set `0..len`, and needles drawn from twice that range. +fn dense_i64(len: usize) -> (Vec, Vec) { + let mut rng = StdRng::seed_from_u64(0); + let set = (0..len as i64).collect(); + let needles = (0..ROWS) + .map(|_| rng.random_range(0..2 * len as i64)) + .collect(); + (set, needles) +} + +fn i64_set(values: &[i64]) -> Scalar { + Scalar::list( + Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)), + values.iter().map(|&v| v.into()).collect(), + Nullability::NonNullable, + ) +} + +fn utf8_set(values: &[String]) -> Scalar { + Scalar::list( + Arc::new(DType::Utf8(Nullability::NonNullable)), + values + .iter() + .map(|v| Scalar::utf8(v.as_str(), Nullability::NonNullable)) + .collect(), + Nullability::NonNullable, + ) +} + +fn chunked(needles: &[ArrayRef]) -> ArrayRef { + let dtype = needles[0].dtype().clone(); + ChunkedArray::try_new(needles.iter().cloned(), dtype) + .unwrap() + .into_array() +} + +fn i64_needles(needles: &[i64], chunks: usize) -> ArrayRef { + let parts: Vec = needles + .chunks(needles.len() / chunks) + .map(|chunk| PrimitiveArray::from_iter(chunk.iter().copied()).into_array()) + .collect(); + if chunks == 1 { + parts[0].clone() + } else { + chunked(&parts) + } +} + +fn utf8_needles(needles: &[String], chunks: usize) -> ArrayRef { + let parts: Vec = needles + .chunks(needles.len() / chunks) + .map(|chunk| VarBinViewArray::from_iter_str(chunk.iter().map(String::as_str)).into_array()) + .collect(); + if chunks == 1 { + parts[0].clone() + } else { + chunked(&parts) + } +} + +fn bench_in_set(bencher: Bencher, set: Scalar, needles: ArrayRef) { + let session = vortex_array::array_session(); + // Optimized as a scan optimizes it, so the set arrives normalized. + let expr = list_contains(lit(set), root()) + .optimize_recursive(needles.dtype()) + .unwrap(); + bencher + .with_inputs(|| { + ( + needles.clone().apply(&expr).unwrap(), + session.create_execution_ctx(), + ) + }) + .bench_values(|(array, mut ctx)| array.execute::(&mut ctx).unwrap()); +} + +#[divan::bench(args = SET_LENS)] +fn i64_random(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); +} + +#[divan::bench(args = SET_LENS)] +fn i64_random_chunked(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + bench_in_set(bencher, i64_set(&set), i64_needles(&needles, CHUNKS)); +} + +#[divan::bench(args = SET_LENS)] +fn i64_dense(bencher: Bencher, set_len: usize) { + let (set, needles) = dense_i64(set_len); + bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); +} + +#[divan::bench(args = SET_LENS)] +fn utf8_random(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + let set: Vec = set.iter().map(|v| format!("value-{v}")).collect(); + let needles: Vec = needles.iter().map(|v| format!("value-{v}")).collect(); + bench_in_set(bencher, utf8_set(&set), utf8_needles(&needles, 1)); +} + +#[divan::bench(args = SET_LENS)] +fn utf8_random_chunked(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + let set: Vec = set.iter().map(|v| format!("value-{v}")).collect(); + let needles: Vec = needles.iter().map(|v| format!("value-{v}")).collect(); + bench_in_set(bencher, utf8_set(&set), utf8_needles(&needles, CHUNKS)); +} + +#[divan::bench(args = [4_096, 16_384])] +fn i64_random_large(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); +} + +#[divan::bench(args = SET_LENS)] +fn decimal_random(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + bench_decimal(bencher, set, needles); +} + +#[divan::bench(args = SET_LENS)] +fn decimal_dense(bencher: Bencher, set_len: usize) { + let (set, needles) = dense_i64(set_len); + bench_decimal(bencher, set, needles); +} + +fn bench_decimal(bencher: Bencher, set: Vec, needles: Vec) { + let dtype = DecimalDType::new(20, 2); + let set = Scalar::list( + DType::Decimal(dtype, Nullability::NonNullable), + set.into_iter() + .map(|value| Scalar::decimal(value.into(), dtype, Nullability::NonNullable)) + .collect(), + Nullability::NonNullable, + ); + let needles = DecimalArray::from_iter::(needles, dtype).into_array(); + bench_in_set(bencher, set, needles); +} + +#[divan::bench(args = SET_LENS)] +fn nested_list_random(bencher: Bencher, set_len: usize) { + let (set, needles) = random_i64(set_len); + let element_dtype = Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)); + let set = Scalar::list( + DType::List(element_dtype.clone(), Nullability::NonNullable), + set.into_iter() + .map(|value| { + Scalar::list( + element_dtype.clone(), + vec![value.into(), (value + 1).into()], + Nullability::NonNullable, + ) + }) + .collect(), + Nullability::NonNullable, + ); + let needles = ListArray::from_iter_slow::( + needles.into_iter().map(|value| vec![value, value + 1]), + element_dtype, + ) + .unwrap() + .into_array(); + bench_in_set(bencher, set, needles); +} From 9ca1a78ac0e31a07d80d6d82f500ec57932a8766 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 25 Sep 2026 10:59:41 -0400 Subject: [PATCH 2/5] Fix Arc cloning in membership benchmarks Signed-off-by: Robert Kruszewski --- vortex-array/benches/list_contains_set.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-array/benches/list_contains_set.rs index 94d3a11c3c6..c132235e7de 100644 --- a/vortex-array/benches/list_contains_set.rs +++ b/vortex-array/benches/list_contains_set.rs @@ -202,11 +202,11 @@ fn nested_list_random(bencher: Bencher, set_len: usize) { let (set, needles) = random_i64(set_len); let element_dtype = Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)); let set = Scalar::list( - DType::List(element_dtype.clone(), Nullability::NonNullable), + DType::List(Arc::clone(&element_dtype), Nullability::NonNullable), set.into_iter() .map(|value| { Scalar::list( - element_dtype.clone(), + Arc::clone(&element_dtype), vec![value.into(), (value + 1).into()], Nullability::NonNullable, ) From f52b1a31e86cb08597b13b78b61de2e4076970ac Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 28 Sep 2026 18:13:23 -0400 Subject: [PATCH 3/5] fixes Signed-off-by: Robert Kruszewski --- vortex-array/benches/list_contains_set.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-array/benches/list_contains_set.rs index c132235e7de..33e8665cb3b 100644 --- a/vortex-array/benches/list_contains_set.rs +++ b/vortex-array/benches/list_contains_set.rs @@ -120,12 +120,14 @@ fn bench_in_set(bencher: Bencher, set: Scalar, needles: ArrayRef) { let session = vortex_array::array_session(); // Optimized as a scan optimizes it, so the set arrives normalized. let expr = list_contains(lit(set), root()) - .optimize_recursive(needles.dtype()) + .bind(needles.dtype()) + .unwrap() + .optimize_recursive() .unwrap(); bencher .with_inputs(|| { ( - needles.clone().apply(&expr).unwrap(), + needles.clone().apply_bound(&expr).unwrap(), session.create_execution_ctx(), ) }) From 89feff430dacf3e82b4742616cc8eb426241a24b Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 28 Sep 2026 23:16:39 -0400 Subject: [PATCH 4/5] Trim list_contains membership benchmarks Keep one benchmark per probe kind, integers, UTF-8 strings, and nested lists, each at a small and a large set, plus integer needles split into chunks, so that every benchmark stays under 1ms of CodSpeed simulation once constant sets are prepared as probes. Nested sets compare whole rows and stay smaller. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Robert Kruszewski --- vortex-array/benches/list_contains_set.rs | 148 +++++----------------- 1 file changed, 34 insertions(+), 114 deletions(-) diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-array/benches/list_contains_set.rs index 33e8665cb3b..87b2057f9c2 100644 --- a/vortex-array/benches/list_contains_set.rs +++ b/vortex-array/benches/list_contains_set.rs @@ -2,8 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! `list_contains(lit([...]), column)`, the `IN` shape: a constant set probed by a column of -//! needles, flat and split into chunks so that the cost of preparing the set shows next to the -//! cost of probing it. +//! needles, so that the cost of preparing the set shows next to the cost of probing it. +//! +//! Each element type exercises one probe: integers, UTF-8 strings, and nested lists that compare +//! whole rows. Integer needles are also split into chunks, which share one prepared set. #![expect(clippy::unwrap_used)] @@ -18,12 +20,10 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ChunkedArray; -use vortex_array::arrays::DecimalArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; -use vortex_array::dtype::DecimalDType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::expr::list_contains; @@ -36,9 +36,11 @@ fn main() { } // Sized to keep CodSpeed simulation under 1ms per benchmark. -const ROWS: usize = 4_096; -const CHUNKS: usize = 16; -const SET_LENS: &[usize] = &[4, 64, 1_024]; +const ROWS: usize = 1_024; +const CHUNKS: usize = 4; +const SET_LENS: &[usize] = &[4, 256]; +/// A nested set compares whole rows to sort its elements and to probe them, so it stays smaller. +const NESTED_SET_LENS: &[usize] = &[4, 32]; /// A random set of `len` values, and needles of which about half are members. fn random_i64(len: usize) -> (Vec, Vec) { @@ -56,66 +58,6 @@ fn random_i64(len: usize) -> (Vec, Vec) { (set, needles) } -/// The set `0..len`, and needles drawn from twice that range. -fn dense_i64(len: usize) -> (Vec, Vec) { - let mut rng = StdRng::seed_from_u64(0); - let set = (0..len as i64).collect(); - let needles = (0..ROWS) - .map(|_| rng.random_range(0..2 * len as i64)) - .collect(); - (set, needles) -} - -fn i64_set(values: &[i64]) -> Scalar { - Scalar::list( - Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)), - values.iter().map(|&v| v.into()).collect(), - Nullability::NonNullable, - ) -} - -fn utf8_set(values: &[String]) -> Scalar { - Scalar::list( - Arc::new(DType::Utf8(Nullability::NonNullable)), - values - .iter() - .map(|v| Scalar::utf8(v.as_str(), Nullability::NonNullable)) - .collect(), - Nullability::NonNullable, - ) -} - -fn chunked(needles: &[ArrayRef]) -> ArrayRef { - let dtype = needles[0].dtype().clone(); - ChunkedArray::try_new(needles.iter().cloned(), dtype) - .unwrap() - .into_array() -} - -fn i64_needles(needles: &[i64], chunks: usize) -> ArrayRef { - let parts: Vec = needles - .chunks(needles.len() / chunks) - .map(|chunk| PrimitiveArray::from_iter(chunk.iter().copied()).into_array()) - .collect(); - if chunks == 1 { - parts[0].clone() - } else { - chunked(&parts) - } -} - -fn utf8_needles(needles: &[String], chunks: usize) -> ArrayRef { - let parts: Vec = needles - .chunks(needles.len() / chunks) - .map(|chunk| VarBinViewArray::from_iter_str(chunk.iter().map(String::as_str)).into_array()) - .collect(); - if chunks == 1 { - parts[0].clone() - } else { - chunked(&parts) - } -} - fn bench_in_set(bencher: Bencher, set: Scalar, needles: ArrayRef) { let session = vortex_array::array_session(); // Optimized as a scan optimizes it, so the set arrives normalized. @@ -134,72 +76,50 @@ fn bench_in_set(bencher: Bencher, set: Scalar, needles: ArrayRef) { .bench_values(|(array, mut ctx)| array.execute::(&mut ctx).unwrap()); } +fn i64_set(values: Vec) -> Scalar { + Scalar::list( + Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)), + values.into_iter().map(Scalar::from).collect(), + Nullability::NonNullable, + ) +} + #[divan::bench(args = SET_LENS)] fn i64_random(bencher: Bencher, set_len: usize) { let (set, needles) = random_i64(set_len); - bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); + let needles = PrimitiveArray::from_iter(needles); + bench_in_set(bencher, i64_set(set), needles.into_array()); } #[divan::bench(args = SET_LENS)] fn i64_random_chunked(bencher: Bencher, set_len: usize) { let (set, needles) = random_i64(set_len); - bench_in_set(bencher, i64_set(&set), i64_needles(&needles, CHUNKS)); -} - -#[divan::bench(args = SET_LENS)] -fn i64_dense(bencher: Bencher, set_len: usize) { - let (set, needles) = dense_i64(set_len); - bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); + let chunks = needles + .chunks(ROWS / CHUNKS) + .map(|chunk| PrimitiveArray::from_iter(chunk.iter().copied()).into_array()); + let needles = ChunkedArray::try_new( + chunks, + DType::Primitive(PType::I64, Nullability::NonNullable), + ) + .unwrap(); + bench_in_set(bencher, i64_set(set), needles.into_array()); } #[divan::bench(args = SET_LENS)] fn utf8_random(bencher: Bencher, set_len: usize) { let (set, needles) = random_i64(set_len); - let set: Vec = set.iter().map(|v| format!("value-{v}")).collect(); - let needles: Vec = needles.iter().map(|v| format!("value-{v}")).collect(); - bench_in_set(bencher, utf8_set(&set), utf8_needles(&needles, 1)); -} - -#[divan::bench(args = SET_LENS)] -fn utf8_random_chunked(bencher: Bencher, set_len: usize) { - let (set, needles) = random_i64(set_len); - let set: Vec = set.iter().map(|v| format!("value-{v}")).collect(); - let needles: Vec = needles.iter().map(|v| format!("value-{v}")).collect(); - bench_in_set(bencher, utf8_set(&set), utf8_needles(&needles, CHUNKS)); -} - -#[divan::bench(args = [4_096, 16_384])] -fn i64_random_large(bencher: Bencher, set_len: usize) { - let (set, needles) = random_i64(set_len); - bench_in_set(bencher, i64_set(&set), i64_needles(&needles, 1)); -} - -#[divan::bench(args = SET_LENS)] -fn decimal_random(bencher: Bencher, set_len: usize) { - let (set, needles) = random_i64(set_len); - bench_decimal(bencher, set, needles); -} - -#[divan::bench(args = SET_LENS)] -fn decimal_dense(bencher: Bencher, set_len: usize) { - let (set, needles) = dense_i64(set_len); - bench_decimal(bencher, set, needles); -} - -fn bench_decimal(bencher: Bencher, set: Vec, needles: Vec) { - let dtype = DecimalDType::new(20, 2); let set = Scalar::list( - DType::Decimal(dtype, Nullability::NonNullable), - set.into_iter() - .map(|value| Scalar::decimal(value.into(), dtype, Nullability::NonNullable)) + Arc::new(DType::Utf8(Nullability::NonNullable)), + set.iter() + .map(|v| Scalar::utf8(format!("value-{v}"), Nullability::NonNullable)) .collect(), Nullability::NonNullable, ); - let needles = DecimalArray::from_iter::(needles, dtype).into_array(); - bench_in_set(bencher, set, needles); + let needles = VarBinViewArray::from_iter_str(needles.iter().map(|v| format!("value-{v}"))); + bench_in_set(bencher, set, needles.into_array()); } -#[divan::bench(args = SET_LENS)] +#[divan::bench(args = NESTED_SET_LENS)] fn nested_list_random(bencher: Bencher, set_len: usize) { let (set, needles) = random_i64(set_len); let element_dtype = Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)); From 27f94c177e3c035d6882992ac3a1e772140db87c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 29 Sep 2026 14:04:34 -0700 Subject: [PATCH 5/5] Trim small set sizes from list_contains benchmarks Keep one set size per benchmark, four cases in all, so CodSpeed runs the sizes that show the probe cost rather than the setup cost. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Robert Kruszewski --- vortex-array/benches/list_contains_set.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-array/benches/list_contains_set.rs index 87b2057f9c2..cdee167ea71 100644 --- a/vortex-array/benches/list_contains_set.rs +++ b/vortex-array/benches/list_contains_set.rs @@ -38,9 +38,9 @@ fn main() { // Sized to keep CodSpeed simulation under 1ms per benchmark. const ROWS: usize = 1_024; const CHUNKS: usize = 4; -const SET_LENS: &[usize] = &[4, 256]; +const SET_LENS: &[usize] = &[256]; /// A nested set compares whole rows to sort its elements and to probe them, so it stays smaller. -const NESTED_SET_LENS: &[usize] = &[4, 32]; +const NESTED_SET_LENS: &[usize] = &[32]; /// A random set of `len` values, and needles of which about half are members. fn random_i64(len: usize) -> (Vec, Vec) {