From 74f019e1e71e64af0cf582bdef452502a36fda1a Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 27 Oct 2026 14:34:44 -0400 Subject: [PATCH 1/7] refactor: add reusable caching array hasher Reuse dictionary value hashes while the values Arc is unchanged so operators do not rehash the same dictionary for every batch. --- datafusion/common/src/hash_utils.rs | 172 ++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index c9cc48a11dd21..37166e21ae03a 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -17,6 +17,10 @@ use foldhash::fast::FixedState; #[cfg(not(feature = "force_hash_collisions"))] use itertools::Itertools; use std::hash::{BuildHasher, Hash, Hasher}; +use std::mem::size_of; +#[cfg(not(feature = "force_hash_collisions"))] +use std::sync::Arc; +use std::sync::Weak; /// [`RandomState`] is optimized for speed and suitable for hash tables and /// bloom filters. [`QualityRandomState`] is optimized for statistical quality @@ -28,6 +32,101 @@ use std::hash::{BuildHasher, Hash, Hasher}; pub type RandomState = FixedState; pub type QualityRandomState = foldhash::quality::FixedState; +type DictionaryHashEntry = (Weak, Vec); + +/// Hashes one array at a time, reusing dictionary value hashes when possible. +#[derive(Debug)] +pub struct CachingArrayHasher { + /// Stored with the cache so value hashes cannot cross hash seeds. + random_state: RandomState, + /// Weak identity avoids retaining values; the hash allocation is reusable. + dictionary: Option, +} + +impl CachingArrayHasher { + /// Creates an empty cache for `random_state`. + pub fn new(random_state: RandomState) -> Self { + Self { + random_state, + dictionary: None, + } + } + + /// Hashes `array`, reusing dictionary value hashes while its values array + /// is unchanged. + /// + /// `hashes_buffer` must have the same length as `array`. If `rehash` is + /// `false`, this initializes each row hash; if `true`, it combines the + /// array hash with the existing row hash. + pub fn create_hashes<'a>( + &mut self, + array: &dyn Array, + hashes_buffer: &'a mut [u64], + rehash: bool, + ) -> Result<&'a mut [u64]> { + #[cfg(feature = "force_hash_collisions")] + { + hash_single_array(array, &self.random_state, hashes_buffer, rehash)?; + } + + #[cfg(not(feature = "force_hash_collisions"))] + { + if !matches!(array.data_type(), DataType::Dictionary(_, _)) { + self.dictionary = None; + hash_single_array(array, &self.random_state, hashes_buffer, rehash)?; + return Ok(hashes_buffer); + } + + downcast_dictionary_array! { + array => { + let values = array.values(); + if !self.dictionary.as_ref().is_some_and(|(cached, _)| { + std::ptr::addr_eq(cached.as_ptr(), Arc::as_ptr(values)) + }) { + let mut hashes = self + .dictionary + .take() + .map(|(_, hashes)| hashes) + .unwrap_or_default(); + hashes.resize(values.len(), 0); + // Fresh hashing leaves null value slots unchanged. + hashes.fill(0); + hash_single_array( + values.as_ref(), + &self.random_state, + &mut hashes, + false, + )?; + self.dictionary = Some((Arc::downgrade(values), hashes)); + } + dispatch_dictionary_scatter( + array, + &self.dictionary.as_ref().unwrap().1, + hashes_buffer, + rehash, + ); + } + _ => unreachable!(), + }; + } + + Ok(hashes_buffer) + } + + /// Returns the current dictionary values and hashes, if any. + pub fn dictionary_hashes(&self) -> Option<(ArrayRef, &[u64])> { + let (values, hashes) = self.dictionary.as_ref()?; + Some((values.upgrade()?, hashes)) + } + + /// Returns the allocated size of the cache. + pub fn size(&self) -> usize { + self.dictionary + .as_ref() + .map_or(0, |entry| entry.1.capacity() * size_of::()) + } +} + /// Fixed quality hash state used by HyperLogLog sketches. /// /// The seed is part of the HLL wire/storage semantics: serialized sketches only @@ -1357,6 +1456,79 @@ mod tests { } } + #[cfg(not(feature = "force_hash_collisions"))] + #[test] + fn caching_array_hasher_reuses_dictionary_allocation() -> Result<()> { + let random_state = RandomState::with_seed(0); + let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); + let dictionary = DictionaryArray::::new( + Int8Array::from(vec![0, 1, 0]), + Arc::clone(&values), + ); + let mut expected = vec![0; dictionary.len()]; + create_hashes([&dictionary as &dyn Array], &random_state, &mut expected)?; + + let mut cache = CachingArrayHasher::new(random_state.clone()); + let mut actual = vec![0; dictionary.len()]; + cache.create_hashes(&dictionary, &mut actual, false)?; + assert_eq!(actual, expected); + + let (cached_values, value_hashes) = cache.dictionary_hashes().unwrap(); + assert!(Arc::ptr_eq(&cached_values, &values)); + let allocation = value_hashes.as_ptr(); + + let prefix = Int32Array::from(vec![10, 20]); + let dictionary = DictionaryArray::::new( + Int8Array::from(vec![1, 0]), + Arc::clone(&values), + ); + let mut expected = vec![0; dictionary.len()]; + create_hashes( + [&prefix as &dyn Array, &dictionary], + &random_state, + &mut expected, + )?; + let mut actual = vec![0; dictionary.len()]; + create_hashes([&prefix as &dyn Array], &random_state, &mut actual)?; + cache.create_hashes(&dictionary, &mut actual, true)?; + assert_eq!(actual, expected); + + let new_values: ArrayRef = Arc::new(StringArray::from(vec![Some("c"), None])); + let dictionary = DictionaryArray::::new( + Int8Array::from(vec![1, 0]), + Arc::clone(&new_values), + ); + actual.clear(); + actual.resize(dictionary.len(), 0); + expected.clear(); + expected.resize(dictionary.len(), 0); + create_hashes([&dictionary as &dyn Array], &random_state, &mut expected)?; + cache.create_hashes(&dictionary, &mut actual, false)?; + + let (cached_values, value_hashes) = cache.dictionary_hashes().unwrap(); + assert_eq!(actual, expected); + assert!(Arc::ptr_eq(&cached_values, &new_values)); + assert_eq!(value_hashes.as_ptr(), allocation); + assert_eq!(value_hashes[1], 0); + Ok(()) + } + + #[test] + fn caching_array_hasher_hashes_non_dictionary() -> Result<()> { + let random_state = RandomState::with_seed(0); + let array = Int32Array::from(vec![1, 2, 3]); + let mut expected = vec![0; array.len()]; + create_hashes([&array as &dyn Array], &random_state, &mut expected)?; + + let mut cache = CachingArrayHasher::new(random_state); + let mut actual = vec![0; array.len()]; + cache.create_hashes(&array, &mut actual, false)?; + + assert_eq!(actual, expected); + assert!(cache.dictionary_hashes().is_none()); + Ok(()) + } + #[test] fn create_hashes_for_decimal_array() -> Result<()> { let array = vec![1, 2, 3, 4] From 049070072b98b6150032ccb14bbd2cbf529a9ec9 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Wed, 28 Oct 2026 00:34:43 -0400 Subject: [PATCH 2/7] test: consolidate caching array hasher coverage --- datafusion/common/src/hash_utils.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 37166e21ae03a..08e85601fed0a 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -1458,7 +1458,7 @@ mod tests { #[cfg(not(feature = "force_hash_collisions"))] #[test] - fn caching_array_hasher_reuses_dictionary_allocation() -> Result<()> { + fn caching_array_hasher() -> Result<()> { let random_state = RandomState::with_seed(0); let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); let dictionary = DictionaryArray::::new( @@ -1510,17 +1510,11 @@ mod tests { assert!(Arc::ptr_eq(&cached_values, &new_values)); assert_eq!(value_hashes.as_ptr(), allocation); assert_eq!(value_hashes[1], 0); - Ok(()) - } - #[test] - fn caching_array_hasher_hashes_non_dictionary() -> Result<()> { - let random_state = RandomState::with_seed(0); let array = Int32Array::from(vec![1, 2, 3]); let mut expected = vec![0; array.len()]; create_hashes([&array as &dyn Array], &random_state, &mut expected)?; - let mut cache = CachingArrayHasher::new(random_state); let mut actual = vec![0; array.len()]; cache.create_hashes(&array, &mut actual, false)?; From 9c32dbaaf42c30282abb4be294ea69aa9f53aa4d Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Wed, 28 Oct 2026 14:30:46 -0400 Subject: [PATCH 3/7] refactor: cache dictionary hashes by column --- datafusion/common/src/hash_utils.rs | 322 ++++++++++++++++++++-------- 1 file changed, 238 insertions(+), 84 deletions(-) diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 08e85601fed0a..419b0ee0fc958 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -34,57 +34,86 @@ pub type QualityRandomState = foldhash::quality::FixedState; type DictionaryHashEntry = (Weak, Vec); -/// Hashes one array at a time, reusing dictionary value hashes when possible. +/// Hashes arrays, reusing dictionary value hashes by column when possible. #[derive(Debug)] pub struct CachingArrayHasher { /// Stored with the cache so value hashes cannot cross hash seeds. random_state: RandomState, - /// Weak identity avoids retaining values; the hash allocation is reusable. - dictionary: Option, + /// One entry per column. Weak identity avoids retaining dictionary values. + dictionaries: Vec>, } impl CachingArrayHasher { - /// Creates an empty cache for `random_state`. - pub fn new(random_state: RandomState) -> Self { + /// Creates one cache slot per column in `schema`. + /// + /// For each dictionary column, the cache tracks the values array with a + /// weak pointer, so it does not keep the values alive, and owns a reusable + /// value-hash buffer. [`Self::create_hashes`] reuses those hashes while the + /// values `Arc` is unchanged, recomputes them when it changes, and scatters + /// them through the dictionary keys into the row hashes. Non-dictionary + /// columns use the regular array hashing path. + pub fn new(schema: &Schema, random_state: RandomState) -> Self { Self { random_state, - dictionary: None, + dictionaries: schema.fields().iter().map(|_| None).collect(), } } - /// Hashes `array`, reusing dictionary value hashes while its values array - /// is unchanged. - /// - /// `hashes_buffer` must have the same length as `array`. If `rehash` is - /// `false`, this initializes each row hash; if `true`, it combines the - /// array hash with the existing row hash. - pub fn create_hashes<'a>( + /// Hashes `arrays`, reusing dictionary value hashes while each column's + /// values array is unchanged. + pub fn create_hashes<'a, I, T>( &mut self, - array: &dyn Array, + arrays: I, hashes_buffer: &'a mut [u64], + ) -> Result<&'a mut [u64]> + where + I: IntoIterator, + I::IntoIter: ExactSizeIterator, + T: AsDynArray, + { + let arrays = arrays.into_iter(); + if arrays.len() != self.dictionaries.len() { + return _internal_err!( + "CachingArrayHasher expected {} arrays, got {}", + self.dictionaries.len(), + arrays.len() + ); + } + for (index, array) in arrays.enumerate() { + self.hash_array(index, array.as_dyn_array(), hashes_buffer, index != 0)?; + } + Ok(hashes_buffer) + } + + fn hash_array( + &mut self, + index: usize, + array: &dyn Array, + hashes_buffer: &mut [u64], rehash: bool, - ) -> Result<&'a mut [u64]> { + ) -> Result<()> { #[cfg(feature = "force_hash_collisions")] { + let _ = index; hash_single_array(array, &self.random_state, hashes_buffer, rehash)?; } #[cfg(not(feature = "force_hash_collisions"))] { + let dictionary = &mut self.dictionaries[index]; if !matches!(array.data_type(), DataType::Dictionary(_, _)) { - self.dictionary = None; + *dictionary = None; hash_single_array(array, &self.random_state, hashes_buffer, rehash)?; - return Ok(hashes_buffer); + return Ok(()); } downcast_dictionary_array! { array => { let values = array.values(); - if !self.dictionary.as_ref().is_some_and(|(cached, _)| { + if !dictionary.as_ref().is_some_and(|(cached, _)| { std::ptr::addr_eq(cached.as_ptr(), Arc::as_ptr(values)) }) { - let mut hashes = self - .dictionary + let mut hashes = dictionary .take() .map(|(_, hashes)| hashes) .unwrap_or_default(); @@ -97,11 +126,11 @@ impl CachingArrayHasher { &mut hashes, false, )?; - self.dictionary = Some((Arc::downgrade(values), hashes)); + *dictionary = Some((Arc::downgrade(values), hashes)); } dispatch_dictionary_scatter( array, - &self.dictionary.as_ref().unwrap().1, + &dictionary.as_ref().unwrap().1, hashes_buffer, rehash, ); @@ -110,20 +139,24 @@ impl CachingArrayHasher { }; } - Ok(hashes_buffer) + Ok(()) } - /// Returns the current dictionary values and hashes, if any. - pub fn dictionary_hashes(&self) -> Option<(ArrayRef, &[u64])> { - let (values, hashes) = self.dictionary.as_ref()?; + /// Returns the current dictionary values and hashes for `index`, if any. + pub fn dictionary_hashes(&self, index: usize) -> Option<(ArrayRef, &[u64])> { + let (values, hashes) = self.dictionaries.get(index)?.as_ref()?; Some((values.upgrade()?, hashes)) } /// Returns the allocated size of the cache. pub fn size(&self) -> usize { - self.dictionary - .as_ref() - .map_or(0, |entry| entry.1.capacity() * size_of::()) + self.dictionaries.capacity() * size_of::>() + + self + .dictionaries + .iter() + .flatten() + .map(|entry| entry.1.capacity() * size_of::()) + .sum::() } } @@ -1457,69 +1490,190 @@ mod tests { } #[cfg(not(feature = "force_hash_collisions"))] - #[test] - fn caching_array_hasher() -> Result<()> { - let random_state = RandomState::with_seed(0); - let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); - let dictionary = DictionaryArray::::new( - Int8Array::from(vec![0, 1, 0]), - Arc::clone(&values), - ); - let mut expected = vec![0; dictionary.len()]; - create_hashes([&dictionary as &dyn Array], &random_state, &mut expected)?; + fn string_dictionary(keys: Vec, values: &ArrayRef) -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int8Array::from(keys), + Arc::clone(values), + )) + } - let mut cache = CachingArrayHasher::new(random_state.clone()); - let mut actual = vec![0; dictionary.len()]; - cache.create_hashes(&dictionary, &mut actual, false)?; + #[cfg(not(feature = "force_hash_collisions"))] + fn caching_hasher( + columns: &[ArrayRef], + random_state: &RandomState, + ) -> CachingArrayHasher { + let fields = columns + .iter() + .enumerate() + .map(|(index, column)| { + Field::new(format!("column_{index}"), column.data_type().clone(), true) + }) + .collect::>(); + CachingArrayHasher::new(&Schema::new(fields), random_state.clone()) + } + + #[cfg(not(feature = "force_hash_collisions"))] + fn assert_hashes_match( + hasher: &mut CachingArrayHasher, + columns: &[ArrayRef], + random_state: &RandomState, + ) -> Result<()> { + let mut expected = vec![0; columns[0].len()]; + create_hashes(columns, random_state, &mut expected)?; + + let mut actual = vec![0; columns[0].len()]; + hasher.create_hashes(columns, &mut actual)?; assert_eq!(actual, expected); + Ok(()) + } - let (cached_values, value_hashes) = cache.dictionary_hashes().unwrap(); - assert!(Arc::ptr_eq(&cached_values, &values)); - let allocation = value_hashes.as_ptr(); + #[cfg(not(feature = "force_hash_collisions"))] + fn assert_cached_values( + hasher: &CachingArrayHasher, + column_index: usize, + expected_values: &ArrayRef, + ) -> *const u64 { + let (cached_values, value_hashes) = + hasher.dictionary_hashes(column_index).unwrap(); + assert!(Arc::ptr_eq(&cached_values, expected_values)); + value_hashes.as_ptr() + } - let prefix = Int32Array::from(vec![10, 20]); - let dictionary = DictionaryArray::::new( - Int8Array::from(vec![1, 0]), - Arc::clone(&values), - ); - let mut expected = vec![0; dictionary.len()]; - create_hashes( - [&prefix as &dyn Array, &dictionary], - &random_state, - &mut expected, - )?; - let mut actual = vec![0; dictionary.len()]; - create_hashes([&prefix as &dyn Array], &random_state, &mut actual)?; - cache.create_hashes(&dictionary, &mut actual, true)?; - assert_eq!(actual, expected); + #[cfg(not(feature = "force_hash_collisions"))] + #[test] + fn caching_array_hasher_hashes_regular_and_dictionary_columns() -> Result<()> { + let random_state = RandomState::with_seed(0); + let dictionary_values: ArrayRef = + Arc::new(StringArray::from(vec!["active", "inactive"])); + let first_batch = vec![ + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + string_dictionary(vec![0, 1, 0], &dictionary_values), + ]; + let mut hasher = caching_hasher(&first_batch, &random_state); + assert_hashes_match(&mut hasher, &first_batch, &random_state)?; - let new_values: ArrayRef = Arc::new(StringArray::from(vec![Some("c"), None])); - let dictionary = DictionaryArray::::new( - Int8Array::from(vec![1, 0]), - Arc::clone(&new_values), - ); - actual.clear(); - actual.resize(dictionary.len(), 0); - expected.clear(); - expected.resize(dictionary.len(), 0); - create_hashes([&dictionary as &dyn Array], &random_state, &mut expected)?; - cache.create_hashes(&dictionary, &mut actual, false)?; - - let (cached_values, value_hashes) = cache.dictionary_hashes().unwrap(); - assert_eq!(actual, expected); - assert!(Arc::ptr_eq(&cached_values, &new_values)); - assert_eq!(value_hashes.as_ptr(), allocation); - assert_eq!(value_hashes[1], 0); + assert!(hasher.dictionary_hashes(0).is_none()); + let value_hash_allocation = assert_cached_values(&hasher, 1, &dictionary_values); - let array = Int32Array::from(vec![1, 2, 3]); - let mut expected = vec![0; array.len()]; - create_hashes([&array as &dyn Array], &random_state, &mut expected)?; + // A later batch has new plain values and dictionary keys, but shares + // the dictionary values array and therefore its cached hashes. + let second_batch = vec![ + Arc::new(Int32Array::from(vec![40, 50])) as ArrayRef, + string_dictionary(vec![1, 0], &dictionary_values), + ]; + assert_hashes_match(&mut hasher, &second_batch, &random_state)?; + assert_eq!( + assert_cached_values(&hasher, 1, &dictionary_values), + value_hash_allocation + ); + Ok(()) + } - let mut actual = vec![0; array.len()]; - cache.create_hashes(&array, &mut actual, false)?; + #[cfg(not(feature = "force_hash_collisions"))] + #[test] + fn caching_array_hasher_updates_dictionary_columns_independently() -> Result<()> { + let random_state = RandomState::with_seed(0); + let country_values: ArrayRef = Arc::new(StringArray::from(vec!["us", "uk"])); + let status_values: ArrayRef = + Arc::new(StringArray::from(vec!["active", "inactive"])); + let first_batch = vec![ + string_dictionary(vec![0, 1, 0], &country_values), + string_dictionary(vec![1, 0, 1], &status_values), + ]; + let mut hasher = caching_hasher(&first_batch, &random_state); + assert_hashes_match(&mut hasher, &first_batch, &random_state)?; + let country_hash_allocation = assert_cached_values(&hasher, 0, &country_values); + let status_hash_allocation = assert_cached_values(&hasher, 1, &status_values); + + // Aggregation batches can rebuild one dictionary while another group + // column continues to share its values array. + let new_country_values: ArrayRef = + Arc::new(StringArray::from(vec![Some("ca"), None])); + let second_batch = vec![ + string_dictionary(vec![1, 0], &new_country_values), + string_dictionary(vec![0, 1], &status_values), + ]; + assert_hashes_match(&mut hasher, &second_batch, &random_state)?; + assert_eq!( + assert_cached_values(&hasher, 0, &new_country_values), + country_hash_allocation + ); + assert_eq!( + assert_cached_values(&hasher, 1, &status_values), + status_hash_allocation + ); + assert_eq!(hasher.dictionary_hashes(0).unwrap().1[1], 0); + + // Reverse the transition: the first dictionary stays shared while the + // second receives a new values array. + let new_status_values: ArrayRef = + Arc::new(StringArray::from(vec!["pending", "complete"])); + let third_batch = vec![ + string_dictionary(vec![0, 1], &new_country_values), + string_dictionary(vec![1, 0], &new_status_values), + ]; + assert_hashes_match(&mut hasher, &third_batch, &random_state)?; + assert_eq!( + assert_cached_values(&hasher, 0, &new_country_values), + country_hash_allocation + ); + assert_eq!( + assert_cached_values(&hasher, 1, &new_status_values), + status_hash_allocation + ); + Ok(()) + } - assert_eq!(actual, expected); - assert!(cache.dictionary_hashes().is_none()); + #[cfg(not(feature = "force_hash_collisions"))] + #[test] + fn caching_array_hasher_tracks_repeated_changes_with_regular_column() -> Result<()> { + let random_state = RandomState::with_seed(0); + let region_values_v1: ArrayRef = + Arc::new(StringArray::from(vec!["east", "west"])); + let category_values_v1: ArrayRef = + Arc::new(StringArray::from(vec!["hardware", "software"])); + let first_batch = vec![ + string_dictionary(vec![0, 1, 0], ®ion_values_v1), + string_dictionary(vec![1, 0, 1], &category_values_v1), + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + ]; + let mut hasher = caching_hasher(&first_batch, &random_state); + assert_hashes_match(&mut hasher, &first_batch, &random_state)?; + + // This mirrors repeated aggregation input: dictionary group columns + // can change independently while regular group columns change normally. + let region_values_v2: ArrayRef = + Arc::new(StringArray::from(vec!["north", "south"])); + let second_batch = vec![ + string_dictionary(vec![1, 0, 1], ®ion_values_v2), + string_dictionary(vec![0, 1, 0], &category_values_v1), + Arc::new(Int32Array::from(vec![40, 50, 60])) as ArrayRef, + ]; + assert_hashes_match(&mut hasher, &second_batch, &random_state)?; + + let category_values_v2: ArrayRef = + Arc::new(StringArray::from(vec!["services", "support"])); + let third_batch = vec![ + string_dictionary(vec![0, 1], ®ion_values_v2), + string_dictionary(vec![1, 0], &category_values_v2), + Arc::new(Int32Array::from(vec![70, 80])) as ArrayRef, + ]; + assert_hashes_match(&mut hasher, &third_batch, &random_state)?; + + // Replacing both dictionaries together must also update both cache + // slots without affecting the regular column. + let region_values_v3: ArrayRef = + Arc::new(StringArray::from(vec!["central", "coastal", "remote"])); + let category_values_v3: ArrayRef = Arc::new(StringArray::from(vec!["other"])); + let fourth_batch = vec![ + string_dictionary(vec![2, 0, 1], ®ion_values_v3), + string_dictionary(vec![0, 0, 0], &category_values_v3), + Arc::new(Int32Array::from(vec![90, 100, 110])) as ArrayRef, + ]; + assert_hashes_match(&mut hasher, &fourth_batch, &random_state)?; + assert_cached_values(&hasher, 0, ®ion_values_v3); + assert_cached_values(&hasher, 1, &category_values_v3); + assert!(hasher.dictionary_hashes(2).is_none()); Ok(()) } From fe597f220a0597120beb06632d80be828ec0bbb4 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 27 Oct 2026 13:45:00 -0400 Subject: [PATCH 4/7] perf: reuse dictionary hashes in repartition --- .../physical-plan/src/repartition/mod.rs | 95 +++++++++++++++++-- 1 file changed, 86 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 987412fe0780e..191f8bc7a70fc 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -20,6 +20,8 @@ use super::{ }; use crate::coalesce::LimitedBatchCoalescer; use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType}; +use crate::hash_utils::CachingArrayHasher; +#[cfg(test)] use crate::hash_utils::create_hashes; use crate::metrics::{BaselineMetrics, SpillMetrics}; use crate::projection::{ProjectionExec, all_columns, make_with_child, update_expr}; @@ -35,7 +37,7 @@ use crate::{ use arrow::array::{PrimitiveArray, RecordBatch, RecordBatchOptions, UInt64Array}; use arrow::compute::take_arrays; -use arrow::datatypes::{DataType, Schema, SchemaRef, UInt32Type}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt32Type}; use arrow_schema::SortOptions; use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; @@ -644,6 +646,7 @@ enum BatchPartitionerState { partition_reducer: StrengthReducedU64, hash_buffer: Vec, indices: Vec>, + hasher: Option, }, RoundRobin { num_partitions: usize, @@ -1023,6 +1026,7 @@ impl BatchPartitioner { partition_reducer: StrengthReducedU64::new(num_partitions as u64), hash_buffer: vec![], indices: vec![vec![]; num_partitions], + hasher: None, }, timer, }) @@ -1237,6 +1241,7 @@ impl BatchPartitioner { partition_reducer, hash_buffer, indices, + hasher, } => { // Tracking time required for distributing indexes across output partitions let timer = self.timer.timer(); @@ -1247,11 +1252,24 @@ impl BatchPartitioner { hash_buffer.clear(); hash_buffer.resize(batch.num_rows(), 0); - create_hashes( - &arrays, - REPARTITION_RANDOM_STATE.random_state(), - hash_buffer, - )?; + let hasher = hasher.get_or_insert_with(|| { + let fields = arrays + .iter() + .enumerate() + .map(|(index, array)| { + Field::new( + format!("hash_{index}"), + array.data_type().clone(), + true, + ) + }) + .collect::>(); + CachingArrayHasher::new( + &Schema::new(fields), + REPARTITION_RANDOM_STATE.random_state().clone(), + ) + }); + hasher.create_hashes(&arrays, hash_buffer)?; for v in indices.iter_mut() { v.clear(); @@ -2647,10 +2665,10 @@ mod tests { }; use arrow::array::{ - Array, ArrayRef, Decimal128Array, Int64Array, StringArray, - TimestampNanosecondArray, UInt32Array, + Array, ArrayRef, Decimal128Array, DictionaryArray, Int32Array, Int64Array, + StringArray, TimestampNanosecondArray, UInt32Array, }; - use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; + use arrow::datatypes::{DataType, Field, Int32Type, Schema, TimeUnit}; use datafusion_common::ScalarValue; use datafusion_common::cast::{as_string_array, as_uint32_array}; use datafusion_common::config::SpillCompression; @@ -2905,6 +2923,65 @@ mod tests { Ok(()) } + #[tokio::test] + async fn hash_partition_reuses_dictionary_values_across_batches() -> Result<()> { + let dictionary_type = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let schema = Arc::new(Schema::new(vec![ + Field::new("key", dictionary_type, true), + Field::new("id", DataType::UInt32, false), + ])); + let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "bb", "ccc"])); + let make_batch = |keys: Vec>, ids: Vec| -> Result { + let dictionary: ArrayRef = Arc::new(DictionaryArray::::try_new( + Int32Array::from(keys), + Arc::clone(&values), + )?); + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![dictionary, Arc::new(UInt32Array::from(ids))], + )?) + }; + let batches = vec![ + make_batch(vec![Some(0), Some(1), Some(2), None], vec![0, 1, 2, 3])?, + make_batch(vec![Some(2), Some(0), None, Some(1)], vec![4, 5, 6, 7])?, + ]; + + let mut expected = vec![Vec::new(); 4]; + for batch in &batches { + let mut hashes = vec![0; batch.num_rows()]; + create_hashes( + [batch.column(0)], + REPARTITION_RANDOM_STATE.random_state(), + &mut hashes, + )?; + let ids = as_uint32_array(batch.column(1))?; + for (hash, id) in hashes.into_iter().zip(ids.values().iter().copied()) { + expected[(hash % 4) as usize].push(id); + } + } + + let output = repartition( + &schema, + vec![batches], + Partitioning::Hash(vec![col("key", &schema)?], 4), + ) + .await?; + let mut actual = vec![Vec::new(); 4]; + for (partition, batches) in output.iter().enumerate() { + for batch in batches { + actual[partition] + .extend(as_uint32_array(batch.column(1))?.values().iter().copied()); + } + } + for values in expected.iter_mut().chain(actual.iter_mut()) { + values.sort_unstable(); + } + + assert_eq!(actual, expected); + Ok(()) + } + #[tokio::test] async fn many_to_many_range_partition() -> Result<()> { let schema = test_schema(false); From 747baf5f92bfa7f50110e8dee68a60aca89ec501 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 09:05:13 -0500 Subject: [PATCH 5/7] refactor: initialize repartition hasher eagerly Pass the hash expression count into CachingArrayHasher so BatchPartitioner can create it up front without constructing a placeholder schema from the first batch. --- datafusion/common/src/hash_utils.rs | 15 +++-------- .../physical-plan/src/repartition/mod.rs | 27 +++++-------------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 419b0ee0fc958..c635ac67c64e9 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -44,7 +44,7 @@ pub struct CachingArrayHasher { } impl CachingArrayHasher { - /// Creates one cache slot per column in `schema`. + /// Creates one cache slot per column. /// /// For each dictionary column, the cache tracks the values array with a /// weak pointer, so it does not keep the values alive, and owns a reusable @@ -52,10 +52,10 @@ impl CachingArrayHasher { /// values `Arc` is unchanged, recomputes them when it changes, and scatters /// them through the dictionary keys into the row hashes. Non-dictionary /// columns use the regular array hashing path. - pub fn new(schema: &Schema, random_state: RandomState) -> Self { + pub fn new(num_columns: usize, random_state: RandomState) -> Self { Self { random_state, - dictionaries: schema.fields().iter().map(|_| None).collect(), + dictionaries: vec![None; num_columns], } } @@ -1502,14 +1502,7 @@ mod tests { columns: &[ArrayRef], random_state: &RandomState, ) -> CachingArrayHasher { - let fields = columns - .iter() - .enumerate() - .map(|(index, column)| { - Field::new(format!("column_{index}"), column.data_type().clone(), true) - }) - .collect::>(); - CachingArrayHasher::new(&Schema::new(fields), random_state.clone()) + CachingArrayHasher::new(columns.len(), random_state.clone()) } #[cfg(not(feature = "force_hash_collisions"))] diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index e29bd6a4953d6..53c6800a1597a 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -37,7 +37,7 @@ use crate::{ use arrow::array::{PrimitiveArray, RecordBatch, RecordBatchOptions, UInt64Array}; use arrow::compute::take_arrays; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt32Type}; +use arrow::datatypes::{DataType, Schema, SchemaRef, UInt32Type}; use arrow_schema::SortOptions; use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; @@ -646,7 +646,7 @@ enum BatchPartitionerState { partition_reducer: StrengthReducedU64, hash_buffer: Vec, indices: Vec>, - hasher: Option, + hasher: CachingArrayHasher, }, RoundRobin { num_partitions: usize, @@ -1034,13 +1034,17 @@ impl BatchPartitioner { return internal_err!("Hash repartition requires at least one partition"); } + let hasher = CachingArrayHasher::new( + exprs.len(), + REPARTITION_RANDOM_STATE.random_state().clone(), + ); Ok(Self { state: BatchPartitionerState::Hash { exprs, partition_reducer: StrengthReducedU64::new(num_partitions as u64), hash_buffer: vec![], indices: vec![vec![]; num_partitions], - hasher: None, + hasher, }, timer, }) @@ -1266,23 +1270,6 @@ impl BatchPartitioner { hash_buffer.clear(); hash_buffer.resize(batch.num_rows(), 0); - let hasher = hasher.get_or_insert_with(|| { - let fields = arrays - .iter() - .enumerate() - .map(|(index, array)| { - Field::new( - format!("hash_{index}"), - array.data_type().clone(), - true, - ) - }) - .collect::>(); - CachingArrayHasher::new( - &Schema::new(fields), - REPARTITION_RANDOM_STATE.random_state().clone(), - ) - }); hasher.create_hashes(&arrays, hash_buffer)?; for v in indices.iter_mut() { From 6d790c5f7441b0f7b786dadc0535140f41a5fa42 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 23:17:14 -0500 Subject: [PATCH 6/7] bench: pin parent of arrow-rs #12008 --- Cargo.lock | 87 ++++++++++++++++++++++-------------------------------- Cargo.toml | 20 +++++++++++++ 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb26ca8d8e2f5..001d3bcf9db06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,8 +159,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-arith", "arrow-array", @@ -182,8 +181,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -196,8 +194,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "ahash", "arrow-buffer", @@ -216,8 +213,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,8 +236,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "bytes", "half", @@ -252,8 +247,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,8 +268,7 @@ dependencies = [ [[package]] name = "arrow-cmp" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -285,8 +278,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-cast", @@ -300,8 +292,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-buffer", "arrow-schema", @@ -313,8 +304,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8709a798bd0825cb6207861aad09093133d64314c25569132dbc7f9f083879a5" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-arith", "arrow-array", @@ -339,8 +329,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -355,8 +344,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -380,8 +368,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -394,8 +381,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -407,8 +393,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "bitflags", "serde", @@ -419,8 +404,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "ahash", "arrow-array", @@ -434,8 +418,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -3011,7 +2994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3536,9 +3519,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -4666,8 +4649,7 @@ dependencies = [ [[package]] name = "parquet" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" dependencies = [ "ahash", "arrow-array", @@ -5040,7 +5022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5059,7 +5041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", @@ -5553,7 +5535,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5612,7 +5594,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6354,7 +6336,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6655,9 +6637,9 @@ checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tonic" -version = "0.14.6" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b" dependencies = [ "async-trait", "axum", @@ -6676,6 +6658,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-stream", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -6684,9 +6667,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484" dependencies = [ "bytes", "prost", @@ -6815,7 +6798,7 @@ version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] [[package]] @@ -7054,9 +7037,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -7308,7 +7291,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3348e1d84374a..a9cb396901895 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,6 +213,26 @@ url = "2.5.7" uuid = "1.23" zstd = { version = "0.14", default-features = false } +[patch.crates-io] +arrow = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-arith = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-array = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-avro = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-buffer = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-cast = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-cmp = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-csv = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-data = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-flight = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-ipc = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-json = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-ord = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-row = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-schema = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-select = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow-string = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +parquet = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } + # Keep this list sorted alphabetically. # See https://github.com/apache/datafusion/issues/18467 for the ongoing effort of # picking useful non-default lints. From e15a3816bdf45033ec1e3edfc8d05981641ad663 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 23:18:32 -0500 Subject: [PATCH 7/7] bench: pin arrow-rs #12008 --- Cargo.lock | 52 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 36 ++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 001d3bcf9db06..5546196a30626 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,7 +159,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-arith", "arrow-array", @@ -181,7 +181,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -194,7 +194,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "ahash", "arrow-buffer", @@ -213,7 +213,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -236,7 +236,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "bytes", "half", @@ -247,7 +247,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -268,7 +268,7 @@ dependencies = [ [[package]] name = "arrow-cmp" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -278,7 +278,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-cast", @@ -292,7 +292,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-buffer", "arrow-schema", @@ -304,7 +304,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-arith", "arrow-array", @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -344,7 +344,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -368,7 +368,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -381,7 +381,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -393,7 +393,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "bitflags", "serde", @@ -404,7 +404,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "ahash", "arrow-array", @@ -418,7 +418,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "arrow-array", "arrow-buffer", @@ -2994,7 +2994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4649,7 +4649,7 @@ dependencies = [ [[package]] name = "parquet" version = "60.1.0" -source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=258b0d203168186a9ed8064ae3c135f7b921b1e7#258b0d203168186a9ed8064ae3c135f7b921b1e7" +source = "git+https://github.com/Rich-T-kid/arrow-rs.git?rev=31c0d5c510af1ed1c4f858131233d120ff503f61#31c0d5c510af1ed1c4f858131233d120ff503f61" dependencies = [ "ahash", "arrow-array", @@ -5022,7 +5022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5041,7 +5041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -5535,7 +5535,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5594,7 +5594,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6336,7 +6336,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6798,7 +6798,7 @@ version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" dependencies = [ - "rand 0.10.1", + "rand 0.9.4", ] [[package]] @@ -7291,7 +7291,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a9cb396901895..402a89092a49d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,24 +214,24 @@ uuid = "1.23" zstd = { version = "0.14", default-features = false } [patch.crates-io] -arrow = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-arith = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-array = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-avro = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-buffer = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-cast = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-cmp = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-csv = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-data = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-flight = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-ipc = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-json = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-ord = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-row = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-schema = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-select = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -arrow-string = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } -parquet = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "258b0d203168186a9ed8064ae3c135f7b921b1e7" } +arrow = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-arith = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-array = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-avro = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-buffer = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-cast = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-cmp = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-csv = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-data = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-flight = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-ipc = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-json = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-ord = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-row = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-schema = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-select = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +arrow-string = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } +parquet = { git = "https://github.com/Rich-T-kid/arrow-rs.git", rev = "31c0d5c510af1ed1c4f858131233d120ff503f61" } # Keep this list sorted alphabetically. # See https://github.com/apache/datafusion/issues/18467 for the ongoing effort of