From 89847f9821a0478ba28ae7c926ad492ccce51be7 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 10 Aug 2026 13:18:37 +0100 Subject: [PATCH 1/6] DF stuff Signed-off-by: Adam Gutglick --- vortex-datafusion/examples/vortex_table.rs | 4 +- vortex-datafusion/src/convert/exprs.rs | 4 +- vortex-datafusion/src/persistent/cache.rs | 3 +- vortex-datafusion/src/persistent/format.rs | 14 ++-- vortex-datafusion/src/persistent/opener.rs | 77 ++++++++----------- vortex-datafusion/src/persistent/sink.rs | 2 +- vortex-datafusion/src/persistent/source.rs | 33 ++++---- .../src/tests/schema_evolution.rs | 2 +- vortex-datafusion/src/v2/source.rs | 14 ++++ 9 files changed, 83 insertions(+), 70 deletions(-) diff --git a/vortex-datafusion/examples/vortex_table.rs b/vortex-datafusion/examples/vortex_table.rs index 2570fc47afd..90a7ba5e1e4 100644 --- a/vortex-datafusion/examples/vortex_table.rs +++ b/vortex-datafusion/examples/vortex_table.rs @@ -70,9 +70,7 @@ async fn main() -> anyhow::Result<()> { .ok_or_else(|| vortex_err!("Path is not valid UTF-8"))?, )?; let config = ListingTableConfig::new(table_url) - .with_listing_options( - ListingOptions::new(format).with_session_config_options(ctx.state().config()), - ) + .with_listing_options(ListingOptions::new(format)) .infer_schema(&ctx.state()) .await?; diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 01bde086e06..14a9a070efd 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -15,12 +15,12 @@ use datafusion_expr::Operator as DFOperator; use datafusion_functions::core::getfield::GetFieldFunc; use datafusion_functions::string::octet_length::OctetLengthFunc; use datafusion_functions_nested::length::ArrayLength; +use datafusion_physical_expr::DynamicFilterTracking; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::projection::ProjectionExpr; use datafusion_physical_expr::projection::ProjectionExprs; use datafusion_physical_expr::utils::collect_columns; -use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr; use datafusion_physical_plan::expressions as df_expr; use itertools::Itertools; use vortex::VortexSessionDefault; @@ -532,7 +532,7 @@ fn try_operator_from_df(value: &DFOperator) -> DFResult { fn can_be_pushed_down_impl(expr: &Arc, schema: &Schema) -> bool { // We currently do not support pushdown of dynamic expressions in DF. // See issue: https://github.com/vortex-data/vortex/issues/4034 - if is_dynamic_physical_expr(expr) { + if DynamicFilterTracking::classify(expr).contains_dynamic_filter() { return false; } diff --git a/vortex-datafusion/src/persistent/cache.rs b/vortex-datafusion/src/persistent/cache.rs index a76574031b6..46a7bd85dfc 100644 --- a/vortex-datafusion/src/persistent/cache.rs +++ b/vortex-datafusion/src/persistent/cache.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use datafusion_common::HashMap; use datafusion_execution::cache::cache_manager::FileMetadata; use vortex::file::Footer; use vortex::file::VortexFile; @@ -41,7 +42,7 @@ impl FileMetadata for CachedVortexMetadata { } #[expect(clippy::disallowed_types)] - fn extra_info(&self) -> std::collections::HashMap { + fn extra_info(&self) -> HashMap { Default::default() } } diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index 4e59821dbfa..20f1e6f34b3 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -533,7 +533,13 @@ impl FileFormat for VortexFormat { }) .map(|f| f.vortex_expect("Failed to spawn infer_schema")) }) - .buffer_unordered(state.config_options().execution.meta_fetch_concurrency) + .buffer_unordered( + state + .config_options() + .execution + .meta_fetch_concurrency + .get(), + ) .try_collect::>() .await .map_err(|e| DataFusionError::Execution(format!("Failed to infer schema: {e}")))?; @@ -880,9 +886,7 @@ mod tests { format: &dyn FileFormat, calls: &ExpressionConvertorCalls, ) -> anyhow::Result<()> { - let source = format.file_source(TableSchema::from_file_schema( - expression_convertor_test_schema(), - )); + let source = format.file_source(TableSchema::from(expression_convertor_test_schema())); let result = source.try_pushdown_filters( vec![expression_convertor_test_filter()], &ConfigOptions::new(), @@ -949,7 +953,7 @@ mod tests { ..Default::default() }; let format = VortexFormat::new_with_options(VortexSession::default(), opts.clone()); - let table_schema = TableSchema::from_file_schema(Arc::new(Schema::empty())); + let table_schema = TableSchema::from(Arc::new(Schema::empty())); let source = format.file_source(table_schema); let source = source diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 89abfe68d18..8bd6a126e44 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -101,7 +101,7 @@ pub(crate) struct VortexOpener { pub has_output_ordering: bool, pub expression_convertor: Arc, - pub file_metadata_cache: Option>, + pub file_metadata_cache: Option>, /// Whether to enable expression pushdown into the underlying Vortex scan. pub projection_pushdown: bool, pub scan_concurrency: Option, @@ -642,6 +642,7 @@ mod tests { use std::sync::Arc; use std::sync::LazyLock; + use arrow_array::record_batch; use arrow_schema::Field; use arrow_schema::Fields; use arrow_schema::SchemaRef; @@ -655,14 +656,13 @@ mod tests { use datafusion::arrow::datatypes::UInt32Type; use datafusion::arrow::util::display::FormatOptions; use datafusion::arrow::util::pretty::pretty_format_batches_with_options; - use datafusion::common::record_batch; use datafusion::logical_expr::col; use datafusion::logical_expr::lit; use datafusion::physical_expr::planner::logical2physical; use datafusion::physical_expr_adapter::DefaultPhysicalExprAdapterFactory; use datafusion::scalar::ScalarValue; use datafusion_common::stats::Precision; - use datafusion_execution::cache::DefaultFilesMetadataCache; + use datafusion_execution::cache::default_cache::DefaultCache; use datafusion_expr::Operator; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr::expressions as df_expr; @@ -885,10 +885,9 @@ mod tests { let mut file = PartitionedFile::new(file_path.to_string(), data_size); file.partition_values = vec![ScalarValue::Int32(Some(1))]; - let table_schema = TableSchema::new( - Arc::clone(&file_schema), - vec![Arc::new(Field::new("part", DataType::Int32, false))], - ); + let table_schema = TableSchema::builder(Arc::clone(&file_schema)) + .with_table_partition_cols(vec![Arc::new(Field::new("part", DataType::Int32, false))]) + .build(); // filter matches partition value let filter = col("part").eq(lit(1)); @@ -941,16 +940,15 @@ mod tests { .collect(), ), ); - let table_schema = TableSchema::new( - file_schema, - vec![Arc::new( + let table_schema = TableSchema::builder(file_schema) + .with_table_partition_cols(vec![Arc::new( Field::new("part", DataType::Int32, false).with_metadata( [("partition".to_string(), "metadata".to_string())] .into_iter() .collect(), ), - )], - ); + )]) + .build(); let projection = ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema()); let expected_schema = Arc::new(projection.project_schema(table_schema.table_schema())?); @@ -993,7 +991,7 @@ mod tests { let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?; let expected_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - let table_schema = TableSchema::from_file_schema(Arc::clone(&expected_schema)); + let table_schema = TableSchema::from(Arc::clone(&expected_schema)); for projection_pushdown in [false, true] { let mut opener = make_opener(Arc::clone(&object_store), table_schema.clone(), None); @@ -1014,10 +1012,9 @@ mod tests { -> anyhow::Result<()> { let object_store = Arc::new(InMemory::new()) as Arc; let file_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - let table_schema = TableSchema::new( - Arc::clone(&file_schema), - vec![Arc::new(Field::new("part", DataType::Int32, false))], - ); + let table_schema = TableSchema::builder(Arc::clone(&file_schema)) + .with_table_partition_cols(vec![Arc::new(Field::new("part", DataType::Int32, false))]) + .build(); let partition_column = Arc::new(df_expr::Column::new("part", 1)) as PhysicalExprRef; let predicate = Arc::new(df_expr::BinaryExpr::new( @@ -1063,11 +1060,7 @@ mod tests { statistics.column_statistics[0].null_count = Precision::Exact(0); let file = PartitionedFile::new(file_path, data_size).with_statistics(Arc::new(statistics)); - let mut opener = make_opener( - object_store, - TableSchema::from_file_schema(batch.schema()), - None, - ); + let mut opener = make_opener(object_store, TableSchema::from(batch.schema()), None); opener.file_pruning_predicate = Some(Arc::new(SnapshotErrorExpr)); let df_metrics = opener.df_metrics.clone(); @@ -1101,7 +1094,7 @@ mod tests { let file = PartitionedFile::new_with_range(file_path.to_string(), file_size, 0, file_size as i64); - let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema)); + let table_schema = TableSchema::from(Arc::clone(&file_schema)); let opener = make_opener(object_store, table_schema, None); let stream = opener.open(file)?.await?; @@ -1121,10 +1114,11 @@ mod tests { write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?; let file = PartitionedFile::new(file_path.to_string(), data_size); - let table_schema = TableSchema::from_file_schema(batch.schema()); + let table_schema = TableSchema::from(batch.schema()); - let cache: Arc = - Arc::new(DefaultFilesMetadataCache::new(64 * 1024 * 1024)); + let cache: Arc = Arc::new( + DefaultCache::::new(64 * 1024 * 1024), + ); let mut opener = make_opener(Arc::clone(&object_store), table_schema, None); opener.file_metadata_cache = Some(Arc::clone(&cache)); @@ -1174,7 +1168,7 @@ mod tests { }; // Table schema has can accommodate both files - let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new( + let table_schema = TableSchema::from(Arc::new(Schema::new(vec![Field::new( "a", DataType::Int32, true, @@ -1277,7 +1271,7 @@ mod tests { filter: None, file_pruning_predicate: None, expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory), - table_schema: TableSchema::from_file_schema(Arc::clone(&table_schema)), + table_schema: TableSchema::from(Arc::clone(&table_schema)), limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), df_metrics: ExecutionPlanMetricsSet::new(), @@ -1341,7 +1335,7 @@ mod tests { let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?; // Table schema has an extra utf8 field. - let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new( + let table_schema = TableSchema::from(Arc::new(Schema::new(vec![Field::new( "my_struct", DataType::Struct(Fields::from(vec![ Field::new( @@ -1403,18 +1397,15 @@ mod tests { // Table schema has columns in DIFFERENT order: c, a, b // and different types that require casting (Utf8 -> Dictionary) - let table_schema = TableSchema::new( - Arc::new(Schema::new(vec![ - Field::new("c", DataType::Int32, true), - Field::new("a", DataType::Int32, true), - Field::new( - "b", - DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), - true, - ), - ])), - vec![], - ); + let table_schema = TableSchema::from(Arc::new(Schema::new(vec![ + Field::new("c", DataType::Int32, true), + Field::new("a", DataType::Int32, true), + Field::new( + "b", + DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), + true, + ), + ]))); // Project columns [0, 2] from table schema, which should give us: c, b // Before the fix, the schema adapter would get confused about which fields @@ -1494,7 +1485,7 @@ mod tests { filter: None, file_pruning_predicate: None, expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory), - table_schema: TableSchema::from_file_schema(schema), + table_schema: TableSchema::from(schema), limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), df_metrics: ExecutionPlanMetricsSet::new(), @@ -1674,7 +1665,7 @@ mod tests { write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?; let file_schema = batch.schema(); - let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema)); + let table_schema = TableSchema::from(Arc::clone(&file_schema)); // Create a projection that includes an arithmetic expression: a + b * 2 let col_a = df_expr::col("a", &file_schema)?; diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs index ed655b7e5d0..37074e5fe44 100644 --- a/vortex-datafusion/src/persistent/sink.rs +++ b/vortex-datafusion/src/persistent/sink.rs @@ -507,7 +507,7 @@ mod tests { #[test] fn test_display_as() { let session = VortexSession::empty(); - let table_schema = TableSchema::new(Arc::new(Schema::empty()), Vec::new()); + let table_schema = TableSchema::from(Arc::new(Schema::empty())); let config = FileSinkConfig { original_url: "".to_owned(), diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index 81fcf937955..fbd37f04ded 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -7,6 +7,7 @@ use std::sync::Weak; use datafusion_common::Result as DFResult; use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::TableSchema; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_scan_config::FileScanConfig; @@ -201,7 +202,7 @@ pub struct VortexSource { pub(crate) vortex_reader_factory: Option>, pub(crate) ordered: bool, vx_metrics_registry: Arc, - file_metadata_cache: Option>, + file_metadata_cache: Option>, /// Options controlling scan planning and execution behavior. options: VortexTableOptions, } @@ -295,10 +296,7 @@ impl VortexSource { } /// Overrides the metadata cache used to reuse Vortex footers across scans. - pub fn with_file_metadata_cache( - mut self, - file_metadata_cache: Arc, - ) -> Self { + pub fn with_file_metadata_cache(mut self, file_metadata_cache: Arc) -> Self { self.file_metadata_cache = Some(file_metadata_cache); self } @@ -534,6 +532,19 @@ impl FileSource for VortexSource { fn table_schema(&self) -> &TableSchema { &self.table_schema } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion_physical_plan::apply_expression_roots( + self.full_predicate + .iter() + .chain(self.vortex_predicate.iter()) + .chain(self.projection.iter().map(|expr| &expr.expr)), + f, + ) + } } #[cfg(test)] @@ -603,10 +614,7 @@ mod tests { } fn sort_test_source(schema: Arc) -> VortexSource { - VortexSource::new( - TableSchema::from_file_schema(schema), - VortexSession::default(), - ) + VortexSource::new(TableSchema::from(schema), VortexSession::default()) } fn octet_length_filter(schema: &Schema) -> PhysicalExprRef { @@ -660,11 +668,8 @@ mod tests { inner: DefaultExpressionConvertor::default(), }) as Arc; - let source = VortexSource::new( - TableSchema::from_file_schema(file_schema), - VortexSession::default(), - ) - .with_expression_convertor(Arc::clone(&expression_convertor)); + let source = VortexSource::new(TableSchema::from(file_schema), VortexSession::default()) + .with_expression_convertor(Arc::clone(&expression_convertor)); let config = FileScanConfigBuilder::new( ObjectStoreUrl::local_filesystem(), diff --git a/vortex-datafusion/src/tests/schema_evolution.rs b/vortex-datafusion/src/tests/schema_evolution.rs index 093fed6c148..bdc8f822af6 100644 --- a/vortex-datafusion/src/tests/schema_evolution.rs +++ b/vortex-datafusion/src/tests/schema_evolution.rs @@ -5,6 +5,7 @@ use std::sync::Arc; +use arrow_array::record_batch; use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::Fields; @@ -19,7 +20,6 @@ use datafusion::arrow::datatypes::UInt32Type; use datafusion::assert_batches_sorted_eq; use datafusion_common::assert_batches_eq; use datafusion_common::create_array; -use datafusion_common::record_batch; use datafusion_expr::col; use datafusion_expr::lit; use datafusion_functions::expr_fn::get_field; diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index 2ea024056d1..6437cb24dc5 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -81,6 +81,7 @@ use datafusion_common::Statistics; use datafusion_common::arrow::array::AsArray; use datafusion_common::arrow::array::RecordBatch; use datafusion_common::stats::Precision as DFPrecision; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::source::DataSource; use datafusion_execution::SendableRecordBatchStream; use datafusion_execution::TaskContext; @@ -652,6 +653,19 @@ impl DataSource for VortexDataSource { .with_updated_node(Arc::new(this) as _), ) } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion_physical_plan::apply_expression_roots( + self.leftover_projection + .iter() + .flat_map(|projection| projection.iter()) + .map(|expr| &expr.expr), + f, + ) + } } /// Convert a Vortex [`Option`] to a DataFusion From 61c9169701b716177de23cfcd85271b47a993991 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 10 Aug 2026 13:23:08 +0100 Subject: [PATCH 2/6] geo 1.95 Signed-off-by: Adam Gutglick --- Cargo.lock | 722 +++++++++++++++++++++++++------------- Cargo.toml | 66 +++- vortex-arrow/src/dtype.rs | 5 +- 3 files changed, 540 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a92a0f5be59..88022aeef77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,10 +235,10 @@ dependencies = [ "arrow-array 58.4.0", "arrow-buffer 58.4.0", "arrow-cast 58.4.0", - "arrow-csv", + "arrow-csv 58.4.0", "arrow-data 58.4.0", "arrow-ipc 58.4.0", - "arrow-json", + "arrow-json 58.4.0", "arrow-ord 58.4.0", "arrow-row 58.4.0", "arrow-schema 58.4.0", @@ -246,6 +246,27 @@ dependencies = [ "arrow-string 58.4.0", ] +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith 59.2.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-csv 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-json 59.2.0", + "arrow-ord 59.2.0", + "arrow-row 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "arrow-string 59.2.0", +] + [[package]] name = "arrow-arith" version = "56.2.1" @@ -274,6 +295,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "56.2.1" @@ -301,9 +336,28 @@ dependencies = [ "arrow-data 58.4.0", "arrow-schema 58.4.0", "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "chrono", "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -311,13 +365,13 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8108cbcae32141577bc55ab490449944c63cfc93d47384906b475d13d89d43d" +checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-schema 59.2.0", "bytes", "bzip2", "crc", @@ -352,7 +406,19 @@ checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.4.8", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", "num-traits", ] @@ -399,6 +465,28 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "atoi", + "base64 0.23.0", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + [[package]] name = "arrow-csv" version = "58.4.0" @@ -414,6 +502,21 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-csv" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +dependencies = [ + "arrow-array 59.2.0", + "arrow-cast 59.2.0", + "arrow-schema 59.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "56.2.1" @@ -439,6 +542,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer 59.2.0", + "arrow-schema 59.2.0", + "half", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-ipc" version = "56.2.1" @@ -465,7 +581,22 @@ dependencies = [ "arrow-schema 58.4.0", "arrow-select 58.4.0", "flatbuffers", - "lz4_flex 0.13.1", + "zstd", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "flatbuffers", + "lz4_flex 0.14.0", "zstd", ] @@ -494,6 +625,31 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "chrono", + "half", + "indexmap 2.14.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "56.2.1" @@ -520,6 +676,19 @@ dependencies = [ "arrow-select 58.4.0", ] +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", +] + [[package]] name = "arrow-row" version = "56.2.1" @@ -546,6 +715,19 @@ dependencies = [ "half", ] +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "half", +] + [[package]] name = "arrow-schema" version = "56.2.1" @@ -557,6 +739,15 @@ name = "arrow-schema" version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "bitflags 2.13.1", "serde_core", @@ -591,6 +782,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash 0.8.12", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "56.2.1" @@ -625,6 +830,23 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -942,7 +1164,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -1567,8 +1789,8 @@ name = "compress-bench" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "async-trait", "bytes", "clap", @@ -1576,7 +1798,7 @@ dependencies = [ "indicatif", "itertools 0.14.0", "lance-bench", - "parquet 58.4.0", + "parquet 59.2.0", "regex", "tempfile", "tokio", @@ -2049,11 +2271,10 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "bzip2", "chrono", @@ -2086,12 +2307,12 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", "parking_lot", - "parquet 58.4.0", + "parquet 59.2.0", "sqlparser", "tempfile", "tokio", @@ -2130,10 +2351,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "dashmap", "datafusion-common", @@ -2145,7 +2365,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2155,10 +2375,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2170,30 +2389,31 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-ipc 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", "chrono", "foldhash 0.2.0", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", - "parquet 58.4.0", + "parquet 59.2.0", "recursive", "sqlparser", "tokio", @@ -2204,8 +2424,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ "futures", "log", @@ -2215,10 +2434,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-compression", "async-trait", "bytes", @@ -2236,7 +2454,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2251,11 +2469,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-ipc 58.4.0", + "arrow 59.2.0", + "arrow-ipc 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -2267,7 +2484,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] @@ -2275,10 +2492,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "arrow-avro", "async-trait", "bytes", @@ -2294,10 +2510,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -2317,10 +2532,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -2340,10 +2554,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -2360,29 +2574,28 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", - "parquet 58.4.0", + "parquet 59.2.0", "tokio", ] [[package]] name = "datafusion-doc" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-buffer 58.4.0", + "arrow 59.2.0", + "arrow-buffer 59.2.0", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2391,19 +2604,21 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.5", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "chrono", "datafusion-common", @@ -2413,7 +2628,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2422,24 +2637,22 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-buffer 58.4.0", - "base64 0.22.1", + "arrow 59.2.0", + "arrow-buffer 59.2.0", + "base64 0.23.0", "blake2", "blake3", "chrono", @@ -2452,7 +2665,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2466,10 +2679,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2478,8 +2690,8 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] @@ -2487,10 +2699,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -2499,11 +2710,10 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-ord 58.4.0", + "arrow 59.2.0", + "arrow-ord 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2515,7 +2725,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2524,10 +2734,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2540,10 +2749,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -2557,8 +2765,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2567,28 +2774,26 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2598,10 +2803,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -2610,7 +2814,7 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2620,31 +2824,29 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr", "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "chrono", "datafusion-common", "datafusion-expr-common", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] @@ -2652,10 +2854,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2664,22 +2865,23 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", - "arrow-data 58.4.0", - "arrow-ipc 58.4.0", - "arrow-ord 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2693,21 +2895,21 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -2720,9 +2922,9 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ + "arrow-schema 59.2.0", "async-trait", "datafusion-common", "datafusion-execution", @@ -2734,10 +2936,9 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "bigdecimal", "chrono", "crc32fast", @@ -2764,10 +2965,9 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "bigdecimal", "chrono", "datafusion-common", @@ -2778,15 +2978,15 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-sqllogictest" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d11faf3e95b017819a10800dc6d612c1f003696dea201700796b9268704f591" +source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "async-trait", "bigdecimal", "clap", @@ -2795,7 +2995,7 @@ dependencies = [ "futures", "half", "indicatif", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "sqllogictest", @@ -3521,8 +3721,7 @@ dependencies = [ [[package]] name = "geoarrow" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec42ac7fb4fdcd6982dab92d24faf436f18c36e47c3f813a33619a2728718a30" +source = "git+https://github.com/geoarrow/geoarrow-rs?rev=855d2e138e54ce503c4488d5bb795ca1ea880d1f#855d2e138e54ce503c4488d5bb795ca1ea880d1f" dependencies = [ "geoarrow-array", "geoarrow-schema", @@ -3531,12 +3730,11 @@ dependencies = [ [[package]] name = "geoarrow-array" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dafe7b7de3fab1a8b7099fd6a6434ca955fa65065f9c19f0f8a133693f3c2b0e" +source = "git+https://github.com/geoarrow/geoarrow-rs?rev=855d2e138e54ce503c4488d5bb795ca1ea880d1f#855d2e138e54ce503c4488d5bb795ca1ea880d1f" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-schema 59.2.0", "geo-traits", "geoarrow-schema", "num-traits", @@ -3547,10 +3745,9 @@ dependencies = [ [[package]] name = "geoarrow-cast" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c308d653690a4e8ef3cbba69696056bd819e624766ece66d64cc26a638acc1" +source = "git+https://github.com/geoarrow/geoarrow-rs?rev=855d2e138e54ce503c4488d5bb795ca1ea880d1f#855d2e138e54ce503c4488d5bb795ca1ea880d1f" dependencies = [ - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "geo-traits", "geoarrow-array", "geoarrow-schema", @@ -3560,10 +3757,9 @@ dependencies = [ [[package]] name = "geoarrow-schema" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4a7edb2a1d87024a93805332a9c8184a0354836271d42c0d18cf628a5e3cd0" +source = "git+https://github.com/geoarrow/geoarrow-rs?rev=855d2e138e54ce503c4488d5bb795ca1ea880d1f#855d2e138e54ce503c4488d5bb795ca1ea880d1f" dependencies = [ - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "geo-traits", "serde", "serde_json", @@ -5545,6 +5741,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rust2" version = "0.16.5" @@ -5893,7 +6098,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-complex", "num-integer", "num-iter", @@ -5911,6 +6116,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -5951,7 +6166,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -6456,7 +6671,7 @@ dependencies = [ "hashbrown 0.16.1", "lz4_flex 0.11.6", "num", - "num-bigint", + "num-bigint 0.4.8", "paste", "seq-macro", "simdutf8", @@ -6485,19 +6700,50 @@ dependencies = [ "bytes", "chrono", "flate2", - "futures", "half", "hashbrown 0.17.1", "lz4_flex 0.13.1", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", - "object_store", "paste", "seq-macro", "simdutf8", "snap", "thrift", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +dependencies = [ + "ahash 0.8.12", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "base64 0.23.0", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "object_store", + "seq-macro", + "simdutf8", + "snap", "tokio", "twox-hash", "zstd", @@ -6505,12 +6751,12 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc70e87931167a4a3fde2ee923023a0624367691e3fd503476a1084dda5a054" +checksum = "3f7e5fff3ed0c07514a7fb8bee3f2ea5a53f36939410ecac4a466620213539a8" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "chrono", "half", "indexmap 2.14.0", @@ -6521,12 +6767,12 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823a9ecee8fd83a68f7165ef13acc840c4d7ed995838ba99d4714b20a2b5e780" +checksum = "ba4d3de89dab8d1aaaf601ae8d71bd07ea88cfca9efc1df5815b982c30f631e1" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "chrono", "half", "indexmap 2.14.0", @@ -6538,12 +6784,12 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f37a91177e2dddb10333546952fcf2b7674b97ebd9449432f24b883d6ea8108" +checksum = "fb19dfe1bd24c17addd761ba4f7000f615e2fa12525871c7baa835dbb3d7f147" dependencies = [ - "arrow-schema 58.4.0", - "base64 0.22.1", + "arrow-schema 59.2.0", + "base64 0.23.0", "chrono", "parquet-variant", "serde_json", @@ -6670,7 +6916,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.9.5", + "rand 0.8.7", "socket2", "thiserror 2.0.19", ] @@ -8090,6 +8336,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -8555,7 +8802,7 @@ dependencies = [ "clap", "futures", "indicatif", - "parquet 58.4.0", + "parquet 59.2.0", "regex", "tabled", "tokio", @@ -9521,11 +9768,11 @@ name = "vortex" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.4.0", + "arrow-array 59.2.0", "codspeed-divan-compat", "fastlanes", "mimalloc", - "parquet 58.4.0", + "parquet 59.2.0", "rand 0.10.2", "serde_json", "tokio", @@ -9591,7 +9838,7 @@ dependencies = [ "arbitrary", "arc-swap", "arcref", - "arrow-buffer 58.4.0", + "arrow-buffer 59.2.0", "async-lock", "bytes", "cfg-if", @@ -9658,12 +9905,12 @@ dependencies = [ name = "vortex-arrow" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-cast 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "codspeed-divan-compat", "itertools 0.14.0", "num-traits", @@ -9686,9 +9933,9 @@ name = "vortex-bench" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "async-trait", "bzip2", "clap", @@ -9708,7 +9955,7 @@ dependencies = [ "noodles-vcf", "parking_lot", "parquet 56.2.1", - "parquet 58.4.0", + "parquet 59.2.0", "rand 0.10.2", "regex", "reqwest 0.13.4", @@ -9751,7 +9998,7 @@ dependencies = [ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", + "arrow-array 59.2.0", "codspeed-divan-compat", "insta", "itertools 0.14.0", @@ -9787,7 +10034,7 @@ dependencies = [ name = "vortex-buffer" version = "0.1.0" dependencies = [ - "arrow-buffer 58.4.0", + "arrow-buffer 59.2.0", "bitvec", "bytes", "codspeed-divan-compat", @@ -9835,13 +10082,13 @@ dependencies = [ name = "vortex-compat" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-select 58.4.0", + "arrow-array 59.2.0", + "arrow-select 59.2.0", "base16ct", "bytes", "clap", "futures", - "parquet 58.4.0", + "parquet 59.2.0", "reqwest 0.13.4", "serde", "serde_json", @@ -9881,11 +10128,11 @@ dependencies = [ name = "vortex-compute" version = "0.1.0" dependencies = [ - "arrow-arith 58.4.0", - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-cast 58.4.0", - "arrow-schema 58.4.0", + "arrow-arith 59.2.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-schema 59.2.0", "codspeed-divan-compat", "num-traits", "rand 0.10.2", @@ -9908,7 +10155,7 @@ name = "vortex-cuda" version = "0.1.0" dependencies = [ "arc-swap", - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "async-trait", "bindgen", "codspeed-criterion-compat-walltime", @@ -9940,7 +10187,7 @@ dependencies = [ name = "vortex-cuda-ffi" version = "0.1.0" dependencies = [ - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "futures", "vortex", "vortex-cuda", @@ -9961,8 +10208,8 @@ name = "vortex-datafusion" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "async-trait", "datafusion", "datafusion-catalog", @@ -10071,7 +10318,7 @@ dependencies = [ name = "vortex-error" version = "0.1.0" dependencies = [ - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "flatbuffers", "jiff", "object_store", @@ -10106,8 +10353,8 @@ dependencies = [ name = "vortex-ffi" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "async-fs", "bytes", "cbindgen", @@ -10280,8 +10527,8 @@ dependencies = [ name = "vortex-jni" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "async-fs", "async-lock", "async-trait", @@ -10305,8 +10552,8 @@ dependencies = [ name = "vortex-json" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "prost 0.14.4", "vortex-array", "vortex-arrow", @@ -10320,8 +10567,8 @@ name = "vortex-layout" version = "0.1.0" dependencies = [ "arcref", - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "async-stream", "async-trait", "bit-vec", @@ -10412,9 +10659,9 @@ dependencies = [ name = "vortex-parquet-variant" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-schema 59.2.0", "chrono", "parquet-variant", "parquet-variant-compute", @@ -10462,9 +10709,10 @@ dependencies = [ name = "vortex-python" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "async-fs", "bytes", "itertools 0.14.0", "log", @@ -10493,7 +10741,7 @@ version = "0.1.0" name = "vortex-python-cuda" version = "0.1.0" dependencies = [ - "arrow-schema 58.4.0", + "arrow-schema 59.2.0", "bytes", "pyo3", "vortex", @@ -10505,9 +10753,9 @@ dependencies = [ name = "vortex-row" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-row 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-row 59.2.0", + "arrow-schema 59.2.0", "codspeed-divan-compat", "mimalloc", "rand 0.10.2", @@ -10603,8 +10851,8 @@ dependencies = [ name = "vortex-spatial" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "codspeed-divan-compat", "geo", "geo-traits", @@ -10652,8 +10900,8 @@ dependencies = [ name = "vortex-tensor" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "codspeed-divan-compat", "half", "itertools 0.14.0", @@ -10675,8 +10923,8 @@ dependencies = [ name = "vortex-test-e2e-cuda" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "futures", "vortex", "vortex-cuda", @@ -10687,8 +10935,8 @@ name = "vortex-tui" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "clap", "console_error_panic_hook", "crossterm", @@ -10700,7 +10948,7 @@ dependencies = [ "humansize", "indicatif", "itertools 0.14.0", - "parquet 58.4.0", + "parquet 59.2.0", "ratatui", "ratzilla", "serde", @@ -10727,9 +10975,9 @@ dependencies = [ name = "vortex-web-wasm" version = "0.1.0" dependencies = [ - "arrow-array 58.4.0", - "arrow-ipc 58.4.0", - "arrow-schema 58.4.0", + "arrow-array 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", "console_error_panic_hook", "futures", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index 65452f18300..648bafc710e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,16 +102,16 @@ anyhow = "1.0.100" arbitrary = "1.3.2" arc-swap = "1.9" arcref = "0.2.0" -arrow-arith = "58.3" -arrow-array = "58.3" -arrow-buffer = "58.3" -arrow-cast = "58.3" -arrow-data = "58.3" -arrow-ipc = "58.3" -arrow-row = "58.3" -arrow-schema = "58.3" -arrow-select = "58.3" -arrow-string = "58.3" +arrow-arith = "59.2" +arrow-array = "59.2" +arrow-buffer = "59.2" +arrow-cast = "59.2" +arrow-data = "59.2" +arrow-ipc = "59.2" +arrow-row = "59.2" +arrow-schema = "59.2" +arrow-select = "59.2" +arrow-string = "59.2" async-fs = "2.2.0" async-lock = "3.4" async-stream = "0.3.6" @@ -205,9 +205,9 @@ opentelemetry = "0.32.0" opentelemetry-otlp = "0.32.0" opentelemetry_sdk = "0.32.0" parking_lot = { version = "0.12.3", features = ["nightly"] } -parquet = "58.3" -parquet-variant = "58.3" -parquet-variant-compute = "58.3" +parquet = "59.2" +parquet-variant = "59.2" +parquet-variant-compute = "59.2" paste = "1.0.15" pco = "1.0.1" percent-encoding = "2.3.2" @@ -442,3 +442,43 @@ incremental = false # This improved build times significantly for default common cases that we use locally [profile.dev.package.vortex-fastlanes] debug = false + +[patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-catalog-listing = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-common-runtime = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource-avro = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource-csv = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource-json = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-doc = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-execution = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-table = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-functions-window-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-macros = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-pruning = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-session = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-sql = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +geoarrow = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } +geoarrow-array = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } +geoarrow-cast = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } +geoarrow-schema = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } diff --git a/vortex-arrow/src/dtype.rs b/vortex-arrow/src/dtype.rs index f58cdf1cafb..d4011ad21e5 100644 --- a/vortex-arrow/src/dtype.rs +++ b/vortex-arrow/src/dtype.rs @@ -335,11 +335,10 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult { let scale = dt.scale(); match precision { - // This code is commented out until DataFusion improves its support for smaller decimals. // // DECIMAL32_MAX_PRECISION - // 0..=9 => DataType::Decimal32(precision, scale), + 0..=9 => DataType::Decimal32(precision, scale), // // DECIMAL64_MAX_PRECISION - // 10..=18 => DataType::Decimal64(precision, scale), + 10..=18 => DataType::Decimal64(precision, scale), // DECIMAL128_MAX_PRECISION 0..=38 => DataType::Decimal128(precision, scale), // DECIMAL256_MAX_PRECISION From cf0a5c4712c0382bef0a2685acff79009737e966 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 10 Aug 2026 13:27:38 +0100 Subject: [PATCH 3/6] touch ups Signed-off-by: Adam Gutglick --- vortex-cuda/build.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index 23ac8618edc..af2af21eadd 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -74,10 +74,9 @@ fn main() { .is_some_and(|n| n.starts_with("bit_unpack_")); match path.extension().and_then(|e| e.to_str()) { + // Only watch hand-written .cuh/.h files, not generated ones + // (generated files are rebuilt when cuda_kernel_generator changes) Some("cuh") | Some("h") if !is_generated => { - // Only watch hand-written .cuh/.h files, not generated ones - // (generated files are rebuilt when cuda_kernel_generator changes) - println!("cargo:rerun-if-changed={}", path.display()); } Some("cu") => { From 9dd15acf482aff46d373e673aaeac3d4401f003d Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 14 Aug 2026 12:47:52 +0100 Subject: [PATCH 4/6] update Signed-off-by: Adam Gutglick --- Cargo.lock | 1176 ++++++++++++++----- Cargo.toml | 104 +- benchmarks/datafusion-bench/src/lib.rs | 9 +- benchmarks/datafusion-bench/src/main.rs | 3 +- benchmarks/lance-bench/src/random_access.rs | 5 +- encodings/parquet-variant/src/array.rs | 26 +- encodings/parquet-variant/src/arrow.rs | 3 +- encodings/parquet-variant/src/kernel.rs | 16 +- vortex-arrow/src/dtype.rs | 12 +- vortex-arrow/src/scalar.rs | 76 +- vortex-bench/src/random_access/mod.rs | 1 + vortex-datafusion/src/persistent/format.rs | 2 +- vortex-tui/src/datafusion_helper.rs | 4 +- 13 files changed, 1039 insertions(+), 398 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88022aeef77..0b8bb3c6be0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -336,6 +336,7 @@ dependencies = [ "arrow-data 58.4.0", "arrow-schema 58.4.0", "chrono", + "chrono-tz", "half", "hashbrown 0.17.1", "num-complex", @@ -581,6 +582,7 @@ dependencies = [ "arrow-schema 58.4.0", "arrow-select 58.4.0", "flatbuffers", + "lz4_flex 0.13.1", "zstd", ] @@ -741,6 +743,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.1", + "serde_core", + "serde_json", ] [[package]] @@ -2271,39 +2275,86 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +dependencies = [ + "arrow 58.4.0", + "arrow-schema 58.4.0", + "async-trait", + "chrono", + "datafusion-catalog 54.1.0", + "datafusion-catalog-listing 54.1.0", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-datasource-arrow 54.1.0", + "datafusion-datasource-csv 54.1.0", + "datafusion-datasource-json 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-functions-aggregate 54.1.0", + "datafusion-functions-nested 54.1.0", + "datafusion-functions-table 54.1.0", + "datafusion-functions-window 54.1.0", + "datafusion-optimizer 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-adapter 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-optimizer 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "datafusion-sql 54.1.0", + "futures", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", "async-trait", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 55.0.0", + "datafusion-catalog-listing 55.0.0", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-datasource-arrow 55.0.0", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", + "datafusion-datasource-csv 55.0.0", + "datafusion-datasource-json 55.0.0", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-aggregate 55.0.0", + "datafusion-functions-nested 55.0.0", + "datafusion-functions-table 55.0.0", + "datafusion-functions-window 55.0.0", + "datafusion-optimizer 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-optimizer 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", + "datafusion-sql 55.0.0", "flate2", "futures", "indexmap 2.14.0", @@ -2328,9 +2379,9 @@ dependencies = [ "anyhow", "clap", "custom-labels", - "datafusion", - "datafusion-common", - "datafusion-physical-plan", + "datafusion 55.0.0", + "datafusion-common 55.0.0", + "datafusion-physical-plan 55.0.0", "futures", "itertools 0.14.0", "object_store", @@ -2351,19 +2402,44 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "dashmap", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "futures", "itertools 0.15.0", "log", @@ -2375,19 +2451,42 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "datafusion-catalog 54.1.0", + "datafusion-common 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-adapter 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 55.0.0", + "datafusion-common 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", "futures", "itertools 0.15.0", "log", @@ -2398,7 +2497,31 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +dependencies = [ + "arrow 58.4.0", + "arrow-ipc 58.4.0", + "arrow-schema 58.4.0", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "libc", + "log", + "object_store", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -2424,7 +2547,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-common-runtime" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "futures", "log", @@ -2434,7 +2568,37 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "bytes", + "chrono", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-adapter 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-compression", @@ -2442,15 +2606,15 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "flate2", "futures", "glob", @@ -2469,20 +2633,44 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +dependencies = [ + "arrow 58.4.0", + "arrow-ipc 58.4.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "futures", "itertools 0.15.0", "object_store", @@ -2491,18 +2679,18 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-avro", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-adapter", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "futures", "object_store", ] @@ -2510,19 +2698,42 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "futures", "object_store", "regex", @@ -2532,19 +2743,42 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-session 54.1.0", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-json" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-session 55.0.0", "futures", "object_store", "tokio", @@ -2553,26 +2787,26 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-pruning 55.0.0", + "datafusion-session 55.0.0", "futures", "itertools 0.15.0", "log", @@ -2585,21 +2819,49 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" + +[[package]] +name = "datafusion-doc" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +dependencies = [ + "arrow 58.4.0", + "arrow-buffer 58.4.0", + "async-trait", + "dashmap", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", "async-trait", "bytes", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", "futures", "log", "object_store", @@ -2615,18 +2877,40 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +dependencies = [ + "arrow 58.4.0", + "arrow-schema 58.4.0", + "async-trait", + "chrono", + "datafusion-common 54.1.0", + "datafusion-doc 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-functions-aggregate-common 54.1.0", + "datafusion-functions-window-common 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-doc 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-functions-window-common 55.0.0", + "datafusion-physical-expr-common 55.0.0", "indexmap 2.14.0", "itertools 0.15.0", "recursive", @@ -2637,10 +2921,22 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "indexmap 2.14.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-expr-common" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", + "datafusion-common 55.0.0", "indexmap 2.14.0", "itertools 0.15.0", ] @@ -2648,7 +2944,39 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +dependencies = [ + "arrow 58.4.0", + "arrow-buffer 58.4.0", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common 54.1.0", + "datafusion-doc 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-macros 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "hex", + "itertools 0.14.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -2657,13 +2985,13 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-doc 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-macros 55.0.0", + "datafusion-physical-expr-common 55.0.0", "hex", "itertools 0.15.0", "log", @@ -2679,17 +3007,38 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-doc 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions-aggregate-common 54.1.0", + "datafusion-macros 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-doc 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-macros 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", "half", "hashbrown 0.17.1", "log", @@ -2699,31 +3048,68 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-physical-expr-common 54.1.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-physical-expr-common 55.0.0", ] [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +dependencies = [ + "arrow 58.4.0", + "arrow-ord 58.4.0", + "datafusion-common 54.1.0", + "datafusion-doc 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-functions-aggregate 54.1.0", + "datafusion-functions-aggregate-common 54.1.0", + "datafusion-macros 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-nested" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-ord 59.2.0", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-doc 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-aggregate 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-macros 55.0.0", + "datafusion-physical-expr-common 55.0.0", "hashbrown 0.17.1", "itertools 0.15.0", "itoa", @@ -2734,49 +3120,103 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +dependencies = [ + "arrow 58.4.0", + "async-trait", + "datafusion-catalog 54.1.0", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-plan 54.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-table" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion-catalog 55.0.0", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-plan 55.0.0", "parking_lot", ] [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-doc 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions-window-common 54.1.0", + "datafusion-macros 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "log", +] + +[[package]] +name = "datafusion-functions-window" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-doc 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions-window-common 55.0.0", + "datafusion-macros 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", "log", ] [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +dependencies = [ + "datafusion-common 54.1.0", + "datafusion-physical-expr-common 54.1.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-physical-expr-common 55.0.0", ] [[package]] name = "datafusion-macros" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +dependencies = [ + "datafusion-doc 54.1.0", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "datafusion-macros" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ - "datafusion-doc", + "datafusion-doc 55.0.0", "quote", "syn 3.0.3", ] @@ -2784,14 +3224,33 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +dependencies = [ + "arrow 58.4.0", + "chrono", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-physical-expr 54.1.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-physical-expr 55.0.0", "indexmap 2.14.0", "itertools 0.15.0", "log", @@ -2803,14 +3262,35 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-functions-aggregate-common 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-physical-expr-common 55.0.0", "half", "hashbrown 0.17.1", "indexmap 2.14.0", @@ -2824,26 +3304,58 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +dependencies = [ + "arrow 58.4.0", + "chrono", + "datafusion-common 54.1.0", + "datafusion-expr-common 54.1.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 55.0.0", + "datafusion-expr-common 55.0.0", "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.15.0", @@ -2854,18 +3366,36 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "datafusion-pruning 54.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-pruning 55.0.0", + "datafusion-session 55.0.0", "itertools 0.15.0", "recursive", ] @@ -2873,7 +3403,40 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +dependencies = [ + "arrow 58.4.0", + "arrow-data 58.4.0", + "arrow-ipc 58.4.0", + "arrow-ord 58.4.0", + "arrow-schema 58.4.0", + "async-trait", + "datafusion-common 54.1.0", + "datafusion-common-runtime 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-functions-aggregate-common 54.1.0", + "datafusion-functions-window-common 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-physical-plan" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "arrow-data 59.2.0", @@ -2882,15 +3445,15 @@ dependencies = [ "arrow-schema 59.2.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-functions-window-common 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", "futures", "half", "hashbrown 0.17.1", @@ -2907,50 +3470,80 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0", + "datafusion-datasource 54.1.0", + "datafusion-expr-common 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-expr-common 54.1.0", + "datafusion-physical-plan 54.1.0", + "log", +] + +[[package]] +name = "datafusion-pruning" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-expr-common 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", "log", ] [[package]] name = "datafusion-session" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +dependencies = [ + "async-trait", + "datafusion-common 54.1.0", + "datafusion-execution 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-plan 54.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow-schema 59.2.0", "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-physical-plan 55.0.0", "parking_lot", ] [[package]] name = "datafusion-spark" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "bigdecimal", "chrono", "crc32fast", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-functions-nested", + "datafusion 55.0.0", + "datafusion-catalog 55.0.0", + "datafusion-common 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-aggregate 55.0.0", + "datafusion-functions-aggregate-common 55.0.0", + "datafusion-functions-nested 55.0.0", "log", "num-traits", "percent-encoding", @@ -2965,14 +3558,32 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +dependencies = [ + "arrow 58.4.0", + "bigdecimal", + "chrono", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions-nested 54.1.0", + "indexmap 2.14.0", + "log", + "regex", + "sqlparser", +] + +[[package]] +name = "datafusion-sql" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions-nested 55.0.0", "indexmap 2.14.0", "log", "recursive", @@ -2983,14 +3594,14 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion?rev=2bfdd4aea181525f5328781b29174d82be0c7fd4#2bfdd4aea181525f5328781b29174d82be0c7fd4" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-55#209fd9406890d97d48c0fc396ce37a5e363270b4" dependencies = [ "arrow 59.2.0", "async-trait", "bigdecimal", "clap", - "datafusion", + "datafusion 55.0.0", "datafusion-spark", "futures", "half", @@ -4877,11 +5488,11 @@ dependencies = [ "crossbeam-queue", "crossbeam-skiplist", "dashmap", - "datafusion", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-physical-expr 54.1.0", + "datafusion-physical-plan 54.1.0", "either", "fst", "futures", @@ -5020,8 +5631,8 @@ dependencies = [ "blake3", "byteorder", "bytes", - "datafusion-common", - "datafusion-sql", + "datafusion-common 54.1.0", + "datafusion-sql 54.1.0", "futures", "itertools 0.14.0", "lance-arrow", @@ -5063,10 +5674,10 @@ dependencies = [ "arrow-select 58.4.0", "async-trait", "chrono", - "datafusion", - "datafusion-common", - "datafusion-functions", - "datafusion-physical-expr", + "datafusion 54.1.0", + "datafusion-common 54.1.0", + "datafusion-functions 54.1.0", + "datafusion-physical-expr 54.1.0", "futures", "jsonb", "lance-arrow", @@ -5164,7 +5775,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "datafusion-common", + "datafusion-common 54.1.0", "futures", "lance-arrow", "lance-core", @@ -5201,10 +5812,10 @@ dependencies = [ "bytes", "chrono", "crossbeam-queue", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion 54.1.0", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", + "datafusion-physical-expr 54.1.0", "dirs", "fst", "futures", @@ -5259,9 +5870,9 @@ dependencies = [ "arrow-select 58.4.0", "async-trait", "bytes", - "datafusion", - "datafusion-common", - "datafusion-expr", + "datafusion 54.1.0", + "datafusion-common 54.1.0", + "datafusion-expr 54.1.0", "futures", "lance-core", "lance-io", @@ -9480,14 +10091,14 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpchgen" version = "2.0.2" -source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=438e9c2dbc25b2fff82c0efc08b3f13b5707874f#438e9c2dbc25b2fff82c0efc08b3f13b5707874f" +source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=f42708c32e7fd816b8ad8d2a880c1802a71b669f#f42708c32e7fd816b8ad8d2a880c1802a71b669f" [[package]] name = "tpchgen-arrow" version = "2.0.2" -source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=438e9c2dbc25b2fff82c0efc08b3f13b5707874f#438e9c2dbc25b2fff82c0efc08b3f13b5707874f" +source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=f42708c32e7fd816b8ad8d2a880c1802a71b669f#f42708c32e7fd816b8ad8d2a880c1802a71b669f" dependencies = [ - "arrow 58.4.0", + "arrow 59.2.0", "tpchgen", ] @@ -10211,20 +10822,20 @@ dependencies = [ "arrow-array 59.2.0", "arrow-schema 59.2.0", "async-trait", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-nested", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion 55.0.0", + "datafusion-catalog 55.0.0", + "datafusion-common 55.0.0", + "datafusion-common-runtime 55.0.0", + "datafusion-datasource 55.0.0", + "datafusion-execution 55.0.0", + "datafusion-expr 55.0.0", + "datafusion-functions 55.0.0", + "datafusion-functions-nested 55.0.0", + "datafusion-physical-expr 55.0.0", + "datafusion-physical-expr-adapter 55.0.0", + "datafusion-physical-expr-common 55.0.0", + "datafusion-physical-plan 55.0.0", + "datafusion-pruning 55.0.0", "futures", "insta", "itertools 0.14.0", @@ -10712,7 +11323,6 @@ dependencies = [ "arrow-array 59.2.0", "arrow-data 59.2.0", "arrow-schema 59.2.0", - "async-fs", "bytes", "itertools 0.14.0", "log", @@ -10881,8 +11491,8 @@ dependencies = [ "anyhow", "async-trait", "bigdecimal", - "datafusion", - "datafusion-functions-nested", + "datafusion 55.0.0", + "datafusion-functions-nested 55.0.0", "datafusion-sqllogictest", "indicatif", "regex", @@ -10940,7 +11550,7 @@ dependencies = [ "clap", "console_error_panic_hook", "crossterm", - "datafusion", + "datafusion 55.0.0", "env_logger", "flatbuffers", "futures", diff --git a/Cargo.toml b/Cargo.toml index 648bafc710e..2d5befaffa5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,21 +139,23 @@ cudarc = { version = "0.19.0", features = [ ] } custom-labels = "0.4.4" dashmap = "6.1.0" -datafusion = { version = "54", default-features = false, features = ["sql"] } -datafusion-catalog = { version = "54" } -datafusion-common = { version = "54" } -datafusion-common-runtime = { version = "54" } -datafusion-datasource = { version = "54", default-features = false } -datafusion-execution = { version = "54" } -datafusion-expr = { version = "54" } -datafusion-functions = { version = "54" } -datafusion-functions-nested = { version = "54" } -datafusion-physical-expr = { version = "54" } -datafusion-physical-expr-adapter = { version = "54" } -datafusion-physical-expr-common = { version = "54" } -datafusion-physical-plan = { version = "54" } -datafusion-pruning = { version = "54" } -datafusion-sqllogictest = { version = "54" } +datafusion = { version = "55.0.0", default-features = false, features = [ + "sql", +] } +datafusion-catalog = { version = "55.0.0" } +datafusion-common = { version = "55.0.0" } +datafusion-common-runtime = { version = "55.0.0" } +datafusion-datasource = { version = "55.0.0", default-features = false } +datafusion-execution = { version = "55.0.0" } +datafusion-expr = { version = "55.0.0" } +datafusion-functions = { version = "55.0.0" } +datafusion-functions-nested = { version = "55.0.0" } +datafusion-physical-expr = { version = "55.0.0" } +datafusion-physical-expr-adapter = { version = "55.0.0" } +datafusion-physical-expr-common = { version = "55.0.0" } +datafusion-physical-plan = { version = "55.0.0" } +datafusion-pruning = { version = "55.0.0" } +datafusion-sqllogictest = { version = "55.0.0" } divan = { package = "codspeed-divan-compat", version = "5.0.0" } enum-iterator = "2.0.0" env_logger = "0.11" @@ -276,8 +278,8 @@ tokio-stream = "0.1.17" tokio-util = "0.7.17" vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" } # Pull these into non-public crates to support DF 58 -tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } -tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } +tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } +tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } tracing = { version = "0.1.41", default-features = false } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3" @@ -444,40 +446,40 @@ incremental = false debug = false [patch.crates-io] -datafusion = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-catalog-listing = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-common-runtime = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource-avro = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource-csv = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource-json = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-doc = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-execution = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-table = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-functions-window-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-macros = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-pruning = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-session = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-sql = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } -datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" } +datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-catalog = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-catalog-listing = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-common-runtime = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource-avro = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource-csv = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource-json = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-doc = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-execution = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-table = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-functions-window-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-macros = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-optimizer = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-pruning = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-sql = { git = "https://github.com/apache/datafusion", branch = "branch-55" } +datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", branch = "branch-55" } geoarrow = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } geoarrow-array = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } geoarrow-cast = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" } diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index aeea953ba3d..c9c89d0c9bf 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -11,9 +11,7 @@ use datafusion::datasource::file_format::csv::CsvFormat; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::provider::DefaultTableFactory; use datafusion::execution::SessionStateBuilder; -use datafusion::execution::cache::DefaultListFilesCache; use datafusion::execution::cache::cache_manager::CacheManagerConfig; -use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::prelude::SessionConfig; use datafusion::prelude::SessionContext; @@ -33,12 +31,7 @@ use vortex_datafusion::VortexTableOptions; pub fn get_session_context() -> SessionContext { let mut rt_builder = RuntimeEnvBuilder::new(); - let file_static_cache = Arc::new(DefaultFileStatisticsCache::default()); - let list_file_cache = Arc::new(DefaultListFilesCache::default()); - let cache_config = CacheManagerConfig::default() - .with_file_statistics_cache(Some(file_static_cache)) - .with_list_files_cache(Some(list_file_cache)); - rt_builder = rt_builder.with_cache_manager(cache_config); + rt_builder = rt_builder.with_cache_manager(CacheManagerConfig::default()); let rt = rt_builder .build_arc() diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 49a9bf92f58..c5f0cc687c5 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -247,8 +247,7 @@ async fn register_benchmark_tables( let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)? .with_table_ref(table_ref.clone()); - let listing_options = ListingOptions::new(Arc::clone(&file_format)) - .with_session_config_options(session.state().config()); + let listing_options = ListingOptions::new(Arc::clone(&file_format)); let mut config = ListingTableConfig::new(table_url).with_listing_options(listing_options); diff --git a/benchmarks/lance-bench/src/random_access.rs b/benchmarks/lance-bench/src/random_access.rs index f38bfca2d5f..ab33eb7605d 100644 --- a/benchmarks/lance-bench/src/random_access.rs +++ b/benchmarks/lance-bench/src/random_access.rs @@ -114,8 +114,7 @@ impl RandomAccessor for LanceRandomAccessor { } async fn take(&self, indices: &[u64]) -> anyhow::Result { - Ok(RandomAccessorRet::RecordBatch( - self.dataset.take(indices, self.projection.clone()).await?, - )) + let batch = self.dataset.take(indices, self.projection.clone()).await?; + Ok(RandomAccessorRet::Native(Box::new(batch))) } } diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index 533819ec852..5d364cae0ef 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -137,15 +137,15 @@ impl ParquetVariant { Validity::NonNullable }); let metadata = session - .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?; + .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?; - let value = arrow_variant - .value_field() - .map(|v| session.from_arrow_array(ArrowArrayRef::clone(v), value_nullable)) - .transpose()?; + let value = Some(session.from_arrow_array( + ArrowArrayRef::clone(arrow_variant.value_column()), + value_nullable, + )?); let typed_value = arrow_variant - .typed_value_field() + .typed_value_column() .map(|tv| session.from_arrow_array(ArrowArrayRef::clone(tv), typed_value_nullable)) .transpose()?; ParquetVariant::try_new(validity, metadata, value, typed_value).map(IntoArray::into_array) @@ -692,7 +692,7 @@ mod tests { } #[test] - fn test_arrow_variant_import_typed_value_only_preserves_storage() -> VortexResult<()> { + fn test_arrow_variant_import_typed_value_only_normalizes_storage() -> VortexResult<()> { let metadata = binary_view_array([b"\x01\x00", b"\x01\x00", b"\x01\x00"]); let typed_value: ArrowArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30])); @@ -711,17 +711,21 @@ mod tests { let parquet_array = vortex_arr .as_opt::() .ok_or_else(|| vortex_err!("expected parquet variant array"))?; - assert!(parquet_array.value().is_none()); + let mut ctx = SESSION.create_execution_ctx(); + let value = parquet_array + .value() + .ok_or_else(|| vortex_err!("expected synthesized value child"))?; + assert!(value.all_invalid(&mut ctx)?); let typed_value = parquet_array .typed_value() .ok_or_else(|| vortex_err!("expected typed_value child"))? .clone() - .execute::(&mut SESSION.create_execution_ctx())?; + .execute::(&mut ctx)?; assert_arrays_eq!( typed_value, - PrimitiveArray::from_iter([10i32, 20, 30]), - &mut SESSION.create_execution_ctx() + PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]), + &mut ctx ); Ok(()) } diff --git a/encodings/parquet-variant/src/arrow.rs b/encodings/parquet-variant/src/arrow.rs index 23590cbb59d..e1c03d704dc 100644 --- a/encodings/parquet-variant/src/arrow.rs +++ b/encodings/parquet-variant/src/arrow.rs @@ -715,7 +715,6 @@ mod tests { let actual = session.arrow().from_arrow_array(exported, &field)?; - assert_arrays_eq!(actual, expected, &mut ctx); - Ok(()) + assert_variant_scalars_eq(&actual, &expected, &session) } } diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index 461d400333c..90aa9eba548 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -841,17 +841,13 @@ mod tests { let metadata = SESSION .arrow() - .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?; - let value = arrow_variant - .value_field() - .map(|value| { - SESSION - .arrow() - .from_arrow_array(ArrowArrayRef::clone(value), value_nullable) - }) - .transpose()?; + .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?; + let value = Some(SESSION.arrow().from_arrow_array( + ArrowArrayRef::clone(arrow_variant.value_column()), + value_nullable, + )?); let typed_value = arrow_variant - .typed_value_field() + .typed_value_column() .map(|typed_value| { SESSION .arrow() diff --git a/vortex-arrow/src/dtype.rs b/vortex-arrow/src/dtype.rs index d4011ad21e5..4641a6644c3 100644 --- a/vortex-arrow/src/dtype.rs +++ b/vortex-arrow/src/dtype.rs @@ -335,12 +335,12 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult { let scale = dt.scale(); match precision { - // // DECIMAL32_MAX_PRECISION + // DECIMAL32_MAX_PRECISION 0..=9 => DataType::Decimal32(precision, scale), - // // DECIMAL64_MAX_PRECISION + // DECIMAL64_MAX_PRECISION 10..=18 => DataType::Decimal64(precision, scale), // DECIMAL128_MAX_PRECISION - 0..=38 => DataType::Decimal128(precision, scale), + 19..=38 => DataType::Decimal128(precision, scale), // DECIMAL256_MAX_PRECISION 39.. => DataType::Decimal256(precision, scale), } @@ -600,7 +600,11 @@ mod test { } #[rstest] - #[case(1, DataType::Decimal128(1, 0))] + #[case(1, DataType::Decimal32(1, 0))] + #[case(9, DataType::Decimal32(9, 0))] + #[case(10, DataType::Decimal64(10, 0))] + #[case(18, DataType::Decimal64(18, 0))] + #[case(19, DataType::Decimal128(19, 0))] #[case(38, DataType::Decimal128(38, 0))] #[case(39, DataType::Decimal256(39, 0))] #[case(76, DataType::Decimal256(76, 0))] diff --git a/vortex-arrow/src/scalar.rs b/vortex-arrow/src/scalar.rs index 09fe7cfcdee..93470b8f0f7 100644 --- a/vortex-arrow/src/scalar.rs +++ b/vortex-arrow/src/scalar.rs @@ -117,19 +117,41 @@ fn decimal_to_arrow(scalar: DecimalScalar<'_>) -> Result, VortexE }; let precision = decimal_dtype.precision(); let scale = decimal_dtype.scale(); - // TODO(joe): Replace with decimal32, etc. once Arrow supports them. match scalar.decimal_value() { Some(value) => { let value = value.as_i256(); - if precision <= 38 { - let value = value.maybe_i128().ok_or_else(|| { + match precision { + 0..=9 => { + let value = value + .maybe_i128() + .and_then(|value| i32::try_from(value).ok()) + .ok_or_else(|| { + vortex_err!( + "Decimal value {value} cannot fit in Arrow Decimal32 for precision {precision}" + ) + })?; + decimal32_scalar(value, precision, scale) + } + 10..=18 => { + let value = value + .maybe_i128() + .and_then(|value| i64::try_from(value).ok()) + .ok_or_else(|| { + vortex_err!( + "Decimal value {value} cannot fit in Arrow Decimal64 for precision {precision}" + ) + })?; + decimal64_scalar(value, precision, scale) + } + 19..=38 => { + let value = value.maybe_i128().ok_or_else(|| { vortex_err!( "Decimal value {value} cannot fit in Arrow Decimal128 for precision {precision}" ) })?; - decimal128_scalar(value, precision, scale) - } else { - decimal256_scalar(value, precision, scale) + decimal128_scalar(value, precision, scale) + } + 39.. => decimal256_scalar(value, precision, scale), } } None => { @@ -142,6 +164,20 @@ fn decimal_to_arrow(scalar: DecimalScalar<'_>) -> Result, VortexE } } +fn decimal32_scalar(value: i32, precision: u8, scale: i8) -> Result, VortexError> { + let array = Decimal32Array::new_scalar(value) + .into_inner() + .with_precision_and_scale(precision, scale)?; + Ok(Arc::new(ArrowScalar::new(array))) +} + +fn decimal64_scalar(value: i64, precision: u8, scale: i8) -> Result, VortexError> { + let array = Decimal64Array::new_scalar(value) + .into_inner() + .with_precision_and_scale(precision, scale)?; + Ok(Arc::new(ArrowScalar::new(array))) +} + fn decimal128_scalar(value: i128, precision: u8, scale: i8) -> Result, VortexError> { let array = Decimal128Array::new_scalar(value) .into_inner() @@ -308,7 +344,7 @@ mod tests { use std::sync::Arc; use arrow_array::Array; - use arrow_array::Decimal128Array; + use arrow_array::Decimal64Array; use arrow_array::Int32Array; use arrow_array::MapArray; use arrow_array::StringViewArray; @@ -492,35 +528,35 @@ mod tests { decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i8, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i8, DataType::Decimal32(5, 2))?; let scalar_i16 = Scalar::decimal( DecimalValue::I16(10000), decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i16, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i16, DataType::Decimal32(5, 2))?; let scalar_i32 = Scalar::decimal( DecimalValue::I32(99999), decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i32, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i32, DataType::Decimal32(5, 2))?; let scalar_i64 = Scalar::decimal( DecimalValue::I64(99999), decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i64, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i64, DataType::Decimal32(5, 2))?; let scalar_i128 = Scalar::decimal( DecimalValue::I128(99999), decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i128, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i128, DataType::Decimal32(5, 2))?; let value_i256 = i256::from_i128(99999); let scalar_i256 = Scalar::decimal( @@ -528,7 +564,7 @@ mod tests { decimal_dtype, Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar_i256, DataType::Decimal128(5, 2))?; + assert_arrow_scalar_data_type(&scalar_i256, DataType::Decimal32(5, 2))?; Ok(()) } @@ -545,21 +581,21 @@ mod tests { } #[test] - fn decimal_i256_with_narrow_precision_exports_decimal128() -> VortexResult<()> { + fn decimal_i256_exports_decimal128() -> VortexResult<()> { let scalar = Scalar::decimal( DecimalValue::I256(i256::from_i128(1234)), - DecimalDType::new(4, 2), + DecimalDType::new(19, 2), Nullability::NonNullable, ); - assert_arrow_scalar_data_type(&scalar, DataType::Decimal128(4, 2)) + assert_arrow_scalar_data_type(&scalar, DataType::Decimal128(19, 2)) } #[test] fn test_null_decimal_to_arrow() -> VortexResult<()> { let decimal_dtype = DecimalDType::new(10, 2); let scalar = Scalar::null(DType::Decimal(decimal_dtype, Nullability::Nullable)); - assert_arrow_scalar_data_type(&scalar, DataType::Decimal128(10, 2))?; + assert_arrow_scalar_data_type(&scalar, DataType::Decimal64(10, 2))?; let decimal_dtype = DecimalDType::new(39, 2); let scalar = Scalar::null(DType::Decimal(decimal_dtype, Nullability::Nullable)); @@ -580,8 +616,8 @@ mod tests { assert!(is_scalar); let decimal = array .as_any() - .downcast_ref::() - .expect("decimal scalar should convert to Decimal128"); + .downcast_ref::() + .expect("decimal scalar should convert to Decimal64"); assert_eq!(decimal.precision(), 12); assert_eq!(decimal.scale(), 3); @@ -667,7 +703,7 @@ mod tests { .as_any() .downcast_ref::() .expect("map scalar should convert to MapArray"); - assert_eq!(map.values().data_type(), &DataType::Decimal128(9, 2)); + assert_eq!(map.values().data_type(), &DataType::Decimal32(9, 2)); let wide_decimal_dtype = DecimalDType::new(39, 2); let dtype = DType::map( diff --git a/vortex-bench/src/random_access/mod.rs b/vortex-bench/src/random_access/mod.rs index 3b9f95f5eaf..808712d121a 100644 --- a/vortex-bench/src/random_access/mod.rs +++ b/vortex-bench/src/random_access/mod.rs @@ -46,6 +46,7 @@ pub trait BenchDataset: Send + Sync { pub enum RandomAccessorRet { RecordBatch(RecordBatch), ArrayRef(ArrayRef), + Native(Box), } /// Trait for format-specific random access (take) operations. diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index 20f1e6f34b3..964c8fa3004 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -107,7 +107,7 @@ const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES: usize = MAX_POSTSCRIPT_SIZE as usi /// let table_url = ListingTableUrl::parse(dir.path().to_str().unwrap())?; /// let config = ListingTableConfig::new(table_url) /// .with_listing_options( -/// ListingOptions::new(format).with_session_config_options(ctx.state().config()), +/// ListingOptions::new(format), /// ) /// .infer_schema(&ctx.state()) /// .await?; diff --git a/vortex-tui/src/datafusion_helper.rs b/vortex-tui/src/datafusion_helper.rs index b02009c45fd..f47d2ec1705 100644 --- a/vortex-tui/src/datafusion_helper.rs +++ b/vortex-tui/src/datafusion_helper.rs @@ -69,9 +69,7 @@ pub async fn create_vortex_context( // it for query must not require a `.vortex` extension. This is safe because the // canonical path above keeps us on the single-file path, so no directory listing // happens that an empty extension could over-match. - let listing_options = ListingOptions::new(format) - .with_file_extension("") - .with_session_config_options(ctx.state().config()); + let listing_options = ListingOptions::new(format).with_file_extension(""); let config = ListingTableConfig::new(table_url) .with_listing_options(listing_options) From d9a9a0d2fcda8a9486a187bb7a56c2138172c8ca Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 14 Aug 2026 12:56:23 +0100 Subject: [PATCH 5/6] clippy Signed-off-by: Adam Gutglick --- vortex-datafusion/src/persistent/cache.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vortex-datafusion/src/persistent/cache.rs b/vortex-datafusion/src/persistent/cache.rs index 46a7bd85dfc..21a32dfda7c 100644 --- a/vortex-datafusion/src/persistent/cache.rs +++ b/vortex-datafusion/src/persistent/cache.rs @@ -41,7 +41,6 @@ impl FileMetadata for CachedVortexMetadata { .unwrap_or(1024 * 64) } - #[expect(clippy::disallowed_types)] fn extra_info(&self) -> HashMap { Default::default() } From 7152f4cac5845c973d16702c6af531dcccc1d423 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 14 Aug 2026 14:14:34 +0100 Subject: [PATCH 6/6] tpch schema Signed-off-by: Adam Gutglick --- vortex-bench/src/tpch/schema.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vortex-bench/src/tpch/schema.rs b/vortex-bench/src/tpch/schema.rs index b8caf7e6e97..0561dcb6790 100644 --- a/vortex-bench/src/tpch/schema.rs +++ b/vortex-bench/src/tpch/schema.rs @@ -42,7 +42,7 @@ pub static PART: LazyLock = LazyLock::new(|| { Field::new("p_type", DataType::Utf8View, false), Field::new("p_size", DataType::Int32, false), Field::new("p_container", DataType::Utf8View, false), - Field::new("p_retailprice", DataType::Decimal128(15, 2), false), + Field::new("p_retailprice", DataType::Decimal64(15, 2), false), Field::new("p_comment", DataType::Utf8View, false), ]) }); @@ -54,7 +54,7 @@ pub static SUPPLIER: LazyLock = LazyLock::new(|| { Field::new("s_address", DataType::Utf8View, false), Field::new("s_nationkey", DataType::Int64, false), Field::new("s_phone", DataType::Utf8View, false), - Field::new("s_acctbal", DataType::Decimal128(15, 2), false), + Field::new("s_acctbal", DataType::Decimal64(15, 2), false), Field::new("s_comment", DataType::Utf8View, false), ]) }); @@ -64,7 +64,7 @@ pub static PARTSUPP: LazyLock = LazyLock::new(|| { Field::new("ps_partkey", DataType::Int64, false), Field::new("ps_suppkey", DataType::Int64, false), Field::new("ps_availqty", DataType::Int32, false), - Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), + Field::new("ps_supplycost", DataType::Decimal64(15, 2), false), Field::new("ps_comment", DataType::Utf8View, false), ]) }); @@ -76,7 +76,7 @@ pub static CUSTOMER: LazyLock = LazyLock::new(|| { Field::new("c_address", DataType::Utf8View, false), Field::new("c_nationkey", DataType::Int64, false), Field::new("c_phone", DataType::Utf8View, false), - Field::new("c_acctbal", DataType::Decimal128(15, 2), false), + Field::new("c_acctbal", DataType::Decimal64(15, 2), false), Field::new("c_mktsegment", DataType::Utf8View, false), Field::new("c_comment", DataType::Utf8View, false), ]) @@ -87,7 +87,7 @@ pub static ORDERS: LazyLock = LazyLock::new(|| { Field::new("o_orderkey", DataType::Int64, false), Field::new("o_custkey", DataType::Int64, false), Field::new("o_orderstatus", DataType::Utf8View, false), - Field::new("o_totalprice", DataType::Decimal128(15, 2), false), + Field::new("o_totalprice", DataType::Decimal64(15, 2), false), Field::new("o_orderdate", DataType::Date32, false), Field::new("o_orderpriority", DataType::Utf8View, false), Field::new("o_clerk", DataType::Utf8View, false), @@ -102,10 +102,10 @@ pub static LINEITEM: LazyLock = LazyLock::new(|| { Field::new("l_partkey", DataType::Int64, false), Field::new("l_suppkey", DataType::Int64, false), Field::new("l_linenumber", DataType::Int32, false), - Field::new("l_quantity", DataType::Decimal128(15, 2), false), - Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), - Field::new("l_discount", DataType::Decimal128(15, 2), false), - Field::new("l_tax", DataType::Decimal128(15, 2), false), + Field::new("l_quantity", DataType::Decimal64(15, 2), false), + Field::new("l_extendedprice", DataType::Decimal64(15, 2), false), + Field::new("l_discount", DataType::Decimal64(15, 2), false), + Field::new("l_tax", DataType::Decimal64(15, 2), false), Field::new("l_returnflag", DataType::Utf8View, false), Field::new("l_linestatus", DataType::Utf8View, false), Field::new("l_shipdate", DataType::Date32, false),