From d0d6558763259aa0797222a4e4930fe0b8af1657 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 19 Mar 2026 10:45:22 -0700 Subject: [PATCH 01/10] Add slicing to BufferHandle Signed-off-by: Nicholas Gates --- vortex-array/public-api.lock | 10 ++++ vortex-array/src/buffer.rs | 81 ++++++++++++++++++++++++++++++++ vortex-cuda/src/device_buffer.rs | 58 +++++++++++++++++++++++ 3 files changed, 149 insertions(+) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index a1e370d08e9..64327ae25df 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -8326,6 +8326,10 @@ pub fn vortex_array::buffer::BufferHandle::as_host_opt(&self) -> core::option::O pub fn vortex_array::buffer::BufferHandle::ensure_aligned(self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult +pub fn vortex_array::buffer::BufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult + +pub fn vortex_array::buffer::BufferHandle::filter_typed(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult + pub fn vortex_array::buffer::BufferHandle::into_host(self) -> futures_core::future::BoxFuture<'static, vortex_buffer::ByteBuffer> pub fn vortex_array::buffer::BufferHandle::into_host_sync(self) -> vortex_buffer::ByteBuffer @@ -8400,6 +8404,8 @@ pub fn vortex_array::buffer::DeviceBuffer::copy_to_host(&self, alignment: vortex pub fn vortex_array::buffer::DeviceBuffer::copy_to_host_sync(&self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult +pub fn vortex_array::buffer::DeviceBuffer::filter(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult> + pub fn vortex_array::buffer::DeviceBuffer::is_empty(&self) -> bool pub fn vortex_array::buffer::DeviceBuffer::len(&self) -> usize @@ -8408,10 +8414,14 @@ pub fn vortex_array::buffer::DeviceBuffer::slice(&self, range: core::ops::range: pub trait vortex_array::buffer::DeviceBufferExt: vortex_array::buffer::DeviceBuffer +pub fn vortex_array::buffer::DeviceBufferExt::filter_typed(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult> + pub fn vortex_array::buffer::DeviceBufferExt::slice_typed(&self, range: core::ops::range::Range) -> alloc::sync::Arc impl vortex_array::buffer::DeviceBufferExt for B +pub fn B::filter_typed(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult> + pub fn B::slice_typed(&self, range: core::ops::range::Range) -> alloc::sync::Arc pub mod vortex_array::builders diff --git a/vortex-array/src/buffer.rs b/vortex-array/src/buffer.rs index 07baf843ead..df511e95c3e 100644 --- a/vortex-array/src/buffer.rs +++ b/vortex-array/src/buffer.rs @@ -12,6 +12,7 @@ use futures::future::BoxFuture; use vortex_buffer::ALIGNMENT_TO_HOST_COPY; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; +use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_utils::dyn_traits::DynEq; @@ -87,6 +88,16 @@ pub trait DeviceBuffer: 'static + Send + Sync + Debug + DynEq + DynHash { /// Note that slice indices are in byte units. fn slice(&self, range: Range) -> Arc; + /// Select and concatenate multiple byte ranges from this buffer into a new buffer. + /// + /// Unlike [`slice`](DeviceBuffer::slice), this method allocates new memory and copies the + /// selected ranges into a contiguous buffer. + /// + /// # Errors + /// + /// Returns an error if the device cannot allocate memory or copy the data. + fn filter(&self, ranges: &[Range]) -> VortexResult>; + /// Return a buffer with the given alignment. Where possible, this will be zero-copy. /// /// # Errors @@ -98,6 +109,16 @@ pub trait DeviceBuffer: 'static + Send + Sync + Debug + DynEq + DynHash { pub trait DeviceBufferExt: DeviceBuffer { /// Slice a range of elements `T` out of the device buffer. fn slice_typed(&self, range: Range) -> Arc; + + /// Select and concatenate multiple element ranges of type `T` from this buffer. + /// + /// # Errors + /// + /// Returns an error if the device cannot allocate memory or copy the data. + fn filter_typed( + &self, + ranges: &[Range], + ) -> VortexResult>; } impl DeviceBufferExt for B { @@ -106,6 +127,17 @@ impl DeviceBufferExt for B { let end_bytes = range.end * size_of::(); self.slice(start_bytes..end_bytes) } + + fn filter_typed( + &self, + ranges: &[Range], + ) -> VortexResult> { + let byte_ranges: Vec> = ranges + .iter() + .map(|r| (r.start * size_of::())..(r.end * size_of::())) + .collect(); + self.filter(&byte_ranges) + } } impl Hash for dyn DeviceBuffer { @@ -202,6 +234,55 @@ impl BufferHandle { } } + /// Select and concatenate multiple byte ranges from this buffer into a new buffer. + /// + /// Unlike [`slice`](BufferHandle::slice), this method allocates a new buffer and copies + /// the selected ranges. + /// + /// # Example + /// + /// ``` + /// # use vortex_array::buffer::BufferHandle; + /// # use vortex_buffer::buffer; + /// let handle = BufferHandle::new_host(buffer![1u8, 2, 3, 4, 5, 6]); + /// let filtered = handle.filter(&[0..2, 4..6]).unwrap(); + /// assert_eq!(filtered.unwrap_host(), buffer![1u8, 2, 5, 6]); + /// ``` + pub fn filter(&self, ranges: &[Range]) -> VortexResult { + match &self.0 { + Inner::Host(host) => { + let total_len: usize = ranges.iter().map(|r| r.len()).sum(); + let mut result = ByteBufferMut::with_capacity_aligned(total_len, host.alignment()); + for range in ranges { + result.extend_from_slice(&host.as_slice()[range.start..range.end]); + } + Ok(BufferHandle::new_host(result.freeze())) + } + Inner::Device(device) => Ok(BufferHandle::new_device(device.filter(ranges)?)), + } + } + + /// Select and concatenate multiple element ranges of type `T` from this buffer. + /// + /// # Example + /// + /// ``` + /// # use vortex_array::buffer::BufferHandle; + /// # use vortex_buffer::{buffer, Buffer}; + /// let values = buffer![1u32, 2u32, 3u32, 4u32, 5u32, 6u32]; + /// let handle = BufferHandle::new_host(values.into_byte_buffer()); + /// let filtered = handle.filter_typed::(&[0..2, 4..6]).unwrap(); + /// let result = Buffer::::from_byte_buffer(filtered.to_host_sync()); + /// assert_eq!(result, buffer![1, 2, 5, 6]); + /// ``` + pub fn filter_typed(&self, ranges: &[Range]) -> VortexResult { + let byte_ranges: Vec> = ranges + .iter() + .map(|r| (r.start * size_of::())..(r.end * size_of::())) + .collect(); + self.filter(&byte_ranges) + } + /// Reinterpret the pointee as a buffer of `T` and slice the provided element range. /// /// # Example diff --git a/vortex-cuda/src/device_buffer.rs b/vortex-cuda/src/device_buffer.rs index c8d2841f10a..49af1728622 100644 --- a/vortex-cuda/src/device_buffer.rs +++ b/vortex-cuda/src/device_buffer.rs @@ -287,6 +287,64 @@ impl DeviceBuffer for CudaDeviceBuffer { })) } + fn filter(&self, ranges: &[Range]) -> VortexResult> { + let total_len: usize = ranges.iter().map(|r| r.len()).sum(); + + let stream = self.allocation.stream(); + stream + .context() + .bind_to_thread() + .map_err(|e| vortex_err!("Failed to bind CUDA context: {}", e))?; + + if total_len == 0 { + let dst_slice: CudaSlice = unsafe { + stream + .alloc::(0) + .map_err(|e| vortex_err!("Failed to allocate device memory: {}", e))? + }; + return Ok(Arc::new(CudaDeviceBuffer::new(dst_slice))); + } + + // Allocate new device memory for the filtered result. + let dst_slice: CudaSlice = unsafe { + stream + .alloc::(total_len) + .map_err(|e| vortex_err!("Failed to allocate device memory for filter: {}", e))? + }; + + let (dst_base_ptr, _) = dst_slice.device_ptr(stream); + + // Copy each selected range from source to the new contiguous buffer. + let mut dst_offset: u64 = 0; + for range in ranges { + assert!( + range.end <= self.len, + "Filter range end {} exceeds buffer size {}", + range.end, + self.len + ); + let src_ptr = self.device_ptr + (self.offset + range.start) as u64; + let len = range.len(); + if len > 0 { + unsafe { + sys::cuMemcpyDtoDAsync_v2( + dst_base_ptr + dst_offset, + src_ptr, + len, + stream.cu_stream(), + ) + .result() + .map_err(|e| { + vortex_err!("Failed to copy device memory during filter: {}", e) + })?; + } + } + dst_offset += len as u64; + } + + Ok(Arc::new(CudaDeviceBuffer::new(dst_slice))) + } + /// Slices the CUDA device buffer to a subrange. /// /// This is a byte range, not elements range, due to the DeviceBuffer interface. From 9fd8a54ab63bb8e1431efc4eb69931d6fc3e3958 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 19 Mar 2026 11:25:55 -0700 Subject: [PATCH 02/10] Add slicing to BufferHandle Signed-off-by: Nicholas Gates --- vortex-layout/public-api.lock | 26 +++ vortex-layout/src/buffer.rs | 356 ++++++++++++++++++++++++++++++++++ vortex-layout/src/lib.rs | 1 + 3 files changed, 383 insertions(+) create mode 100644 vortex-layout/src/buffer.rs diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 2cb3dec9d9a..4515fc59412 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -6,6 +6,32 @@ pub mod vortex_layout::aliases::paste pub use vortex_layout::aliases::paste::paste +pub mod vortex_layout::buffer + +pub struct vortex_layout::buffer::LazyBufferHandle + +impl vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::byte_ranges(&self) -> core::option::Option<&[core::ops::range::Range]> + +pub fn vortex_layout::buffer::LazyBufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> Self + +pub async fn vortex_layout::buffer::LazyBufferHandle::materialize(&self) -> vortex_error::VortexResult + +pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId) -> Self + +pub fn vortex_layout::buffer::LazyBufferHandle::segment_id(&self) -> vortex_layout::segments::SegmentId + +pub fn vortex_layout::buffer::LazyBufferHandle::slice(&self, range: core::ops::range::Range) -> Self + +impl core::clone::Clone for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::clone(&self) -> vortex_layout::buffer::LazyBufferHandle + +impl core::fmt::Debug for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub mod vortex_layout::display pub struct vortex_layout::display::DisplayLayoutTree diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs new file mode 100644 index 00000000000..79235c8d583 --- /dev/null +++ b/vortex-layout/src/buffer.rs @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Debug; +use std::fmt::Formatter; +use std::ops::Range; +use std::sync::Arc; + +use vortex_array::buffer::BufferHandle; +use vortex_error::VortexResult; + +use crate::segments::SegmentId; +use crate::segments::SegmentSource; + +/// A lazy buffer handle that defers segment I/O until materialization. +/// +/// Wraps a [`SegmentSource`] and [`SegmentId`] together with an optional byte +/// selection. Operations like [`slice`](Self::slice) and [`filter`](Self::filter) +/// accumulate without triggering I/O, allowing the system to determine the exact +/// byte ranges needed before reading. +#[derive(Clone)] +pub struct LazyBufferHandle { + source: Arc, + segment_id: SegmentId, + selection: Selection, +} + +/// Byte selection within a segment buffer. +#[derive(Clone, Debug)] +enum Selection { + /// The entire segment is selected. + All, + /// A single contiguous byte range within the segment. + Range(Range), + /// Multiple non-overlapping, sorted byte ranges within the segment. + Ranges(Arc<[Range]>), +} + +impl LazyBufferHandle { + /// Create a new lazy handle selecting the entire segment. + pub fn new(source: Arc, segment_id: SegmentId) -> Self { + Self { + source, + segment_id, + selection: Selection::All, + } + } + + /// Returns the segment ID. + pub fn segment_id(&self) -> SegmentId { + self.segment_id + } + + /// Returns the byte ranges that will be read from the segment, or `None` if the + /// entire segment is selected. + pub fn byte_ranges(&self) -> Option<&[Range]> { + match &self.selection { + Selection::All => None, + Selection::Range(r) => Some(std::slice::from_ref(r)), + Selection::Ranges(ranges) => Some(ranges), + } + } + + /// Narrow to a contiguous byte range within the current selection. + /// + /// The range is interpreted relative to the current selection's logical byte + /// offsets (i.e., offsets into the bytes that would be produced by materializing + /// the current selection). + /// + /// # Panics + /// + /// Panics if the range exceeds the bounds of the current selection (when + /// those bounds are known). + pub fn slice(&self, range: Range) -> Self { + let selection = match &self.selection { + Selection::All => Selection::Range(range), + Selection::Range(base) => { + let start = base.start + range.start; + let end = base.start + range.end; + assert!( + end <= base.end, + "slice range {}..{} exceeds current selection 0..{}", + range.start, + range.end, + base.len(), + ); + Selection::Range(start..end) + } + Selection::Ranges(existing) => slice_into_ranges(existing, range), + }; + Self { + source: Arc::clone(&self.source), + segment_id: self.segment_id, + selection, + } + } + + /// Select multiple byte ranges within the current view. + /// + /// Ranges are interpreted relative to the current selection's logical byte + /// offsets and must be sorted and non-overlapping. + /// + /// # Panics + /// + /// Panics if any range exceeds the bounds of the current selection (when + /// those bounds are known). + pub fn filter(&self, ranges: &[Range]) -> Self { + let selection = match &self.selection { + Selection::All => Selection::Ranges(Arc::from(ranges)), + Selection::Range(base) => { + let absolute: Arc<[Range]> = ranges + .iter() + .map(|r| { + let abs = (base.start + r.start)..(base.start + r.end); + assert!( + abs.end <= base.end, + "filter range {}..{} exceeds current selection 0..{}", + r.start, + r.end, + base.len(), + ); + abs + }) + .collect(); + Selection::Ranges(absolute) + } + Selection::Ranges(existing) => { + // Each input range is relative to the concatenated output of + // the existing ranges. Map them back to absolute segment offsets. + let mut result = Vec::new(); + for r in ranges { + match slice_into_ranges(existing, r.clone()) { + Selection::All => unreachable!(), + Selection::Range(abs) => result.push(abs), + Selection::Ranges(abs) => result.extend_from_slice(&abs), + } + } + Selection::Ranges(result.into()) + } + }; + Self { + source: Arc::clone(&self.source), + segment_id: self.segment_id, + selection, + } + } + + /// Materialize the lazy buffer by performing I/O and applying the selection. + /// + /// # Errors + /// + /// Returns an error if the segment cannot be loaded or the selection cannot be + /// applied. + pub async fn materialize(&self) -> VortexResult { + let buffer = self.source.request(self.segment_id).await?; + match &self.selection { + Selection::All => Ok(buffer), + Selection::Range(range) => Ok(buffer.slice(range.clone())), + Selection::Ranges(ranges) => buffer.filter(ranges), + } + } +} + +impl Debug for LazyBufferHandle { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyBufferHandle") + .field("segment_id", &self.segment_id) + .field("selection", &self.selection) + .finish() + } +} + +/// Map a logical byte range into the given set of existing absolute ranges. +/// +/// The `range` is interpreted as an offset into the concatenated output of +/// `existing`. The result contains the corresponding absolute segment byte +/// ranges. +/// +/// # Example +/// +/// Given `existing = [10..20, 30..50]` (30 logical bytes), +/// `slice_into_ranges(existing, 5..25)` returns `Ranges([15..20, 30..40])`. +fn slice_into_ranges(existing: &[Range], range: Range) -> Selection { + let mut result = Vec::new(); + let mut offset: usize = 0; + + for er in existing { + let er_len = er.len(); + let next_offset = offset + er_len; + + // Skip ranges entirely before the slice start. + if next_offset <= range.start { + offset = next_offset; + continue; + } + + // Stop once past the slice end. + if offset >= range.end { + break; + } + + // Intersect [range.start, range.end) with the logical span [offset, next_offset) + // and map back to absolute segment bytes. + let rel_start = range.start.saturating_sub(offset); + let rel_end = (range.end - offset).min(er_len); + result.push((er.start + rel_start)..(er.start + rel_end)); + + offset = next_offset; + } + + match result.len() { + 0 => Selection::Ranges(Arc::from([])), + 1 => Selection::Range(result.remove(0)), + _ => Selection::Ranges(result.into()), + } +} + +#[cfg(test)] +mod tests { + use std::ops::Range; + use std::sync::Arc; + + use futures::FutureExt; + use vortex_array::buffer::BufferHandle; + use vortex_buffer::ByteBuffer; + use vortex_error::VortexResult; + use vortex_io::runtime::single::block_on; + + use super::*; + use crate::segments::SegmentFuture; + use crate::segments::SegmentId; + use crate::segments::SegmentSource; + + /// A trivial in-memory segment source for tests. + struct SingleSegment(BufferHandle); + + impl SegmentSource for SingleSegment { + fn request(&self, _id: SegmentId) -> SegmentFuture { + let handle = self.0.clone(); + async move { Ok(handle) }.boxed() + } + } + + fn lazy(data: &[u8]) -> LazyBufferHandle { + let buf = BufferHandle::new_host(ByteBuffer::copy_from(data)); + LazyBufferHandle::new(Arc::new(SingleSegment(buf)), SegmentId::from(0u32)) + } + + #[test] + fn materialize_all() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[1, 2, 3, 4, 5, 6]).materialize().await?; + assert_eq!(handle.unwrap_host().as_slice(), &[1, 2, 3, 4, 5, 6]); + Ok(()) + }) + } + + #[test] + fn slice_single() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[1, 2, 3, 4, 5, 6]).slice(1..5).materialize().await?; + assert_eq!(handle.unwrap_host().as_slice(), &[2, 3, 4, 5]); + Ok(()) + }) + } + + #[test] + fn slice_of_slice() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[1, 2, 3, 4, 5, 6]) + .slice(1..5) + .slice(1..3) + .materialize() + .await?; + assert_eq!(handle.unwrap_host().as_slice(), &[3, 4]); + Ok(()) + }) + } + + #[test] + fn filter_from_all() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[1, 2, 3, 4, 5, 6]) + .filter(&[0..2, 4..6]) + .materialize() + .await?; + assert_eq!(handle.unwrap_host().as_slice(), &[1, 2, 5, 6]); + Ok(()) + }) + } + + #[test] + fn filter_of_slice() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[1, 2, 3, 4, 5, 6]) + .slice(1..5) + .filter(&[0..1, 2..4]) + .materialize() + .await?; + // slice(1..5) → [2, 3, 4, 5] + // filter([0..1, 2..4]) → [2, 4, 5] + assert_eq!(handle.unwrap_host().as_slice(), &[2, 4, 5]); + Ok(()) + }) + } + + #[test] + fn slice_of_filter() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[10, 20, 30, 40, 50, 60]) + .filter(&[0..2, 4..6]) + .slice(1..3) + .materialize() + .await?; + // filter([0..2, 4..6]) selects [10, 20, 50, 60] (4 logical bytes) + // slice(1..3) → logical bytes 1..3 → [20, 50] + assert_eq!(handle.unwrap_host().as_slice(), &[20, 50]); + Ok(()) + }) + } + + #[test] + fn filter_of_filter() -> VortexResult<()> { + block_on(|_| async { + let handle = lazy(&[10, 20, 30, 40, 50, 60]) + .filter(&[0..2, 4..6]) + .filter(&[0..1, 3..4]) + .materialize() + .await?; + // First filter selects [10, 20, 50, 60] (logical bytes 0..4) + // Second filter selects logical [0..1, 3..4] → [10, 60] + assert_eq!(handle.unwrap_host().as_slice(), &[10, 60]); + Ok(()) + }) + } + + #[test] + fn byte_ranges_none_for_all() { + let lazy = lazy(&[1, 2, 3]); + assert!(lazy.byte_ranges().is_none()); + } + + #[test] + fn byte_ranges_after_slice() { + let lazy = lazy(&[1, 2, 3, 4, 5]).slice(1..4); + let expected = [Range { start: 1, end: 4 }]; + assert_eq!(lazy.byte_ranges(), Some(expected.as_slice())); + } + + #[test] + fn byte_ranges_after_filter() { + let lazy = lazy(&[1, 2, 3, 4, 5]).filter(&[0..2, 3..5]); + let expected = [Range { start: 0, end: 2 }, Range { start: 3, end: 5 }]; + assert_eq!(lazy.byte_ranges(), Some(expected.as_slice())); + } +} diff --git a/vortex-layout/src/lib.rs b/vortex-layout/src/lib.rs index f7f3787da79..24595a3f6b4 100644 --- a/vortex-layout/src/lib.rs +++ b/vortex-layout/src/lib.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +pub mod buffer; pub mod layouts; pub use children::*; From ca88f56b4a5b345b0c9926fd8a6d7ccc4b81a285 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 19 Mar 2026 13:01:29 -0700 Subject: [PATCH 03/10] Add slicing to BufferHandle Signed-off-by: Nicholas Gates --- vortex-array/public-api.lock | 134 ++++++++++++++ vortex-array/src/array/mod.rs | 13 ++ vortex-array/src/vtable/dyn_.rs | 7 + vortex-array/src/vtable/mod.rs | 17 ++ vortex-layout/public-api.lock | 40 ++++- vortex-layout/src/buffer.rs | 218 ++++++++++++++++++++++- vortex-layout/src/layouts/flat/reader.rs | 52 ++++-- 7 files changed, 458 insertions(+), 23 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 64327ae25df..67f1955e17c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -930,6 +930,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::bool::BoolArray @@ -1138,6 +1140,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -1304,6 +1308,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -1510,6 +1516,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::decimal::DecimalArray @@ -1718,6 +1726,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -1822,6 +1832,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -2118,6 +2130,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -2254,6 +2268,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -2470,6 +2486,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::fixed_size_list::FixedSizeListArray @@ -2618,6 +2636,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::list::ListArray @@ -2786,6 +2806,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::listview::ListViewArray @@ -2964,6 +2986,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::masked::MaskedArray @@ -3102,6 +3126,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -3312,6 +3338,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::primitive::PrimitiveArray @@ -3620,6 +3648,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3702,6 +3732,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -3820,6 +3852,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -4078,6 +4112,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::struct_::StructArray @@ -4294,6 +4330,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbin::VarBinArray @@ -4696,6 +4734,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbinview::VarBinViewArray @@ -4886,6 +4926,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -5026,6 +5068,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::BoolArray @@ -5206,6 +5250,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -5370,6 +5416,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -5512,6 +5560,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::DecimalArray @@ -5688,6 +5738,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -5842,6 +5894,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -5976,6 +6030,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -6108,6 +6164,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::FixedSizeListArray @@ -6254,6 +6312,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListArray @@ -6400,6 +6460,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListViewArray @@ -6558,6 +6620,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::MaskedArray @@ -6692,6 +6756,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -6834,6 +6900,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::PrimitiveArray @@ -7068,6 +7136,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -7140,6 +7210,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -7256,6 +7328,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -7392,6 +7466,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::StructArray @@ -7632,6 +7708,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinArray @@ -7848,6 +7926,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinViewArray @@ -8026,6 +8106,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -20402,6 +20484,8 @@ pub fn vortex_array::vtable::DynVTable::reduce(&self, array: &vortex_array::Arra pub fn vortex_array::vtable::DynVTable::reduce_parent(&self, array: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::vtable::DynVTable::with_buffers(&self, array: &vortex_array::ArrayRef, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::vtable::DynVTable::with_children(&self, array: &vortex_array::ArrayRef, children: alloc::vec::Vec) -> vortex_error::VortexResult pub trait vortex_array::vtable::OperationsVTable @@ -20544,6 +20628,8 @@ pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::vtable::VTable::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::vtable::VTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::vtable::VTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Bool @@ -20598,6 +20684,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Chunked @@ -20652,6 +20740,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Constant @@ -20706,6 +20796,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Decimal @@ -20760,6 +20852,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Extension @@ -20814,6 +20908,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Filter @@ -20868,6 +20964,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::FixedSizeList @@ -20922,6 +21020,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::List @@ -20976,6 +21076,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::ListView @@ -21030,6 +21132,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Masked @@ -21084,6 +21188,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Primitive @@ -21138,6 +21244,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Shared @@ -21192,6 +21300,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Struct @@ -21246,6 +21356,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBin @@ -21300,6 +21412,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBinView @@ -21354,6 +21468,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Variant @@ -21408,6 +21524,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::dict::Dict @@ -21462,6 +21580,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::null::Null @@ -21516,6 +21636,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21570,6 +21692,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::slice::Slice @@ -21624,6 +21748,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub trait vortex_array::vtable::ValidityChild @@ -22100,6 +22226,8 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::ArrayAdapter::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::scalar_fn::ReduceNode for vortex_array::ArrayAdapter @@ -22484,6 +22612,8 @@ pub fn vortex_array::DynArray::validity_mask(&self) -> vortex_error::VortexResul pub fn vortex_array::DynArray::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::DynArray::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::DynArray::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::DynArray for alloc::sync::Arc @@ -22534,6 +22664,8 @@ pub fn alloc::sync::Arc::validity_mask(&self) -> vor pub fn alloc::sync::Arc::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn alloc::sync::Arc::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn alloc::sync::Arc::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::DynArray for vortex_array::ArrayAdapter @@ -22584,6 +22716,8 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::ArrayAdapter::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult pub trait vortex_array::DynArrayEq: vortex_array::hash::private::SealedEq diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 79eb980d317..154a44a4fd2 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -165,6 +165,9 @@ pub trait DynArray: /// Replaces the children of the array with the given array references. fn with_children(&self, children: Vec) -> VortexResult; + + /// Replaces the buffers of the array with the given buffer handles. + fn with_buffers(&self, buffers: Vec) -> VortexResult; } impl DynArray for Arc { @@ -278,6 +281,10 @@ impl DynArray for Arc { fn with_children(&self, children: Vec) -> VortexResult { self.as_ref().with_children(children) } + + fn with_buffers(&self, buffers: Vec) -> VortexResult { + self.as_ref().with_buffers(buffers) + } } /// A reference counted pointer to a dynamic [`DynArray`] trait object. @@ -663,6 +670,12 @@ impl DynArray for ArrayAdapter { V::with_children(&mut this, children)?; Ok(this.into_array()) } + + fn with_buffers(&self, buffers: Vec) -> VortexResult { + let mut this = self.0.clone(); + V::with_buffers(&mut this, buffers)?; + Ok(this.into_array()) + } } impl ArrayHash for ArrayAdapter { diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 64ca5e99c50..cdb6e2f3683 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -48,6 +48,7 @@ pub trait DynVTable: 'static + private::Sealed + Send + Sync + Debug { session: &VortexSession, ) -> VortexResult; fn with_children(&self, array: &ArrayRef, children: Vec) -> VortexResult; + fn with_buffers(&self, array: &ArrayRef, buffers: Vec) -> VortexResult; /// See [`VTable::reduce`] fn reduce(&self, array: &ArrayRef) -> VortexResult>; @@ -101,6 +102,12 @@ impl DynVTable for ArrayVTableAdapter { Ok(array.into_array()) } + fn with_buffers(&self, array: &ArrayRef, buffers: Vec) -> VortexResult { + let mut array = array.as_::().clone(); + V::with_buffers(&mut array, buffers)?; + Ok(array.into_array()) + } + fn reduce(&self, array: &ArrayRef) -> VortexResult> { let Some(reduced) = V::reduce(downcast::(array))? else { return Ok(None); diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 2b7ab9bd620..c96ec7a3b9c 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -181,6 +181,23 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// of children must be expected. fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()>; + /// Replaces the buffers in `array` with `buffers`. + /// + /// The default implementation rebuilds the array via [`build()`](VTable::build), which + /// re-runs all validation. This is correct for replacing lazy device buffers with + /// materialized host buffers. + fn with_buffers(array: &mut Self::Array, buffers: Vec) -> VortexResult<()> { + let metadata = Self::metadata(array)?; + let dtype = Self::dtype(array).clone(); + let len = Self::len(array); + let children: Vec = (0..Self::nchildren(array)) + .map(|i| Self::child(array, i)) + .collect(); + let children_slice: &[ArrayRef] = &children; + *array = Self::build(&dtype, len, &metadata, &buffers, &children_slice)?; + Ok(()) + } + /// Execute this array by returning an [`ExecutionStep`] that tells the scheduler what to /// do next. /// diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 4515fc59412..e0bf15ea6e3 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -12,13 +12,19 @@ pub struct vortex_layout::buffer::LazyBufferHandle impl vortex_layout::buffer::LazyBufferHandle +pub fn vortex_layout::buffer::LazyBufferHandle::alignment(&self) -> vortex_buffer::alignment::Alignment + pub fn vortex_layout::buffer::LazyBufferHandle::byte_ranges(&self) -> core::option::Option<&[core::ops::range::Range]> pub fn vortex_layout::buffer::LazyBufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> Self +pub fn vortex_layout::buffer::LazyBufferHandle::is_empty(&self) -> bool + +pub fn vortex_layout::buffer::LazyBufferHandle::len(&self) -> usize + pub async fn vortex_layout::buffer::LazyBufferHandle::materialize(&self) -> vortex_error::VortexResult -pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId) -> Self +pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId, alignment: vortex_buffer::alignment::Alignment) -> Self pub fn vortex_layout::buffer::LazyBufferHandle::segment_id(&self) -> vortex_layout::segments::SegmentId @@ -28,10 +34,42 @@ impl core::clone::Clone for vortex_layout::buffer::LazyBufferHandle pub fn vortex_layout::buffer::LazyBufferHandle::clone(&self) -> vortex_layout::buffer::LazyBufferHandle +impl core::cmp::Eq for vortex_layout::buffer::LazyBufferHandle + +impl core::cmp::PartialEq for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::eq(&self, other: &Self) -> bool + impl core::fmt::Debug for vortex_layout::buffer::LazyBufferHandle pub fn vortex_layout::buffer::LazyBufferHandle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::hash(&self, state: &mut H) + +impl vortex_array::buffer::DeviceBuffer for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::aligned(self: alloc::sync::Arc, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult> + +pub fn vortex_layout::buffer::LazyBufferHandle::alignment(&self) -> vortex_buffer::alignment::Alignment + +pub fn vortex_layout::buffer::LazyBufferHandle::as_any(&self) -> &dyn core::any::Any + +pub fn vortex_layout::buffer::LazyBufferHandle::copy_to_host(&self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult>> + +pub fn vortex_layout::buffer::LazyBufferHandle::copy_to_host_sync(&self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult + +pub fn vortex_layout::buffer::LazyBufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult> + +pub fn vortex_layout::buffer::LazyBufferHandle::len(&self) -> usize + +pub fn vortex_layout::buffer::LazyBufferHandle::slice(&self, range: core::ops::range::Range) -> alloc::sync::Arc + +pub fn vortex_layout::buffer::create_lazy_array_parts(array_tree: vortex_buffer::ByteBuffer, source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId) -> vortex_error::VortexResult + +pub async fn vortex_layout::buffer::materialize_recursive(array: &vortex_array::array::ArrayRef) -> vortex_error::VortexResult + pub mod vortex_layout::display pub struct vortex_layout::display::DisplayLayoutTree diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs index 79235c8d583..eab11c644df 100644 --- a/vortex-layout/src/buffer.rs +++ b/vortex-layout/src/buffer.rs @@ -1,13 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::any::Any; use std::fmt::Debug; use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; use std::ops::Range; use std::sync::Arc; +use futures::FutureExt; +use futures::future::BoxFuture; +use vortex_array::ArrayRef; +use vortex_array::ArrayVisitor; +use vortex_array::DynArray; use vortex_array::buffer::BufferHandle; +use vortex_array::buffer::DeviceBuffer; +use vortex_array::serde::ArrayParts; +use vortex_buffer::Alignment; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_error::vortex_panic; use crate::segments::SegmentId; use crate::segments::SegmentSource; @@ -23,10 +36,11 @@ pub struct LazyBufferHandle { source: Arc, segment_id: SegmentId, selection: Selection, + alignment: Alignment, } /// Byte selection within a segment buffer. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] enum Selection { /// The entire segment is selected. All, @@ -36,16 +50,48 @@ enum Selection { Ranges(Arc<[Range]>), } +#[allow(clippy::same_name_method)] impl LazyBufferHandle { /// Create a new lazy handle selecting the entire segment. - pub fn new(source: Arc, segment_id: SegmentId) -> Self { + pub fn new( + source: Arc, + segment_id: SegmentId, + alignment: Alignment, + ) -> Self { Self { source, segment_id, selection: Selection::All, + alignment, } } + /// Returns the length of the selected byte range(s). + /// + /// # Panics + /// + /// Panics if the entire segment is selected ([`Selection::All`]) since the + /// length is not known without performing I/O. + pub fn len(&self) -> usize { + match &self.selection { + Selection::All => { + vortex_panic!("len() is not available for Selection::All; slice first") + } + Selection::Range(r) => r.len(), + Selection::Ranges(rs) => rs.iter().map(|r| r.len()).sum(), + } + } + + /// Returns whether the buffer is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the alignment of the buffer. + pub fn alignment(&self) -> Alignment { + self.alignment + } + /// Returns the segment ID. pub fn segment_id(&self) -> SegmentId { self.segment_id @@ -92,6 +138,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + alignment: self.alignment, } } @@ -142,6 +189,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + alignment: self.alignment, } } @@ -166,10 +214,169 @@ impl Debug for LazyBufferHandle { f.debug_struct("LazyBufferHandle") .field("segment_id", &self.segment_id) .field("selection", &self.selection) + .field("alignment", &self.alignment) .finish() } } +impl PartialEq for LazyBufferHandle { + fn eq(&self, other: &Self) -> bool { + self.segment_id == other.segment_id + && self.selection == other.selection + && self.alignment == other.alignment + } +} + +impl Eq for LazyBufferHandle {} + +impl Hash for LazyBufferHandle { + fn hash(&self, state: &mut H) { + self.segment_id.hash(state); + self.selection.hash(state); + self.alignment.hash(state); + } +} + +impl DeviceBuffer for LazyBufferHandle { + fn as_any(&self) -> &dyn Any { + self + } + + fn len(&self) -> usize { + self.len() + } + + fn alignment(&self) -> Alignment { + self.alignment + } + + fn copy_to_host_sync(&self, alignment: Alignment) -> VortexResult { + futures::executor::block_on(async { + let handle = self.materialize().await?; + Ok(handle.try_into_host_sync()?.aligned(alignment)) + }) + } + + fn copy_to_host( + &self, + alignment: Alignment, + ) -> VortexResult>> { + let this = self.clone(); + Ok(async move { + let handle = this.materialize().await?; + Ok(handle.try_into_host_sync()?.aligned(alignment)) + } + .boxed()) + } + + fn slice(&self, range: Range) -> Arc { + Arc::new(LazyBufferHandle::slice(self, range)) + } + + fn filter(&self, ranges: &[Range]) -> VortexResult> { + Ok(Arc::new(LazyBufferHandle::filter(self, ranges))) + } + + fn aligned(self: Arc, alignment: Alignment) -> VortexResult> { + if self.alignment.is_aligned_to(alignment) { + Ok(self) + } else { + Ok(Arc::new(LazyBufferHandle { + source: Arc::clone(&self.source), + segment_id: self.segment_id, + selection: self.selection.clone(), + alignment, + })) + } + } +} + +/// Build an [`ArrayParts`] with lazy device buffers that defer segment I/O. +/// +/// Each buffer descriptor in the flatbuffer is turned into a [`LazyBufferHandle`] +/// that records the segment source, segment ID, byte range, and alignment but +/// does **not** perform any I/O. The returned [`ArrayParts`] can be decoded into +/// an array tree and manipulated (sliced, filtered, optimized) before the lazy +/// buffers are materialized with [`materialize_recursive`]. +pub fn create_lazy_array_parts( + array_tree: ByteBuffer, + source: Arc, + segment_id: SegmentId, +) -> VortexResult { + use flatbuffers::root; + use vortex_flatbuffers::FlatBuffer; + use vortex_flatbuffers::array as fba; + + let fb_aligned = FlatBuffer::align_from(array_tree.clone()); + let fb_array = root::(fb_aligned.as_ref())?; + + let mut offset: usize = 0; + let buffers: Vec = fb_array + .buffers() + .unwrap_or_default() + .iter() + .map(|fb_buf| { + offset += fb_buf.padding() as usize; + let buffer_len = fb_buf.length() as usize; + let alignment = Alignment::from_exponent(fb_buf.alignment_exponent()); + + let lazy = LazyBufferHandle::new(Arc::clone(&source), segment_id, alignment) + .slice(offset..offset + buffer_len); + + offset += buffer_len; + BufferHandle::new_device(Arc::new(lazy)) + }) + .collect(); + + ArrayParts::from_flatbuffer_with_buffers(array_tree, buffers) +} + +/// Recursively walk the array tree and materialize any [`LazyBufferHandle`] +/// device buffers by performing I/O, returning a new tree with host-resident +/// buffers. +pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { + // 1. Recursively materialize children. + let children = array.children(); + let mut new_children = Vec::with_capacity(children.len()); + let mut any_child_changed = false; + for child in &children { + let new_child = Box::pin(materialize_recursive(child)).await?; + any_child_changed |= !Arc::ptr_eq(child, &new_child); + new_children.push(new_child); + } + let current = if any_child_changed { + array.with_children(new_children)? + } else { + array.clone() + }; + + // 2. Check for lazy device buffers. + let handles = current.buffer_handles(); + let any_lazy = handles.iter().any(|h| { + h.as_device_opt() + .and_then(|d| d.as_any().downcast_ref::()) + .is_some() + }); + if !any_lazy { + return Ok(current); + } + + // 3. Materialize lazy buffers, ensuring proper alignment. + let mut materialized = Vec::with_capacity(handles.len()); + for handle in &handles { + if let Some(lazy) = handle + .as_device_opt() + .and_then(|d| d.as_any().downcast_ref::()) + { + let buf = lazy.materialize().await?; + materialized.push(buf.ensure_aligned(lazy.alignment())?); + } else { + materialized.push(handle.clone()); + } + } + current.with_buffers(materialized) +} + /// Map a logical byte range into the given set of existing absolute ranges. /// /// The `range` is interpreted as an offset into the concatenated output of @@ -222,6 +429,7 @@ mod tests { use futures::FutureExt; use vortex_array::buffer::BufferHandle; + use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_io::runtime::single::block_on; @@ -243,7 +451,11 @@ mod tests { fn lazy(data: &[u8]) -> LazyBufferHandle { let buf = BufferHandle::new_host(ByteBuffer::copy_from(data)); - LazyBufferHandle::new(Arc::new(SingleSegment(buf)), SegmentId::from(0u32)) + LazyBufferHandle::new( + Arc::new(SingleSegment(buf)), + SegmentId::from(0u32), + Alignment::none(), + ) } #[test] diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 727566d3db8..fd7997b75d0 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -22,6 +22,8 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutReader; +use crate::buffer::create_lazy_array_parts; +use crate::buffer::materialize_recursive; use crate::layouts::SharedArrayFuture; use crate::layouts::flat::FlatLayout; use crate::segments::SegmentSource; @@ -60,30 +62,37 @@ impl FlatReader { let row_count = usize::try_from(self.layout.row_count()).vortex_expect("row count must fit in usize"); - // We create the segment_fut here to ensure we give the segment reader visibility into - // how to prioritize this segment, even if the `array` future has already been initialized. - // This is gross... see the function's TODO for a maybe better solution? - let segment_fut = self.segment_source.request(self.layout.segment_id()); - let ctx = self.layout.array_ctx().clone(); let session = self.session.clone(); let dtype = self.layout.dtype().clone(); let array_tree = self.layout.array_tree().cloned(); - async move { - let segment = segment_fut.await?; - let parts = if let Some(array_tree) = array_tree { - // Use the pre-stored flatbuffer from layout metadata combined with segment buffers. - ArrayParts::from_flatbuffer_and_segment(array_tree, segment)? - } else { - // Parse the flatbuffer from the segment itself. - ArrayParts::try_from(segment)? - }; - parts - .decode(&dtype, row_count, &ctx, &session) - .map_err(Arc::new) + + if let Some(array_tree) = array_tree { + // Build lazy — no segment I/O yet. Buffers are LazyBufferHandles wrapped as + // device buffers that will be materialized after slice/filter/optimize. + let source = Arc::clone(&self.segment_source); + let segment_id = self.layout.segment_id(); + async move { + let parts = create_lazy_array_parts(array_tree, source, segment_id)?; + parts + .decode(&dtype, row_count, &ctx, &session) + .map_err(Arc::new) + } + .boxed() + .shared() + } else { + // Legacy path: segment contains both flatbuffer and data buffers. + let segment_fut = self.segment_source.request(self.layout.segment_id()); + async move { + let segment = segment_fut.await?; + let parts = ArrayParts::try_from(segment)?; + parts + .decode(&dtype, row_count, &ctx, &session) + .map_err(Arc::new) + } + .boxed() + .shared() } - .boxed() - .shared() } } @@ -152,6 +161,7 @@ impl LayoutReader for FlatReader { // after this. let array = array.apply(&expr)?; let array = array.filter(mask.clone())?; + let array = materialize_recursive(&array).await?; let mut ctx = session.create_execution_ctx(); let array_mask = array.execute::(&mut ctx)?; @@ -159,6 +169,7 @@ impl LayoutReader for FlatReader { } else { // Run over the full array, with a simpler bitand at the end. let array = array.apply(&expr)?; + let array = materialize_recursive(&array).await?; let mut ctx = session.create_execution_ctx(); let array_mask = array.execute::(&mut ctx)?; @@ -213,6 +224,9 @@ impl LayoutReader for FlatReader { // Evaluate the projection expression. array = array.apply(&expr)?; + // Materialize any remaining lazy device buffers before returning. + array = materialize_recursive(&array).await?; + Ok(array) } .boxed()) From a1a2d19ab1a7242aa9ce1db2c04e0a89a5b8e698 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 19 Mar 2026 14:09:30 -0700 Subject: [PATCH 04/10] Add slicing to BufferHandle Signed-off-by: Nicholas Gates --- Cargo.lock | 1 + encodings/bytebool/Cargo.toml | 1 + encodings/bytebool/public-api.lock | 4 ++ encodings/bytebool/src/rules.rs | 2 + encodings/bytebool/src/slice.rs | 20 +++++++ vortex-array/public-api.lock | 36 ++++++++++++ .../src/arrays/decimal/compute/rules.rs | 29 ++++++++++ .../src/arrays/primitive/compute/rules.rs | 2 + .../src/arrays/primitive/compute/slice.rs | 22 ++++++++ .../src/arrays/varbinview/compute/rules.rs | 2 + .../src/arrays/varbinview/compute/slice.rs | 22 ++++++++ vortex-layout/src/layouts/flat/writer.rs | 55 ++++++++++--------- 12 files changed, 170 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a09716070c..72b187dd9e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9914,6 +9914,7 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-mask", "vortex-session", ] diff --git a/encodings/bytebool/Cargo.toml b/encodings/bytebool/Cargo.toml index 53197452ff5..bca2ed71aa9 100644 --- a/encodings/bytebool/Cargo.toml +++ b/encodings/bytebool/Cargo.toml @@ -21,6 +21,7 @@ num-traits = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-mask = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 192d025cf34..a7af1f45d43 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -14,6 +14,10 @@ impl vortex_array::arrays::dict::take::TakeExecute for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::take(array: &vortex_bytebool::ByteBoolArray, indices: &vortex_array::array::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::kernel::FilterReduce for vortex_bytebool::ByteBool + +pub fn vortex_bytebool::ByteBool::filter(array: &vortex_bytebool::ByteBoolArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::slice(array: &vortex_bytebool::ByteBoolArray, range: core::ops::range::Range) -> vortex_error::VortexResult> diff --git a/encodings/bytebool/src/rules.rs b/encodings/bytebool/src/rules.rs index f67d3567326..b4dc61fa1c7 100644 --- a/encodings/bytebool/src/rules.rs +++ b/encodings/bytebool/src/rules.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; @@ -12,4 +13,5 @@ pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(ByteBool)), ParentRuleSet::lift(&MaskReduceAdaptor(ByteBool)), ParentRuleSet::lift(&SliceReduceAdaptor(ByteBool)), + ParentRuleSet::lift(&FilterReduceAdaptor(ByteBool)), ]); diff --git a/encodings/bytebool/src/slice.rs b/encodings/bytebool/src/slice.rs index c80be024fd1..8cf24bda5e4 100644 --- a/encodings/bytebool/src/slice.rs +++ b/encodings/bytebool/src/slice.rs @@ -5,9 +5,11 @@ use std::ops::Range; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::arrays::filter::FilterReduce; use vortex_array::arrays::slice::SliceReduce; use vortex_array::vtable::ValidityHelper; use vortex_error::VortexResult; +use vortex_mask::Mask; use crate::ByteBool; use crate::ByteBoolArray; @@ -23,3 +25,21 @@ impl SliceReduce for ByteBool { )) } } + +impl FilterReduce for ByteBool { + fn filter(array: &ByteBoolArray, mask: &Mask) -> VortexResult> { + let ranges: Vec> = mask + .slices() + .unwrap_or_else(|| unreachable!(), || unreachable!()) + .iter() + .map(|&(s, e)| s..e) + .collect(); + Ok(Some( + ByteBoolArray::new( + array.buffer().filter_typed::(&ranges)?, + array.validity().filter(mask)?, + ) + .into_array(), + )) + } +} diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 67f1955e17c..c8a15145e59 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -1434,6 +1434,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::take(array: &vortex_array::arrays::DecimalArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::filter(array: &vortex_array::arrays::DecimalArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -2386,6 +2390,10 @@ impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Consta pub fn vortex_array::arrays::Constant::filter(array: &vortex_array::arrays::ConstantArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::filter(array: &vortex_array::arrays::DecimalArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Extension pub fn vortex_array::arrays::Extension::filter(array: &vortex_array::arrays::ExtensionArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> @@ -2394,6 +2402,14 @@ impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::filter(array: &vortex_array::arrays::MaskedArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::filter(array: &vortex_array::arrays::PrimitiveArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::filter(array: &vortex_array::arrays::VarBinViewArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::dict::Dict pub fn vortex_array::arrays::dict::Dict::filter(array: &vortex_array::arrays::dict::DictArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> @@ -3256,6 +3272,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::take(array: &vortex_array::arrays::PrimitiveArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::filter(array: &vortex_array::arrays::PrimitiveArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -4662,6 +4682,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::VarBinVie pub fn vortex_array::arrays::VarBinView::take(array: &vortex_array::arrays::VarBinViewArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::filter(array: &vortex_array::arrays::VarBinViewArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -5478,6 +5502,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::take(array: &vortex_array::arrays::DecimalArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::filter(array: &vortex_array::arrays::DecimalArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -6818,6 +6846,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::take(array: &vortex_array::arrays::PrimitiveArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::filter(array: &vortex_array::arrays::PrimitiveArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -7854,6 +7886,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::VarBinVie pub fn vortex_array::arrays::VarBinView::take(array: &vortex_array::arrays::VarBinViewArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::filter(array: &vortex_array::arrays::VarBinViewArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs index a7d7dab51e0..f84df1dac95 100644 --- a/vortex-array/src/arrays/decimal/compute/rules.rs +++ b/vortex-array/src/arrays/decimal/compute/rules.rs @@ -4,6 +4,7 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; @@ -11,6 +12,8 @@ use crate::arrays::Decimal; use crate::arrays::DecimalArray; use crate::arrays::Masked; use crate::arrays::MaskedArray; +use crate::arrays::filter::FilterReduce; +use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduce; use crate::arrays::slice::SliceReduceAdaptor; use crate::match_each_decimal_value_type; @@ -23,6 +26,7 @@ pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DecimalMaskedValidityRule), ParentRuleSet::lift(&MaskReduceAdaptor(Decimal)), ParentRuleSet::lift(&SliceReduceAdaptor(Decimal)), + ParentRuleSet::lift(&FilterReduceAdaptor(Decimal)), ]); /// Rule to push down validity masking from MaskedArray parent into DecimalArray child. @@ -72,3 +76,28 @@ impl SliceReduce for Decimal { Ok(Some(result)) } } + +impl FilterReduce for Decimal { + fn filter(array: &DecimalArray, mask: &Mask) -> VortexResult> { + let ranges: Vec> = mask + .slices() + .unwrap_or_else(|| unreachable!(), || unreachable!()) + .iter() + .map(|&(s, e)| s..e) + .collect(); + let result = match_each_decimal_value_type!(array.values_type(), |D| { + // SAFETY: Filtering preserves all DecimalArray invariants — values within + // precision bounds remain valid, and we correctly filter the validity. + unsafe { + DecimalArray::new_unchecked_handle( + array.buffer_handle().filter_typed::(&ranges)?, + array.values_type(), + array.decimal_dtype(), + array.validity().filter(mask)?, + ) + } + .into_array() + }); + Ok(Some(result)) + } +} diff --git a/vortex-array/src/arrays/primitive/compute/rules.rs b/vortex-array/src/arrays/primitive/compute/rules.rs index df6eb35d888..f3a423cea20 100644 --- a/vortex-array/src/arrays/primitive/compute/rules.rs +++ b/vortex-array/src/arrays/primitive/compute/rules.rs @@ -9,6 +9,7 @@ use crate::arrays::Masked; use crate::arrays::MaskedArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -19,6 +20,7 @@ pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&PrimitiveMaskedValidityRule), ParentRuleSet::lift(&MaskReduceAdaptor(Primitive)), ParentRuleSet::lift(&SliceReduceAdaptor(Primitive)), + ParentRuleSet::lift(&FilterReduceAdaptor(Primitive)), ]); /// Rule to push down validity masking from MaskedArray parent into PrimitiveArray child. diff --git a/vortex-array/src/arrays/primitive/compute/slice.rs b/vortex-array/src/arrays/primitive/compute/slice.rs index 2844163e557..38115d9f527 100644 --- a/vortex-array/src/arrays/primitive/compute/slice.rs +++ b/vortex-array/src/arrays/primitive/compute/slice.rs @@ -4,11 +4,13 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::filter::FilterReduce; use crate::arrays::slice::SliceReduce; use crate::dtype::NativePType; use crate::match_each_native_ptype; @@ -27,3 +29,23 @@ impl SliceReduce for Primitive { Ok(Some(result)) } } + +impl FilterReduce for Primitive { + fn filter(array: &PrimitiveArray, mask: &Mask) -> VortexResult> { + let ranges: Vec> = mask + .slices() + .unwrap_or_else(|| unreachable!(), || unreachable!()) + .iter() + .map(|&(s, e)| s..e) + .collect(); + let result = match_each_native_ptype!(array.ptype(), |T| { + PrimitiveArray::from_buffer_handle( + array.buffer_handle().filter_typed::(&ranges)?, + T::PTYPE, + array.validity().filter(mask)?, + ) + .into_array() + }); + Ok(Some(result)) + } +} diff --git a/vortex-array/src/arrays/varbinview/compute/rules.rs b/vortex-array/src/arrays/varbinview/compute/rules.rs index 5ec24dca7de..3a7b98cd5c5 100644 --- a/vortex-array/src/arrays/varbinview/compute/rules.rs +++ b/vortex-array/src/arrays/varbinview/compute/rules.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors use crate::arrays::VarBinView; +use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::fns::cast::CastReduceAdaptor; @@ -10,4 +11,5 @@ pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(VarBinView)), ParentRuleSet::lift(&MaskReduceAdaptor(VarBinView)), ParentRuleSet::lift(&SliceReduceAdaptor(VarBinView)), + ParentRuleSet::lift(&FilterReduceAdaptor(VarBinView)), ]); diff --git a/vortex-array/src/arrays/varbinview/compute/slice.rs b/vortex-array/src/arrays/varbinview/compute/slice.rs index 02582841601..47082ea1add 100644 --- a/vortex-array/src/arrays/varbinview/compute/slice.rs +++ b/vortex-array/src/arrays/varbinview/compute/slice.rs @@ -5,11 +5,13 @@ use std::ops::Range; use std::sync::Arc; use vortex_error::VortexResult; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; +use crate::arrays::filter::FilterReduce; use crate::arrays::slice::SliceReduce; use crate::arrays::varbinview::BinaryView; @@ -28,3 +30,23 @@ impl SliceReduce for VarBinView { )) } } + +impl FilterReduce for VarBinView { + fn filter(array: &VarBinViewArray, mask: &Mask) -> VortexResult> { + let ranges: Vec> = mask + .slices() + .unwrap_or_else(|| unreachable!(), || unreachable!()) + .iter() + .map(|&(s, e)| s..e) + .collect(); + Ok(Some( + VarBinViewArray::new_handle( + array.views_handle().filter_typed::(&ranges)?, + Arc::clone(array.buffers()), + array.dtype().clone(), + array.validity()?.filter(mask)?, + ) + .into_array(), + )) + } +} diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 0e83f717f49..2b3f06d7944 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -207,6 +207,7 @@ mod tests { use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; + use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; @@ -413,40 +414,42 @@ mod tests { } #[test] - fn flat_invalid_array_fails() -> VortexResult<()> { + fn flat_filter_array_reduces_to_primitive() -> VortexResult<()> { block_on(|handle| async { let prim: PrimitiveArray = (0..10).collect(); let filter = prim.filter(Mask::from_indices(10, vec![2, 3]))?; let ctx = ArrayContext::empty(); - // Write the array into a byte buffer. - let (layout, _segments) = { - let segments = Arc::new(TestSegments::default()); - let (ptr, eof) = SequenceId::root().split(); - // Only allow primitive encodings - filter arrays should fail. - let allowed = ArrayRegistry::default(); - allowed.register(Primitive::ID, Primitive); - let layout = FlatLayoutStrategy::default() - .with_allow_encodings(allowed) - .write_stream( - ctx, - segments.clone(), - filter.to_array_stream().sequenced(ptr), - eof, - handle, - ) - .await; + // FilterReduce reduces FilterArray(PrimitiveArray) → PrimitiveArray during + // optimization, so the write should succeed even with only Primitive allowed. + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let allowed = ArrayRegistry::default(); + allowed.register(Primitive::ID, Primitive); + let layout = FlatLayoutStrategy::default() + .with_allow_encodings(allowed) + .write_stream( + ctx, + segments.clone(), + filter.to_array_stream().sequenced(ptr), + eof, + handle, + ) + .await?; - (layout, segments) - }; + let result = layout + .new_reader("".into(), segments, &SESSION)? + .projection_evaluation( + &(0..layout.row_count()), + &root(), + MaskFuture::new_true(layout.row_count().try_into()?), + )? + .await?; - let err = layout.expect_err("expected error"); - assert!( - err.to_string() - .contains("normalize forbids encoding (vortex.filter)"), - "unexpected error: {err}" - ); + let expected = + PrimitiveArray::new(buffer![2i32, 3], Validity::NonNullable).into_array(); + assert_arrays_eq!(result, expected); Ok(()) }) From a804764c3d2ac4d6898116fe16b42df50c852ede Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 19 Mar 2026 14:14:30 -0700 Subject: [PATCH 05/10] Add slicing to BufferHandle Signed-off-by: Nicholas Gates --- vortex-array/public-api.lock | 12 ++++++ .../arrays/fixed_size_list/compute/rules.rs | 2 + .../arrays/fixed_size_list/compute/slice.rs | 38 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index c8a15145e59..92520709be3 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2398,6 +2398,10 @@ impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Extens pub fn vortex_array::arrays::Extension::filter(array: &vortex_array::arrays::ExtensionArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::filter(array: &vortex_array::arrays::FixedSizeListArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::filter(array: &vortex_array::arrays::MaskedArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> @@ -2434,6 +2438,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::FixedSize pub fn vortex_array::arrays::FixedSizeList::take(array: &vortex_array::arrays::FixedSizeListArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::filter(array: &vortex_array::arrays::FixedSizeListArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> @@ -6124,6 +6132,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::FixedSize pub fn vortex_array::arrays::FixedSizeList::take(array: &vortex_array::arrays::FixedSizeListArray, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::filter::FilterReduce for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::filter(array: &vortex_array::arrays::FixedSizeListArray, mask: &vortex_mask::Mask) -> vortex_error::VortexResult> + impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::slice(array: &Self::Array, range: core::ops::range::Range) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/fixed_size_list/compute/rules.rs b/vortex-array/src/arrays/fixed_size_list/compute/rules.rs index da1d91423e2..149b4bfe6f3 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/rules.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/rules.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use crate::arrays::FixedSizeList; +use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::fns::cast::CastReduceAdaptor; @@ -11,4 +12,5 @@ pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new ParentRuleSet::lift(&CastReduceAdaptor(FixedSizeList)), ParentRuleSet::lift(&MaskReduceAdaptor(FixedSizeList)), ParentRuleSet::lift(&SliceReduceAdaptor(FixedSizeList)), + ParentRuleSet::lift(&FilterReduceAdaptor(FixedSizeList)), ]); diff --git a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs index 9db5d8a7c08..1b06fb7651f 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs @@ -4,11 +4,13 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; +use crate::arrays::filter::FilterReduce; use crate::arrays::slice::SliceReduce; use crate::vtable::ValidityHelper; @@ -33,3 +35,39 @@ impl SliceReduce for FixedSizeList { )) } } + +impl FilterReduce for FixedSizeList { + fn filter(array: &FixedSizeListArray, mask: &Mask) -> VortexResult> { + let list_size = array.list_size() as usize; + let new_len = mask.true_count(); + + let filtered_elements = if list_size == 0 { + // Degenerate case: elements array is empty regardless of filter. + array.elements().clone() + } else { + let elements_len = array.elements().len(); + let expanded_slices: Vec<(usize, usize)> = mask + .slices() + .unwrap_or_else(|| unreachable!(), || unreachable!()) + .iter() + .map(|&(s, e)| (s * list_size, e * list_size)) + .collect(); + let elements_mask = Mask::from_slices(elements_len, expanded_slices); + array.elements().filter(elements_mask)? + }; + + // SAFETY: Filtering preserves FixedSizeListArray invariants — each selected list's + // elements are contiguously preserved, maintaining elements.len() == new_len * list_size. + Ok(Some( + unsafe { + FixedSizeListArray::new_unchecked( + filtered_elements, + array.list_size(), + array.validity().filter(mask)?, + new_len, + ) + } + .into_array(), + )) + } +} From a454bfbca62e94abbba7adc7427870f160667b3e Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 15:18:01 -0400 Subject: [PATCH 06/10] Lazy Buffers Signed-off-by: Nicholas Gates --- vortex-file/public-api.lock | 10 + vortex-file/src/open.rs | 6 + vortex-file/src/segments/source.rs | 269 ++++++++++++++++++++++++--- vortex-layout/public-api.lock | 22 ++- vortex-layout/src/buffer.rs | 181 ++++++++++++++---- vortex-layout/src/segments/cache.rs | 94 +++++++++- vortex-layout/src/segments/shared.rs | 80 +++++++- vortex-layout/src/segments/source.rs | 37 ++++ vortex-layout/src/segments/test.rs | 4 + 9 files changed, 638 insertions(+), 65 deletions(-) diff --git a/vortex-file/public-api.lock b/vortex-file/public-api.lock index fc73045c63e..5334a6e9193 100644 --- a/vortex-file/public-api.lock +++ b/vortex-file/public-api.lock @@ -40,6 +40,10 @@ impl vortex_layout::segments::source::SegmentSource for vortex_file::segments::F pub fn vortex_file::segments::FileSegmentSource::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::source::SegmentFuture +pub fn vortex_file::segments::FileSegmentSource::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::source::SegmentFuture + +pub fn vortex_file::segments::FileSegmentSource::segment_len(&self, id: vortex_layout::segments::SegmentId) -> core::option::Option + pub struct vortex_file::segments::InitialReadSegmentCache pub vortex_file::segments::InitialReadSegmentCache::fallback: alloc::sync::Arc @@ -58,12 +62,18 @@ pub vortex_file::segments::RequestMetrics::coalesced_requests: vortex_metrics::c pub vortex_file::segments::RequestMetrics::individual_requests: vortex_metrics::counter::Counter +pub vortex_file::segments::RequestMetrics::logical_requested_bytes: vortex_metrics::counter::Counter + pub vortex_file::segments::RequestMetrics::num_requests_coalesced: vortex_metrics::histogram::Histogram impl vortex_file::segments::RequestMetrics pub fn vortex_file::segments::RequestMetrics::new(metrics_registry: &dyn vortex_metrics::MetricsRegistry, labels: alloc::vec::Vec) -> Self +impl core::clone::Clone for vortex_file::segments::RequestMetrics + +pub fn vortex_file::segments::RequestMetrics::clone(&self) -> vortex_file::segments::RequestMetrics + pub mod vortex_file::v2 pub struct vortex_file::v2::FileStatsLayoutReader diff --git a/vortex-file/src/open.rs b/vortex-file/src/open.rs index ac8fdd83097..07d1bcc492c 100644 --- a/vortex-file/src/open.rs +++ b/vortex-file/src/open.rs @@ -12,6 +12,7 @@ use vortex_buffer::ByteBuffer; use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_io::InstrumentedReadAt; use vortex_io::VortexReadAt; use vortex_io::session::RuntimeSessionExt; use vortex_layout::segments::InstrumentedSegmentCache; @@ -205,6 +206,11 @@ impl VortexOpenOptions { .metrics_registry .clone() .unwrap_or_else(|| Arc::new(DefaultMetricsRegistry::default())); + let reader = InstrumentedReadAt::new_with_labels( + reader, + metrics_registry.as_ref(), + self.labels.clone(), + ); let footer = if let Some(footer) = self.footer { footer diff --git a/vortex-file/src/segments/source.rs b/vortex-file/src/segments/source.rs index 8f83150c4bb..3ab91799f08 100644 --- a/vortex-file/src/segments/source.rs +++ b/vortex-file/src/segments/source.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicUsize; @@ -12,9 +13,11 @@ use futures::FutureExt; use futures::StreamExt; use futures::channel::mpsc; use futures::future; +use futures::future::try_join_all; use vortex_array::buffer::BufferHandle; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; +use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_error::vortex_panic; @@ -41,6 +44,14 @@ pub enum ReadEvent { Dropped(RequestId), } +fn apply_ranges(buffer: BufferHandle, ranges: &[Range]) -> VortexResult { + match ranges { + [] => buffer.filter(&[]), + [range] => Ok(buffer.slice(range.clone())), + _ => buffer.filter(ranges), + } +} + /// A [`SegmentSource`] for file-like IO. /// ## Coalescing and Pre-fetching /// @@ -68,6 +79,7 @@ pub struct FileSegmentSource { events: mpsc::UnboundedSender, /// The next read request ID. next_id: Arc, + metrics: RequestMetrics, } impl FileSegmentSource { @@ -103,7 +115,7 @@ impl FileSegmentSource { StreamExt::boxed(recv), coalesce_config, max_alignment, - metrics, + metrics.clone(), ) .boxed(); @@ -129,52 +141,137 @@ impl FileSegmentSource { segments, events: send, next_id: Arc::new(AtomicUsize::new(0)), + metrics, } } -} -impl SegmentSource for FileSegmentSource { - fn request(&self, id: SegmentId) -> SegmentFuture { - // We eagerly register the read request here assuming the behaviour of [`FileRead`], where - // coalescing becomes effective prior to the future being polled. - let spec = *match self.segments.get(*id as usize) { - Some(spec) => spec, - None => { - return future::ready(Err(vortex_err!("Missing segment: {}", id))).boxed(); - } - }; - - let SegmentSpec { - offset, - length, - alignment, - } = spec; + fn segment_spec(&self, id: SegmentId) -> VortexResult { + self.segments + .get(*id as usize) + .copied() + .ok_or_else(|| vortex_err!("Missing segment: {}", id)) + } + fn submit_read(&self, offset: u64, length: usize, alignment: Alignment) -> SegmentFuture { let (send, recv) = oneshot::channel(); let id = self.next_id.fetch_add(1, Ordering::Relaxed); let event = ReadEvent::Request(ReadRequest { id, offset, - length: length as usize, + length, alignment, callback: send, }); - // If we fail to submit the event, we create a future that has failed. if let Err(e) = self.events.unbounded_send(event) { return future::ready(Err(vortex_err!("Failed to submit read request: {e}"))).boxed(); } - let fut = ReadFuture { + ReadFuture { id, recv: recv.into_future(), polled: false, finished: false, events: self.events.clone(), + } + .boxed() + } +} + +impl SegmentSource for FileSegmentSource { + fn segment_len(&self, id: SegmentId) -> Option { + self.segments.get(*id as usize).map(|spec| spec.length as usize) + } + + fn request(&self, id: SegmentId) -> SegmentFuture { + // We eagerly register the read request here assuming the behaviour of [`FileRead`], where + // coalescing becomes effective prior to the future being polled. + let spec = match self.segment_spec(id) { + Ok(spec) => spec, + Err(err) => return future::ready(Err(err)).boxed(), }; - // One allocation: we only box the returned SegmentFuture, not the inner ReadFuture. - fut.boxed() + let requested_bytes = self.metrics.logical_requested_bytes.clone(); + let future = self.submit_read(spec.offset, spec.length as usize, spec.alignment); + async move { + let buffer = future.await?; + requested_bytes.add(u64::from(spec.length)); + Ok(buffer) + } + .boxed() + } + + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + let spec = match self.segment_spec(id) { + Ok(spec) => spec, + Err(err) => return future::ready(Err(err)).boxed(), + }; + + let segment_len = spec.length as usize; + for range in &ranges { + if range.start > range.end || range.end > segment_len { + return future::ready(Err(vortex_err!( + "Segment {} range {}..{} out of bounds for segment length {}", + id, + range.start, + range.end, + segment_len + ))) + .boxed(); + } + } + + let total_len: usize = ranges.iter().map(Range::len).sum(); + let requested_bytes = self.metrics.logical_requested_bytes.clone(); + + match ranges.as_slice() { + [] => { + requested_bytes.add(0); + future::ready(Ok(BufferHandle::new_host(ByteBuffer::empty()))).boxed() + } + [range] => { + let future = self.submit_read( + spec.offset + range.start as u64, + range.len(), + Alignment::none(), + ); + async move { + let buffer = future.await?; + requested_bytes.add(total_len as u64); + Ok(buffer) + } + .boxed() + } + _ => { + let read_futures = ranges + .into_iter() + .map(|range| { + self.submit_read( + spec.offset + range.start as u64, + range.len(), + Alignment::none(), + ) + }) + .collect::>(); + async move { + let chunks = try_join_all(read_futures.into_iter().map(|future| async move { + let handle = future.await?; + handle.try_into_host()?.await + })) + .await?; + // Follow-up: teach the lower I/O API to support scatter/gather into a + // caller-owned destination buffer so this gather copy can be removed. + let mut gathered = + ByteBufferMut::with_capacity_aligned(total_len, Alignment::none()); + for chunk in chunks { + gathered.extend_from_slice(chunk.as_ref()); + } + requested_bytes.add(total_len as u64); + Ok(BufferHandle::new_host(gathered.freeze())) + } + .boxed() + } + } } } @@ -231,10 +328,12 @@ impl Drop for ReadFuture { } } +#[derive(Clone)] pub struct RequestMetrics { pub individual_requests: Counter, pub coalesced_requests: Counter, pub num_requests_coalesced: Histogram, + pub logical_requested_bytes: Counter, } impl RequestMetrics { @@ -247,8 +346,11 @@ impl RequestMetrics { .add_labels(labels.clone()) .counter("io.requests.coalesced"), num_requests_coalesced: MetricBuilder::new(metrics_registry) - .add_labels(labels) + .add_labels(labels.clone()) .histogram("io.requests.coalesced.num_coalesced"), + logical_requested_bytes: MetricBuilder::new(metrics_registry) + .add_labels(labels) + .counter("vortex.file.segments.requested_bytes"), } } } @@ -270,6 +372,10 @@ impl BufferSegmentSource { } impl SegmentSource for BufferSegmentSource { + fn segment_len(&self, id: SegmentId) -> Option { + self.segments.get(*id as usize).map(|spec| spec.length as usize) + } + fn request(&self, id: SegmentId) -> SegmentFuture { let spec = match self.segments.get(*id as usize) { Some(spec) => spec, @@ -294,4 +400,119 @@ impl SegmentSource for BufferSegmentSource { let slice = self.buffer.slice(start..end).aligned(spec.alignment); future::ready(Ok(BufferHandle::new_host(slice))).boxed() } + + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + let spec = match self.segments.get(*id as usize) { + Some(spec) => spec, + None => { + return future::ready(Err(vortex_err!("Missing segment: {}", id))).boxed(); + } + }; + + let start = spec.offset as usize; + let end = start + spec.length as usize; + if end > self.buffer.len() { + return future::ready(Err(vortex_err!( + "Segment {} range {}..{} out of bounds for buffer of length {}", + *id, + start, + end, + self.buffer.len() + ))) + .boxed(); + } + + let segment = BufferHandle::new_host(self.buffer.slice(start..end).aligned(spec.alignment)); + future::ready(apply_ranges(segment, &ranges)).boxed() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use futures::future::BoxFuture; + use vortex_buffer::ByteBuffer; + use vortex_io::InstrumentedReadAt; + use vortex_io::VortexReadAt; + use vortex_metrics::DefaultMetricsRegistry; + use vortex_metrics::MetricValue; + + use super::*; + + #[derive(Clone)] + struct YieldingReadAt(ByteBuffer); + + impl VortexReadAt for YieldingReadAt { + fn coalesce_config(&self) -> Option { + self.0.coalesce_config() + } + + fn concurrency(&self) -> usize { + self.0.concurrency() + } + + fn size(&self) -> BoxFuture<'static, VortexResult> { + self.0.size() + } + + fn read_at( + &self, + offset: u64, + length: usize, + alignment: Alignment, + ) -> BoxFuture<'static, VortexResult> { + let inner = self.0.clone(); + async move { + tokio::task::yield_now().await; + inner.read_at(offset, length, alignment).await + } + .boxed() + } + } + + #[tokio::test] + async fn request_ranges_packs_bytes_and_exposes_metrics() { + let metrics_registry = DefaultMetricsRegistry::default(); + let reader = InstrumentedReadAt::new( + YieldingReadAt(ByteBuffer::from((0u8..64).collect::>())), + &metrics_registry, + ); + let metrics = RequestMetrics::new(&metrics_registry, vec![]); + let source = FileSegmentSource::open( + Arc::from([SegmentSpec { + offset: 10, + length: 20, + alignment: Alignment::none(), + }]), + reader, + Handle::find().expect("tokio runtime should provide a Vortex handle"), + metrics, + ); + + let result = source + .request_ranges(SegmentId::from(0), vec![1..4, 8..10]) + .await + .unwrap() + .unwrap_host(); + assert_eq!(result.as_slice(), &[11, 12, 13, 18, 19]); + + let snapshot = metrics_registry.snapshot(); + let mut logical_bytes = 0_u64; + let mut physical_bytes = 0_u64; + for metric in snapshot.iter() { + match metric.value() { + MetricValue::Counter(counter) => match metric.name().as_ref() { + "vortex.file.segments.requested_bytes" => logical_bytes = counter.value(), + "vortex.io.read.total_size" => physical_bytes = counter.value(), + _ => {} + }, + MetricValue::Histogram(_) => {} + _ => {} + } + } + + assert_eq!(logical_bytes, 5); + assert!(physical_bytes >= logical_bytes); + } } diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index e0bf15ea6e3..af209bd5ef6 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -24,7 +24,7 @@ pub fn vortex_layout::buffer::LazyBufferHandle::len(&self) -> usize pub async fn vortex_layout::buffer::LazyBufferHandle::materialize(&self) -> vortex_error::VortexResult -pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId, alignment: vortex_buffer::alignment::Alignment) -> Self +pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId, segment_len: usize, alignment: vortex_buffer::alignment::Alignment) -> Self pub fn vortex_layout::buffer::LazyBufferHandle::segment_id(&self) -> vortex_layout::segments::SegmentId @@ -1076,6 +1076,10 @@ impl vortex_layout::segments::SegmentSource for vortex_layout::segments::Segment pub fn vortex_layout::segments::SegmentCacheSourceAdapter::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::SegmentFuture +pub fn vortex_layout::segments::SegmentCacheSourceAdapter::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::SegmentFuture + +pub fn vortex_layout::segments::SegmentCacheSourceAdapter::segment_len(&self, id: vortex_layout::segments::SegmentId) -> core::option::Option + pub struct vortex_layout::segments::SegmentId(_) impl core::clone::Clone for vortex_layout::segments::SegmentId @@ -1142,6 +1146,10 @@ impl vortex_layout::segments::Segment pub fn vortex_layout::segments::SharedSegmentSource::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::SegmentFuture +pub fn vortex_layout::segments::SharedSegmentSource::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::SegmentFuture + +pub fn vortex_layout::segments::SharedSegmentSource::segment_len(&self, id: vortex_layout::segments::SegmentId) -> core::option::Option + pub trait vortex_layout::segments::SegmentCache: core::marker::Send + core::marker::Sync pub fn vortex_layout::segments::SegmentCache::get<'life0, 'async_trait>(&'life0 self, id: vortex_layout::segments::SegmentId) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait @@ -1174,14 +1182,26 @@ pub trait vortex_layout::segments::SegmentSource: 'static + core::marker::Send + pub fn vortex_layout::segments::SegmentSource::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::SegmentFuture +pub fn vortex_layout::segments::SegmentSource::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::SegmentFuture + +pub fn vortex_layout::segments::SegmentSource::segment_len(&self, _id: vortex_layout::segments::SegmentId) -> core::option::Option + impl vortex_layout::segments::SegmentSource for vortex_layout::segments::SegmentCacheSourceAdapter pub fn vortex_layout::segments::SegmentCacheSourceAdapter::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::SegmentFuture +pub fn vortex_layout::segments::SegmentCacheSourceAdapter::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::SegmentFuture + +pub fn vortex_layout::segments::SegmentCacheSourceAdapter::segment_len(&self, id: vortex_layout::segments::SegmentId) -> core::option::Option + impl vortex_layout::segments::SegmentSource for vortex_layout::segments::SharedSegmentSource pub fn vortex_layout::segments::SharedSegmentSource::request(&self, id: vortex_layout::segments::SegmentId) -> vortex_layout::segments::SegmentFuture +pub fn vortex_layout::segments::SharedSegmentSource::request_ranges(&self, id: vortex_layout::segments::SegmentId, ranges: alloc::vec::Vec>) -> vortex_layout::segments::SegmentFuture + +pub fn vortex_layout::segments::SharedSegmentSource::segment_len(&self, id: vortex_layout::segments::SegmentId) -> core::option::Option + pub type vortex_layout::segments::SegmentFuture = futures_core::future::BoxFuture<'static, vortex_error::VortexResult> pub type vortex_layout::segments::SegmentSinkRef = alloc::sync::Arc diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs index eab11c644df..14b95f7de2f 100644 --- a/vortex-layout/src/buffer.rs +++ b/vortex-layout/src/buffer.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use futures::FutureExt; use futures::future::BoxFuture; +use futures::future::try_join_all; use vortex_array::ArrayRef; use vortex_array::ArrayVisitor; use vortex_array::DynArray; @@ -19,8 +20,8 @@ use vortex_array::buffer::DeviceBuffer; use vortex_array::serde::ArrayParts; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; +use vortex_error::vortex_err; use vortex_error::VortexResult; -use vortex_error::vortex_panic; use crate::segments::SegmentId; use crate::segments::SegmentSource; @@ -36,6 +37,7 @@ pub struct LazyBufferHandle { source: Arc, segment_id: SegmentId, selection: Selection, + len: usize, alignment: Alignment, } @@ -53,33 +55,26 @@ enum Selection { #[allow(clippy::same_name_method)] impl LazyBufferHandle { /// Create a new lazy handle selecting the entire segment. + /// + /// `segment_len` is the full logical length of the segment in bytes. pub fn new( source: Arc, segment_id: SegmentId, + segment_len: usize, alignment: Alignment, ) -> Self { Self { source, segment_id, selection: Selection::All, + len: segment_len, alignment, } } /// Returns the length of the selected byte range(s). - /// - /// # Panics - /// - /// Panics if the entire segment is selected ([`Selection::All`]) since the - /// length is not known without performing I/O. pub fn len(&self) -> usize { - match &self.selection { - Selection::All => { - vortex_panic!("len() is not available for Selection::All; slice first") - } - Selection::Range(r) => r.len(), - Selection::Ranges(rs) => rs.iter().map(|r| r.len()).sum(), - } + self.len } /// Returns whether the buffer is empty. @@ -118,6 +113,8 @@ impl LazyBufferHandle { /// Panics if the range exceeds the bounds of the current selection (when /// those bounds are known). pub fn slice(&self, range: Range) -> Self { + validate_slice_range(&range, self.len); + let new_len = range.len(); let selection = match &self.selection { Selection::All => Selection::Range(range), Selection::Range(base) => { @@ -138,6 +135,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + len: new_len, alignment: self.alignment, } } @@ -152,6 +150,7 @@ impl LazyBufferHandle { /// Panics if any range exceeds the bounds of the current selection (when /// those bounds are known). pub fn filter(&self, ranges: &[Range]) -> Self { + validate_filter_ranges(ranges, self.len); let selection = match &self.selection { Selection::All => Selection::Ranges(Arc::from(ranges)), Selection::Range(base) => { @@ -189,6 +188,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + len: ranges.iter().map(Range::len).sum(), alignment: self.alignment, } } @@ -200,11 +200,18 @@ impl LazyBufferHandle { /// Returns an error if the segment cannot be loaded or the selection cannot be /// applied. pub async fn materialize(&self) -> VortexResult { - let buffer = self.source.request(self.segment_id).await?; match &self.selection { - Selection::All => Ok(buffer), - Selection::Range(range) => Ok(buffer.slice(range.clone())), - Selection::Ranges(ranges) => buffer.filter(ranges), + Selection::All => self.source.request(self.segment_id).await, + Selection::Range(range) => { + self.source + .request_ranges(self.segment_id, vec![range.clone()]) + .await + } + Selection::Ranges(ranges) => { + self.source + .request_ranges(self.segment_id, ranges.iter().cloned().collect()) + .await + } } } } @@ -214,6 +221,7 @@ impl Debug for LazyBufferHandle { f.debug_struct("LazyBufferHandle") .field("segment_id", &self.segment_id) .field("selection", &self.selection) + .field("len", &self.len) .field("alignment", &self.alignment) .finish() } @@ -223,6 +231,7 @@ impl PartialEq for LazyBufferHandle { fn eq(&self, other: &Self) -> bool { self.segment_id == other.segment_id && self.selection == other.selection + && self.len == other.len && self.alignment == other.alignment } } @@ -233,6 +242,7 @@ impl Hash for LazyBufferHandle { fn hash(&self, state: &mut H) { self.segment_id.hash(state); self.selection.hash(state); + self.len.hash(state); self.alignment.hash(state); } } @@ -285,6 +295,7 @@ impl DeviceBuffer for LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection: self.selection.clone(), + len: self.len, alignment, })) } @@ -307,6 +318,9 @@ pub fn create_lazy_array_parts( use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::array as fba; + let segment_len = source + .segment_len(segment_id) + .ok_or_else(|| vortex_err!("Segment {} length is not available", segment_id))?; let fb_aligned = FlatBuffer::align_from(array_tree.clone()); let fb_array = root::(fb_aligned.as_ref())?; @@ -320,7 +334,7 @@ pub fn create_lazy_array_parts( let buffer_len = fb_buf.length() as usize; let alignment = Alignment::from_exponent(fb_buf.alignment_exponent()); - let lazy = LazyBufferHandle::new(Arc::clone(&source), segment_id, alignment) + let lazy = LazyBufferHandle::new(Arc::clone(&source), segment_id, segment_len, alignment) .slice(offset..offset + buffer_len); offset += buffer_len; @@ -337,13 +351,16 @@ pub fn create_lazy_array_parts( pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { // 1. Recursively materialize children. let children = array.children(); - let mut new_children = Vec::with_capacity(children.len()); - let mut any_child_changed = false; - for child in &children { - let new_child = Box::pin(materialize_recursive(child)).await?; - any_child_changed |= !Arc::ptr_eq(child, &new_child); - new_children.push(new_child); - } + let new_children = try_join_all( + children + .iter() + .map(|child| Box::pin(materialize_recursive(child))), + ) + .await?; + let any_child_changed = children + .iter() + .zip(new_children.iter()) + .any(|(child, new_child)| !Arc::ptr_eq(child, new_child)); let current = if any_child_changed { array.with_children(new_children)? } else { @@ -362,18 +379,19 @@ pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { } // 3. Materialize lazy buffers, ensuring proper alignment. - let mut materialized = Vec::with_capacity(handles.len()); - for handle in &handles { + let materialized = try_join_all(handles.iter().cloned().map(|handle| async move { if let Some(lazy) = handle .as_device_opt() .and_then(|d| d.as_any().downcast_ref::()) + .cloned() { let buf = lazy.materialize().await?; - materialized.push(buf.ensure_aligned(lazy.alignment())?); + buf.ensure_aligned(lazy.alignment()) } else { - materialized.push(handle.clone()); + Ok(handle) } - } + })) + .await?; current.with_buffers(materialized) } @@ -422,10 +440,42 @@ fn slice_into_ranges(existing: &[Range], range: Range) -> Selectio } } +fn validate_slice_range(range: &Range, len: usize) { + assert!( + range.start <= range.end && range.end <= len, + "slice range {}..{} exceeds current selection 0..{}", + range.start, + range.end, + len, + ); +} + +fn validate_filter_ranges(ranges: &[Range], len: usize) { + let mut prev_end = 0; + for range in ranges { + assert!( + range.start <= range.end && range.end <= len, + "filter range {}..{} exceeds current selection 0..{}", + range.start, + range.end, + len, + ); + assert!( + range.start >= prev_end, + "filter ranges must be sorted and non-overlapping: {}..{} follows byte {}", + range.start, + range.end, + prev_end, + ); + prev_end = range.end; + } +} + #[cfg(test)] mod tests { use std::ops::Range; use std::sync::Arc; + use std::sync::Mutex; use futures::FutureExt; use vortex_array::buffer::BufferHandle; @@ -438,23 +488,52 @@ mod tests { use crate::segments::SegmentFuture; use crate::segments::SegmentId; use crate::segments::SegmentSource; + use crate::segments::apply_ranges; /// A trivial in-memory segment source for tests. - struct SingleSegment(BufferHandle); + struct SingleSegment { + buffer: BufferHandle, + ranged_requests: Arc>>>>, + } impl SegmentSource for SingleSegment { + fn segment_len(&self, _id: SegmentId) -> Option { + Some(self.buffer.len()) + } + fn request(&self, _id: SegmentId) -> SegmentFuture { - let handle = self.0.clone(); + let handle = self.buffer.clone(); async move { Ok(handle) }.boxed() } + + fn request_ranges(&self, _id: SegmentId, ranges: Vec>) -> SegmentFuture { + self.ranged_requests + .lock() + .expect("range request log poisoned") + .push(ranges.clone()); + let handle = self.buffer.clone(); + async move { apply_ranges(handle, &ranges) }.boxed() + } } fn lazy(data: &[u8]) -> LazyBufferHandle { + lazy_with_requests(data).0 + } + + fn lazy_with_requests(data: &[u8]) -> (LazyBufferHandle, Arc>>>>) { let buf = BufferHandle::new_host(ByteBuffer::copy_from(data)); - LazyBufferHandle::new( - Arc::new(SingleSegment(buf)), - SegmentId::from(0u32), - Alignment::none(), + let ranged_requests = Arc::new(Mutex::new(Vec::new())); + ( + LazyBufferHandle::new( + Arc::new(SingleSegment { + buffer: buf, + ranged_requests: ranged_requests.clone(), + }), + SegmentId::from(0u32), + data.len(), + Alignment::none(), + ), + ranged_requests, ) } @@ -552,6 +631,13 @@ mod tests { assert!(lazy.byte_ranges().is_none()); } + #[test] + fn len_for_all_is_known_without_materialization() { + let lazy = lazy(&[1, 2, 3, 4, 5]); + assert_eq!(lazy.len(), 5); + assert!(!lazy.is_empty()); + } + #[test] fn byte_ranges_after_slice() { let lazy = lazy(&[1, 2, 3, 4, 5]).slice(1..4); @@ -565,4 +651,27 @@ mod tests { let expected = [Range { start: 0, end: 2 }, Range { start: 3, end: 5 }]; assert_eq!(lazy.byte_ranges(), Some(expected.as_slice())); } + + #[test] + fn materialize_uses_request_ranges_for_sliced_buffer() -> VortexResult<()> { + block_on(|_| async { + let (lazy, ranged_requests) = lazy_with_requests(&[1, 2, 3, 4, 5, 6]); + let handle = lazy.slice(1..5).materialize().await?; + assert_eq!(handle.unwrap_host().as_slice(), &[2, 3, 4, 5]); + assert_eq!( + ranged_requests + .lock() + .expect("range request log poisoned") + .as_slice(), + &[vec![1..5]] + ); + Ok(()) + }) + } + + #[test] + #[should_panic(expected = "slice range 0..10 exceeds current selection 0..5")] + fn slice_from_all_checks_bounds() { + drop(lazy(&[1, 2, 3, 4, 5]).slice(0..10)); + } } diff --git a/vortex-layout/src/segments/cache.rs b/vortex-layout/src/segments/cache.rs index 174c6b1e637..078136e116c 100644 --- a/vortex-layout/src/segments/cache.rs +++ b/vortex-layout/src/segments/cache.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; use std::sync::Arc; use async_trait::async_trait; @@ -21,6 +22,7 @@ use vortex_metrics::MetricsRegistry; use crate::segments::SegmentFuture; use crate::segments::SegmentId; use crate::segments::SegmentSource; +use crate::segments::apply_ranges; /// A cache for storing and retrieving individual segment data. #[async_trait] @@ -136,16 +138,20 @@ impl SegmentCacheSourceAdapter { } impl SegmentSource for SegmentCacheSourceAdapter { + fn segment_len(&self, id: SegmentId) -> Option { + self.source.segment_len(id) + } + fn request(&self, id: SegmentId) -> SegmentFuture { let cache = self.cache.clone(); - let delegate = self.source.request(id); + let source = self.source.clone(); async move { if let Ok(Some(segment)) = cache.get(id).await { tracing::debug!("Resolved segment {} from cache", id); return Ok(BufferHandle::new_host(segment)); } - let result = delegate.await?; + let result = source.request(id).await?; // Cache only CPU buffers; device buffers are not cached. if let Some(buffer) = result.as_host_opt() && let Err(e) = cache.put(id, buffer.clone()).await @@ -156,4 +162,88 @@ impl SegmentSource for SegmentCacheSourceAdapter { } .boxed() } + + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + let cache = self.cache.clone(); + let source = self.source.clone(); + + async move { + if let Ok(Some(segment)) = cache.get(id).await { + tracing::debug!("Resolved segment {} from cache for ranged read", id); + return apply_ranges(BufferHandle::new_host(segment), &ranges); + } + source.request_ranges(id, ranges).await + } + .boxed() + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use futures::FutureExt; + + use super::*; + + struct FixedCache(ByteBuffer); + + #[async_trait] + impl SegmentCache for FixedCache { + async fn get(&self, _id: SegmentId) -> VortexResult> { + Ok(Some(self.0.clone())) + } + + async fn put(&self, _id: SegmentId, _buffer: ByteBuffer) -> VortexResult<()> { + Ok(()) + } + } + + #[derive(Default)] + struct CountingSource { + requests: AtomicUsize, + ranged_requests: AtomicUsize, + } + + impl SegmentSource for CountingSource { + fn segment_len(&self, _id: SegmentId) -> Option { + Some(4) + } + + fn request(&self, _id: SegmentId) -> SegmentFuture { + self.requests.fetch_add(1, Ordering::Relaxed); + async { Ok(BufferHandle::new_host(ByteBuffer::from(vec![9, 9, 9, 9]))) }.boxed() + } + + fn request_ranges(&self, _id: SegmentId, ranges: Vec>) -> SegmentFuture { + self.ranged_requests.fetch_add(1, Ordering::Relaxed); + async move { + let full = BufferHandle::new_host(ByteBuffer::from(vec![9, 9, 9, 9])); + apply_ranges(full, &ranges) + } + .boxed() + } + } + + #[tokio::test] + async fn cache_hit_skips_underlying_requests() { + let source = Arc::new(CountingSource::default()); + let adapter = SegmentCacheSourceAdapter::new( + Arc::new(FixedCache(ByteBuffer::from(vec![1, 2, 3, 4]))), + source.clone(), + ); + + let full = adapter.request(SegmentId::from(0)).await.unwrap(); + assert_eq!(full.unwrap_host().as_slice(), &[1, 2, 3, 4]); + + let ranges = adapter + .request_ranges(SegmentId::from(0), vec![1..3]) + .await + .unwrap(); + assert_eq!(ranges.unwrap_host().as_slice(), &[2, 3]); + + assert_eq!(source.requests.load(Ordering::Relaxed), 0); + assert_eq!(source.ranged_requests.load(Ordering::Relaxed), 0); + } } diff --git a/vortex-layout/src/segments/shared.rs b/vortex-layout/src/segments/shared.rs index c794daf608e..bd676c1c009 100644 --- a/vortex-layout/src/segments/shared.rs +++ b/vortex-layout/src/segments/shared.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; use std::sync::Arc; use futures::FutureExt; @@ -22,11 +23,17 @@ use crate::segments::SegmentSource; /// request. pub struct SharedSegmentSource { inner: S, - in_flight: DashMap>, + in_flight: DashMap>, } type SharedSegmentFuture = BoxFuture<'static, SharedVortexResult>; +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +enum RequestKey { + Full(SegmentId), + Ranges(SegmentId, Vec>), +} + impl SharedSegmentSource { /// Create a new `SharedSegmentSource` wrapping the provided inner source. pub fn new(inner: S) -> Self { @@ -38,9 +45,14 @@ impl SharedSegmentSource { } impl SegmentSource for SharedSegmentSource { + fn segment_len(&self, id: SegmentId) -> Option { + self.inner.segment_len(id) + } + fn request(&self, id: SegmentId) -> SegmentFuture { loop { - match self.in_flight.entry(id) { + let key = RequestKey::Full(id); + match self.in_flight.entry(key) { Entry::Occupied(e) => { if let Some(shared_future) = e.get().upgrade() { return shared_future.map_err(VortexError::from).boxed(); @@ -61,6 +73,35 @@ impl SegmentSource for SharedSegmentSource { } } } + + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + loop { + let key = RequestKey::Ranges(id, ranges.clone()); + match self.in_flight.entry(key) { + Entry::Occupied(e) => { + if let Some(shared_future) = e.get().upgrade() { + return shared_future.map_err(VortexError::from).boxed(); + } else { + e.remove(); + } + } + Entry::Vacant(e) => { + let future = self + .inner + .request_ranges(id, ranges.clone()) + .map_err(Arc::new) + .boxed() + .shared(); + e.insert( + future + .downgrade() + .vortex_expect("just created, cannot be polled to completion"), + ); + return future.map_err(VortexError::from).boxed(); + } + } + } + } } #[cfg(test)] @@ -80,13 +121,28 @@ mod tests { struct CountingSegmentSource { segments: TestSegments, request_count: Arc, + range_request_count: Arc, } impl SegmentSource for CountingSegmentSource { + fn segment_len(&self, id: SegmentId) -> Option { + self.segments.segment_len(id) + } + fn request(&self, id: SegmentId) -> SegmentFuture { self.request_count.fetch_add(1, Ordering::SeqCst); self.segments.request(id) } + + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + self.range_request_count.fetch_add(1, Ordering::SeqCst); + let segments = self.segments.clone(); + async move { + let buffer = segments.request(id).await?; + crate::segments::apply_ranges(buffer, &ranges) + } + .boxed() + } } #[tokio::test] @@ -118,6 +174,26 @@ mod tests { assert_eq!(source.request_count.load(Ordering::Relaxed), 1); } + #[tokio::test] + async fn test_shared_source_deduplicates_concurrent_ranged_requests() { + let source = CountingSegmentSource::default(); + + let data = ByteBuffer::from(vec![1, 2, 3, 4, 5, 6]); + let seq_id = SequenceId::root().downgrade(); + source.segments.write(seq_id, vec![data]).await.unwrap(); + + let shared_source = SharedSegmentSource::new(source.clone()); + let id = SegmentId::from(0); + let ranges = vec![1..3, 4..6]; + let future1 = shared_source.request_ranges(id, ranges.clone()); + let future2 = shared_source.request_ranges(id, ranges); + + let (result1, result2) = futures::join!(future1, future2); + assert_eq!(result1.unwrap().unwrap_host().as_slice(), &[2, 3, 5, 6]); + assert_eq!(result2.unwrap().unwrap_host().as_slice(), &[2, 3, 5, 6]); + assert_eq!(source.range_request_count.load(Ordering::Relaxed), 1); + } + #[tokio::test] async fn test_shared_source_handles_dropped_futures() { let source = CountingSegmentSource::default(); diff --git a/vortex-layout/src/segments/source.rs b/vortex-layout/src/segments/source.rs index a48a79b2889..6bcaa52b76c 100644 --- a/vortex-layout/src/segments/source.rs +++ b/vortex-layout/src/segments/source.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; + +use futures::FutureExt; use futures::future::BoxFuture; use vortex_array::buffer::BufferHandle; use vortex_error::VortexResult; @@ -9,8 +12,42 @@ use crate::segments::SegmentId; /// Static future resolving to a segment byte buffer. pub type SegmentFuture = BoxFuture<'static, VortexResult>; +/// Apply a set of segment-relative byte ranges to an already-resolved segment buffer. +pub(crate) fn apply_ranges( + buffer: BufferHandle, + ranges: &[Range], +) -> VortexResult { + match ranges { + [] => buffer.filter(&[]), + [range] => Ok(buffer.slice(range.clone())), + _ => buffer.filter(ranges), + } +} + /// A trait for providing segment data to a [`crate::LayoutReader`]. pub trait SegmentSource: 'static + Send + Sync { + /// Return the full length of a segment in bytes if known without issuing I/O. + fn segment_len(&self, _id: SegmentId) -> Option { + None + } + /// Request a segment, returning a future that will eventually resolve to the segment data. fn request(&self, id: SegmentId) -> SegmentFuture; + + /// Request a set of segment-relative byte ranges and return them packed into one contiguous buffer. + /// + /// Implementations may satisfy this by issuing multiple underlying reads, but the returned + /// [`BufferHandle`] must contain the concatenated bytes in the same order as `ranges`. + /// + /// Follow-up: push scatter/gather support into the lower I/O API so sources can fill a + /// caller-owned destination buffer directly and avoid the gather copy that this interface may + /// currently require. + fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { + let future = self.request(id); + async move { + let buffer = future.await?; + apply_ranges(buffer, &ranges) + } + .boxed() + } } diff --git a/vortex-layout/src/segments/test.rs b/vortex-layout/src/segments/test.rs index d880d15cc1a..44ce2fb0da0 100644 --- a/vortex-layout/src/segments/test.rs +++ b/vortex-layout/src/segments/test.rs @@ -26,6 +26,10 @@ pub struct TestSegments { } impl SegmentSource for TestSegments { + fn segment_len(&self, id: SegmentId) -> Option { + self.segments.lock().get(*id as usize).map(ByteBuffer::len) + } + fn request(&self, id: SegmentId) -> SegmentFuture { let buffer = self.segments.lock().get(*id as usize).cloned(); async move { From 138b5deb9c30374f7bbf3659f59ce7f6bbf75fee Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 15:56:15 -0400 Subject: [PATCH 07/10] Lazy Buffers Signed-off-by: Nicholas Gates --- encodings/bytebool/src/slice.rs | 15 +++++++++------ vortex-array/src/arrays/decimal/compute/rules.rs | 15 +++++++++------ .../src/arrays/fixed_size_list/compute/slice.rs | 12 +++++++++--- .../src/arrays/primitive/compute/slice.rs | 15 +++++++++------ .../src/arrays/varbinview/compute/slice.rs | 15 +++++++++------ vortex-array/src/buffer.rs | 4 ++++ vortex-cuda/src/device_buffer.rs | 3 +++ vortex-file/src/segments/source.rs | 4 ++++ vortex-layout/src/segments/source.rs | 11 +++++++++++ 9 files changed, 67 insertions(+), 27 deletions(-) diff --git a/encodings/bytebool/src/slice.rs b/encodings/bytebool/src/slice.rs index e8072f9eb1c..7da64a7c623 100644 --- a/encodings/bytebool/src/slice.rs +++ b/encodings/bytebool/src/slice.rs @@ -9,6 +9,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::filter::FilterReduce; use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; +use vortex_mask::AllOr; use vortex_mask::Mask; use crate::ByteBool; @@ -28,12 +29,14 @@ impl SliceReduce for ByteBool { impl FilterReduce for ByteBool { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - let ranges: Vec> = mask - .slices() - .unwrap_or_else(|| unreachable!(), || unreachable!()) - .iter() - .map(|&(s, e)| s..e) - .collect(); + let ranges = match mask.slices() { + AllOr::Some(slices) => slices, + // Precondition: FilterReduce only runs for non-trivial masks. + AllOr::All | AllOr::None => { + unreachable!("precondition violated: expected a Mask::Values slice list") + } + }; + let ranges: Vec> = ranges.iter().map(|&(s, e)| s..e).collect(); Ok(Some( ByteBoolData::new( array.buffer().filter_typed::(&ranges)?, diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs index ffe232a79f3..28b82e983c0 100644 --- a/vortex-array/src/arrays/decimal/compute/rules.rs +++ b/vortex-array/src/arrays/decimal/compute/rules.rs @@ -4,6 +4,7 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::AllOr; use vortex_mask::Mask; use crate::ArrayRef; @@ -78,12 +79,14 @@ impl SliceReduce for Decimal { impl FilterReduce for Decimal { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - let ranges: Vec> = mask - .slices() - .unwrap_or_else(|| unreachable!(), || unreachable!()) - .iter() - .map(|&(s, e)| s..e) - .collect(); + let ranges = match mask.slices() { + AllOr::Some(slices) => slices, + // Precondition: FilterReduce only runs for non-trivial masks. + AllOr::All | AllOr::None => { + unreachable!("precondition violated: expected a Mask::Values slice list") + } + }; + let ranges: Vec> = ranges.iter().map(|&(s, e)| s..e).collect(); let result = match_each_decimal_value_type!(array.values_type(), |D| { // SAFETY: Filtering preserves all DecimalArray invariants — values within // precision bounds remain valid, and we correctly filter the validity. diff --git a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs index b429d986075..cd708d9c2b9 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs @@ -4,6 +4,7 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::AllOr; use vortex_mask::Mask; use crate::ArrayRef; @@ -46,9 +47,14 @@ impl FilterReduce for FixedSizeList { array.elements().clone() } else { let elements_len = array.elements().len(); - let expanded_slices: Vec<(usize, usize)> = mask - .slices() - .unwrap_or_else(|| unreachable!(), || unreachable!()) + let slices = match mask.slices() { + AllOr::Some(slices) => slices, + // Precondition: FilterReduce only runs for non-trivial masks. + AllOr::All | AllOr::None => { + unreachable!("precondition violated: expected a Mask::Values slice list") + } + }; + let expanded_slices: Vec<(usize, usize)> = slices .iter() .map(|&(s, e)| (s * list_size, e * list_size)) .collect(); diff --git a/vortex-array/src/arrays/primitive/compute/slice.rs b/vortex-array/src/arrays/primitive/compute/slice.rs index e0a624af5d5..dc71730ebd0 100644 --- a/vortex-array/src/arrays/primitive/compute/slice.rs +++ b/vortex-array/src/arrays/primitive/compute/slice.rs @@ -4,6 +4,7 @@ use std::ops::Range; use vortex_error::VortexResult; +use vortex_mask::AllOr; use vortex_mask::Mask; use crate::ArrayRef; @@ -32,12 +33,14 @@ impl SliceReduce for Primitive { impl FilterReduce for Primitive { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - let ranges: Vec> = mask - .slices() - .unwrap_or_else(|| unreachable!(), || unreachable!()) - .iter() - .map(|&(s, e)| s..e) - .collect(); + let ranges = match mask.slices() { + AllOr::Some(slices) => slices, + // Precondition: FilterReduce only runs for non-trivial masks. + AllOr::All | AllOr::None => { + unreachable!("precondition violated: expected a Mask::Values slice list") + } + }; + let ranges: Vec> = ranges.iter().map(|&(s, e)| s..e).collect(); let result = match_each_native_ptype!(array.ptype(), |T| { PrimitiveArray::from_buffer_handle( array.buffer_handle().filter_typed::(&ranges)?, diff --git a/vortex-array/src/arrays/varbinview/compute/slice.rs b/vortex-array/src/arrays/varbinview/compute/slice.rs index 33d617a72df..9b42ddf1ec7 100644 --- a/vortex-array/src/arrays/varbinview/compute/slice.rs +++ b/vortex-array/src/arrays/varbinview/compute/slice.rs @@ -5,6 +5,7 @@ use std::ops::Range; use std::sync::Arc; use vortex_error::VortexResult; +use vortex_mask::AllOr; use vortex_mask::Mask; use crate::ArrayRef; @@ -34,12 +35,14 @@ impl SliceReduce for VarBinView { impl FilterReduce for VarBinView { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - let ranges: Vec> = mask - .slices() - .unwrap_or_else(|| unreachable!(), || unreachable!()) - .iter() - .map(|&(s, e)| s..e) - .collect(); + let ranges = match mask.slices() { + AllOr::Some(slices) => slices, + // Precondition: FilterReduce only runs for non-trivial masks. + AllOr::All | AllOr::None => { + unreachable!("precondition violated: expected a Mask::Values slice list") + } + }; + let ranges: Vec> = ranges.iter().map(|&(s, e)| s..e).collect(); Ok(Some( VarBinViewArray::new_handle( array.views_handle().filter_typed::(&ranges)?, diff --git a/vortex-array/src/buffer.rs b/vortex-array/src/buffer.rs index df511e95c3e..3b20a7699ca 100644 --- a/vortex-array/src/buffer.rs +++ b/vortex-array/src/buffer.rs @@ -93,6 +93,10 @@ pub trait DeviceBuffer: 'static + Send + Sync + Debug + DynEq + DynHash { /// Unlike [`slice`](DeviceBuffer::slice), this method allocates new memory and copies the /// selected ranges into a contiguous buffer. /// + /// Follow-up: introduce a lazy/composite device-buffer representation for deferred multi-range + /// gathers. That would let accelerators accumulate slice/filter plans and realize them later + /// with a device-native gather/compaction kernel instead of eagerly copying here. + /// /// # Errors /// /// Returns an error if the device cannot allocate memory or copy the data. diff --git a/vortex-cuda/src/device_buffer.rs b/vortex-cuda/src/device_buffer.rs index 49af1728622..86f50babe07 100644 --- a/vortex-cuda/src/device_buffer.rs +++ b/vortex-cuda/src/device_buffer.rs @@ -306,6 +306,9 @@ impl DeviceBuffer for CudaDeviceBuffer { } // Allocate new device memory for the filtered result. + // Follow-up: represent multi-range device selections lazily so CUDA can defer this gather + // and realize it later with a dedicated gather/compaction kernel (for example via CUB or + // a custom kernel) instead of eagerly issuing one memcpy per selected range here. let dst_slice: CudaSlice = unsafe { stream .alloc::(total_len) diff --git a/vortex-file/src/segments/source.rs b/vortex-file/src/segments/source.rs index 3ab91799f08..ba2fbc6fb51 100644 --- a/vortex-file/src/segments/source.rs +++ b/vortex-file/src/segments/source.rs @@ -261,6 +261,10 @@ impl SegmentSource for FileSegmentSource { .await?; // Follow-up: teach the lower I/O API to support scatter/gather into a // caller-owned destination buffer so this gather copy can be removed. + // Local files should be able to use vectored `preadv`/`preadv2`-style reads + // into the final output slices, while other backends can continue to issue a + // smaller number of merged contiguous reads. A later follow-up should thread + // through `DIRECT_IO` alignment/padding constraints for local files as well. let mut gathered = ByteBufferMut::with_capacity_aligned(total_len, Alignment::none()); for chunk in chunks { diff --git a/vortex-layout/src/segments/source.rs b/vortex-layout/src/segments/source.rs index 6bcaa52b76c..b28abc99ff8 100644 --- a/vortex-layout/src/segments/source.rs +++ b/vortex-layout/src/segments/source.rs @@ -42,6 +42,17 @@ pub trait SegmentSource: 'static + Send + Sync { /// Follow-up: push scatter/gather support into the lower I/O API so sources can fill a /// caller-owned destination buffer directly and avoid the gather copy that this interface may /// currently require. + /// + /// The intended split is: + /// - this API continues to describe the logical bytes to materialize; + /// - sources may normalize/merge ranges before issuing physical reads; + /// - lower I/O backends may optionally expose vectored reads into output slices. + /// + /// That lets local files eventually use `preadv`/`preadv2`-style reads for sparse ranges, + /// while remote/object-store backends can still choose a smaller number of coalesced + /// contiguous reads when that is cheaper. A later follow-up should also thread through + /// alignment requirements for `DIRECT_IO`, since that may force padded physical reads even + /// when the logical request stays sparse. fn request_ranges(&self, id: SegmentId, ranges: Vec>) -> SegmentFuture { let future = self.request(id); async move { From 0fb30575622250191f525a7f73f88c5a78178e51 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 17:05:27 -0400 Subject: [PATCH 08/10] Follow up Signed-off-by: Nicholas Gates --- Cargo.lock | 897 +----------------- encodings/alp/Cargo.toml | 6 - encodings/bytebool/Cargo.toml | 4 - encodings/datetime-parts/Cargo.toml | 5 - encodings/decimal-byte-parts/Cargo.toml | 4 - encodings/fastlanes/Cargo.toml | 9 - encodings/fsst/Cargo.toml | 6 - encodings/parquet-variant/Cargo.toml | 4 - encodings/pco/Cargo.toml | 4 - encodings/runend/Cargo.toml | 9 - encodings/sequence/Cargo.toml | 5 - encodings/sparse/Cargo.toml | 5 - encodings/zigzag/Cargo.toml | 4 - encodings/zstd/Cargo.toml | 5 - vortex-array/Cargo.toml | 11 - .../aggregate_fn/fns/is_sorted/primitive.rs | 6 +- vortex-array/src/arrays/bool/array.rs | 4 +- .../src/arrays/primitive/vtable/operations.rs | 2 +- vortex-btrblocks/Cargo.toml | 6 - vortex-buffer/Cargo.toml | 5 - vortex-compressor/Cargo.toml | 4 - vortex-cuda/Cargo.toml | 8 - vortex-datafusion/Cargo.toml | 9 - vortex-duckdb/Cargo.toml | 9 - vortex-error/Cargo.toml | 4 - vortex-ffi/Cargo.toml | 4 - vortex-file/Cargo.toml | 7 - vortex-file/src/segments/source.rs | 13 +- vortex-io/Cargo.toml | 7 - vortex-ipc/Cargo.toml | 4 - vortex-jni/Cargo.toml | 3 - vortex-layout/Cargo.toml | 8 - vortex-layout/src/buffer.rs | 11 +- vortex-layout/src/segments/shared.rs | 2 +- vortex-layout/src/segments/source.rs | 5 +- vortex-mask/Cargo.toml | 4 - vortex-tensor/Cargo.toml | 3 - vortex/Cargo.toml | 14 - 38 files changed, 65 insertions(+), 1055 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a4f4b14629..6d76215b3a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,17 +19,6 @@ dependencies = [ "cpufeatures 0.2.17", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.12" @@ -83,12 +72,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "0.6.21" @@ -325,7 +308,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-buffer 57.3.0", "arrow-data 57.3.0", "arrow-schema 57.3.0", @@ -344,7 +327,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-buffer 58.0.0", "arrow-data 58.0.0", "arrow-schema 58.0.0", @@ -641,7 +624,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array 57.3.0", "arrow-buffer 57.3.0", "arrow-data 57.3.0", @@ -655,7 +638,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array 58.0.0", "arrow-buffer 58.0.0", "arrow-data 58.0.0", @@ -902,28 +885,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-lc-rs" -version = "1.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "base64" version = "0.22.1" @@ -1102,30 +1063,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "borsh" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" -dependencies = [ - "borsh-derive", - "bytes", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "brotli" version = "8.0.2" @@ -1170,40 +1107,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" -[[package]] -name = "byte-unit" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c6d47a4e2961fb8721bcfc54feae6455f2f64e7054f9bc67e875f0e77f4c58d" -dependencies = [ - "rust_decimal", - "schemars 1.2.1", - "serde", - "utf8-width", -] - -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytecount" version = "0.6.9" @@ -1270,12 +1173,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "castaway" version = "0.2.4" @@ -1384,33 +1281,6 @@ dependencies = [ "phf", ] -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1429,7 +1299,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.9", + "libloading", ] [[package]] @@ -1452,7 +1322,6 @@ dependencies = [ "anstyle", "clap_lex", "strsim", - "terminal_size", ] [[package]] @@ -1473,15 +1342,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" -dependencies = [ - "cc", -] - [[package]] name = "codespan-reporting" version = "0.13.1" @@ -1493,109 +1353,12 @@ dependencies = [ "unicode-width 0.2.2", ] -[[package]] -name = "codspeed" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b684e94583e85a5ca7e1a6454a89d76a5121240f2fb67eb564129d9bafdb9db0" -dependencies = [ - "anyhow", - "cc", - "colored", - "getrandom 0.2.17", - "glob", - "libc", - "nix", - "serde", - "serde_json", - "statrs", -] - -[[package]] -name = "codspeed-criterion-compat-walltime" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96389aaa4bbb872ea4924dc0335b2bb181bcf28d6eedbe8fea29afcc5bde36a6" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "codspeed", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "codspeed-divan-compat" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e4bf8c7793c170fd0fcf3be97b9032b2ae39c2b9e8818aba3cc10ca0f0c6c0" -dependencies = [ - "clap", - "codspeed", - "codspeed-divan-compat-macros", - "codspeed-divan-compat-walltime", - "regex", -] - -[[package]] -name = "codspeed-divan-compat-macros" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78aae02f2a278588e16e8ca62ea1915b8ab30f8230a09926671bba19ede801a4" -dependencies = [ - "divan-macros", - "itertools 0.14.0", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "codspeed-divan-compat-walltime" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ffd32c0c59ab8b674b15be65ba7c59aebac047036cfa7fa1e11bc2c178b81f" -dependencies = [ - "cfg-if", - "clap", - "codspeed", - "condtype", - "divan-macros", - "libc", - "regex-lite", -] - [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "colored" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" -dependencies = [ - "lazy_static", - "windows-sys 0.59.0", -] - [[package]] name = "combine" version = "4.6.7" @@ -1682,12 +1445,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "condtype" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" - [[package]] name = "console" version = "0.15.11" @@ -1836,16 +1593,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1961,7 +1708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3aa12038120eb13347a6ae2ffab1d34efe78150125108627fd85044dd4d6ff1e" dependencies = [ "half", - "libloading 0.8.9", + "libloading", ] [[package]] @@ -2321,7 +2068,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "arrow-ipc 57.3.0", "chrono", @@ -2343,7 +2090,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" dependencies = [ - "ahash 0.8.12", + "ahash", "apache-avro", "arrow 58.0.0", "arrow-ipc 58.0.0", @@ -2833,7 +2580,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-doc 51.0.0", @@ -2854,7 +2601,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-doc 53.0.0", @@ -2876,7 +2623,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr-common 51.0.0", @@ -2889,7 +2636,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-expr-common 53.0.0", @@ -3099,7 +2846,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr 51.0.0", @@ -3121,7 +2868,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-expr 53.0.0", @@ -3175,7 +2922,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr-common 51.0.0", @@ -3189,7 +2936,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 58.0.0", "chrono", "datafusion-common 53.0.0", @@ -3243,7 +2990,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 57.3.0", "arrow-ord 57.3.0", "arrow-schema 57.3.0", @@ -3274,7 +3021,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow 58.0.0", "arrow-ord 58.0.0", "arrow-schema 58.0.0", @@ -3572,17 +3319,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "divan-macros" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "document-features" version = "0.2.12" @@ -3619,12 +3355,6 @@ dependencies = [ "vortex-duckdb", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -3955,12 +3685,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "fsst" version = "2.0.1" @@ -4401,22 +4125,13 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] - [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.12", + "ahash", "allocator-api2", ] @@ -4858,18 +4573,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "insta" -version = "1.46.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" -dependencies = [ - "console 0.15.11", - "once_cell", - "similar", - "tempfile", -] - [[package]] name = "instability" version = "0.3.12" @@ -4914,32 +4617,12 @@ dependencies = [ "serde", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.11.0" @@ -4982,15 +4665,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "java-locator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f" -dependencies = [ - "glob", -] - [[package]] name = "jiff" version = "0.2.23" @@ -5041,9 +4715,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "java-locator", "jni-sys 0.3.1", - "libloading 0.7.4", "log", "thiserror 1.0.69", "walkdir", @@ -5579,7 +5251,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2acdba67f84190067532fce07b51a435dd390d7cdc1129a05003e5cb3274cf0" dependencies = [ - "reqwest 0.12.28", + "reqwest", "serde", "serde_json", "serde_repr", @@ -5747,16 +5419,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libloading" version = "0.8.9" @@ -6170,18 +5832,6 @@ version = "6.6.666" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6" -[[package]] -name = "nix" -version = "0.31.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -6473,7 +6123,7 @@ dependencies = [ "percent-encoding", "quick-xml", "rand 0.9.2", - "reqwest 0.12.28", + "reqwest", "ring", "rustls-pki-types", "serde", @@ -6515,12 +6165,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -6551,7 +6195,7 @@ dependencies = [ "bytes", "http", "opentelemetry", - "reqwest 0.12.28", + "reqwest", ] [[package]] @@ -6566,7 +6210,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost 0.14.3", - "reqwest 0.12.28", + "reqwest", "thiserror 2.0.18", "tracing", ] @@ -6712,7 +6356,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array 57.3.0", "arrow-buffer 57.3.0", "arrow-cast 57.3.0", @@ -6746,7 +6390,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array 58.0.0", "arrow-buffer 58.0.0", "arrow-data 58.0.0", @@ -6972,17 +6616,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "ping" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" -dependencies = [ - "rand 0.9.2", - "socket2", - "thiserror 2.0.18", -] - [[package]] name = "piper" version = "0.2.5" @@ -7007,38 +6640,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] -name = "plotters" -version = "0.3.7" +name = "polling" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "polling" -version = "3.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ "cfg-if", "concurrent-queue", @@ -7129,28 +6734,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -7244,26 +6827,6 @@ dependencies = [ "cc", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "public-api" version = "0.51.0" @@ -7438,7 +7001,6 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ - "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -7639,7 +7201,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" dependencies = [ - "ahash 0.8.12", + "ahash", "brotli", "paste", "rand 0.9.2", @@ -7806,26 +7368,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "regex" version = "1.12.3" @@ -7877,15 +7419,6 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -7933,46 +7466,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "reqwest" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "mime", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "ring" version = "0.17.14" @@ -7987,35 +7480,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "roaring" version = "0.10.12" @@ -8103,22 +7567,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "rust_decimal" -version = "1.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", -] - [[package]] name = "rustc-hash" version = "2.1.1" @@ -8182,7 +7630,6 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ - "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -8213,40 +7660,12 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - [[package]] name = "rustls-webpki" version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -8273,15 +7692,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scc" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" -dependencies = [ - "sdd", -] - [[package]] name = "schannel" version = "0.1.29" @@ -8303,18 +7713,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "schemars" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - [[package]] name = "schemars_derive" version = "0.8.22" @@ -8345,18 +7743,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" -[[package]] -name = "sdd" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "3.7.0" @@ -8480,15 +7866,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_test" -version = "1.0.177" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" -dependencies = [ - "serde", -] - [[package]] name = "serde_tokenstream" version = "0.2.3" @@ -8526,32 +7903,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "serial_test" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" -dependencies = [ - "futures-executor", - "futures-util", - "log", - "once_cell", - "parking_lot", - "scc", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "sha1" version = "0.10.6" @@ -8872,16 +8223,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "statrs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" -dependencies = [ - "approx", - "num-traits", -] - [[package]] name = "std_prelude" version = "0.2.12" @@ -8989,7 +8330,7 @@ dependencies = [ "prost-build", "prost-types", "regress", - "schemars 0.8.22", + "schemars", "semver", "serde", "serde_json", @@ -9291,15 +8632,6 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" -[[package]] -name = "temp-env" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" -dependencies = [ - "parking_lot", -] - [[package]] name = "tempfile" version = "3.27.0" @@ -9322,43 +8654,12 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" -dependencies = [ - "rustix 1.1.4", - "windows-sys 0.60.2", -] - [[package]] name = "termtree" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d1330fe7f7f872cd05165130b10602d667b205fd85be09be2814b115d4ced9" -[[package]] -name = "test-with" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27838d769fa9bf364bf4a352ec88862b6e6cb96a8e6705cc78fbb16ff26ee8b9" -dependencies = [ - "byte-unit", - "chrono", - "num_cpus", - "ping", - "proc-macro-error2", - "proc-macro2", - "quote", - "regex", - "reqwest 0.13.2", - "syn 2.0.117", - "sysinfo", - "uzers", - "which", -] - [[package]] name = "testing_table" version = "0.3.0" @@ -9500,16 +8801,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.11.0" @@ -9883,7 +9174,7 @@ dependencies = [ "proc-macro2", "quote", "regress", - "schemars 0.8.22", + "schemars", "semver", "serde", "serde_json", @@ -9900,7 +9191,7 @@ checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" dependencies = [ "proc-macro2", "quote", - "schemars 0.8.22", + "schemars", "semver", "serde", "serde_json", @@ -10005,12 +9296,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" -[[package]] -name = "utf8-width" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -10035,16 +9320,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "uzers" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d283dc7e8c901e79e32d077866eaf599156cbf427fffa8289aecc52c5c3f63" -dependencies = [ - "libc", - "log", -] - [[package]] name = "valuable" version = "0.1.1" @@ -10061,18 +9336,6 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vortex" version = "0.1.0" dependencies = [ - "anyhow", - "arrow-array 58.0.0", - "codspeed-divan-compat", - "fastlanes", - "mimalloc", - "parquet 58.0.0", - "rand 0.10.0", - "serde_json", - "tokio", - "tracing", - "tracing-subscriber", - "vortex", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -10106,12 +9369,9 @@ dependencies = [ name = "vortex-alp" version = "0.1.0" dependencies = [ - "codspeed-divan-compat", "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rand 0.10.0", - "rstest", "rustc-hash", "vortex-array", "vortex-buffer", @@ -10140,7 +9400,6 @@ dependencies = [ "async-lock", "bytes", "cfg-if", - "codspeed-divan-compat", "cudarc", "enum-iterator", "flatbuffers", @@ -10149,7 +9408,6 @@ dependencies = [ "goldenfile", "half", "humansize", - "insta", "inventory", "itertools 0.14.0", "jiff", @@ -10162,20 +9420,16 @@ dependencies = [ "primitive-types", "prost 0.14.3", "rand 0.10.0", - "rand_distr 0.6.0", "rstest", "rstest_reuse", "rustc-hash", "serde", - "serde_json", - "serde_test", "simdutf8", "static_assertions", "tabled", "termtree", "tracing", "uuid", - "vortex-array", "vortex-buffer", "vortex-error", "vortex-flatbuffers", @@ -10210,7 +9464,7 @@ dependencies = [ "parquet 58.0.0", "rand 0.10.0", "regex", - "reqwest 0.12.28", + "reqwest", "serde", "serde_json", "sysinfo", @@ -10233,15 +9487,12 @@ dependencies = [ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ - "codspeed-divan-compat", "getrandom 0.4.2", "itertools 0.14.0", "num-traits", "pco", "rand 0.10.0", - "rstest", "rustc-hash", - "test-with", "tracing", "vortex-alp", "vortex-array", @@ -10269,11 +9520,8 @@ dependencies = [ "arrow-buffer 58.0.0", "bitvec", "bytes", - "codspeed-divan-compat", "itertools 0.14.0", "memmap2", - "num-traits", - "rstest", "serde", "simdutf8", "tracing", @@ -10285,7 +9533,6 @@ name = "vortex-bytebool" version = "0.1.0" dependencies = [ "num-traits", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10303,7 +9550,7 @@ dependencies = [ "clap", "futures", "parquet 58.0.0", - "reqwest 0.12.28", + "reqwest", "serde", "serde_json", "sha2", @@ -10326,7 +9573,6 @@ dependencies = [ "num-traits", "parking_lot", "rand 0.10.0", - "rstest", "rustc-hash", "tracing", "vortex-array", @@ -10341,7 +9587,7 @@ name = "vortex-cub" version = "0.1.0" dependencies = [ "bindgen", - "libloading 0.8.9", + "libloading", "paste", "vortex-array", "vortex-cuda-macros", @@ -10355,7 +9601,6 @@ dependencies = [ "async-trait", "bindgen", "bytes", - "codspeed-criterion-compat-walltime", "cudarc", "fastlanes", "futures", @@ -10364,13 +9609,11 @@ dependencies = [ "object_store 0.13.1", "parking_lot", "prost 0.14.3", - "rstest", "tokio", "tracing", "vortex", "vortex-array", "vortex-cub", - "vortex-cuda", "vortex-cuda-macros", "vortex-error", "vortex-nvcomp", @@ -10403,10 +9646,8 @@ dependencies = [ name = "vortex-datafusion" version = "0.1.0" dependencies = [ - "anyhow", "arrow-schema 58.0.0", "async-trait", - "datafusion 53.0.0", "datafusion-catalog 53.0.0", "datafusion-common 53.0.0", "datafusion-common-runtime 53.0.0", @@ -10420,15 +9661,11 @@ dependencies = [ "datafusion-physical-plan 53.0.0", "datafusion-pruning 53.0.0", "futures", - "insta", "itertools 0.14.0", "object_store 0.13.1", - "rstest", - "tempfile", "tokio", "tokio-stream", "tracing", - "url", "vortex", "vortex-utils", ] @@ -10439,7 +9676,6 @@ version = "0.1.0" dependencies = [ "num-traits", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10453,7 +9689,6 @@ version = "0.1.0" dependencies = [ "num-traits", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10465,7 +9700,6 @@ dependencies = [ name = "vortex-duckdb" version = "0.1.0" dependencies = [ - "anyhow", "async-fs", "async-trait", "bindgen", @@ -10475,21 +9709,15 @@ dependencies = [ "custom-labels", "futures", "itertools 0.14.0", - "jiff", "kanal", "num-traits", "object_store 0.13.1", "parking_lot", "paste", - "reqwest 0.12.28", - "rstest", - "tempfile", + "reqwest", "tracing", "url", "vortex", - "vortex-array", - "vortex-runend", - "vortex-sequence", "vortex-utils", "zip", ] @@ -10503,8 +9731,6 @@ dependencies = [ "jiff", "object_store 0.13.1", "prost 0.14.3", - "serial_test", - "temp-env", "tokio", ] @@ -10513,19 +9739,15 @@ name = "vortex-fastlanes" version = "0.1.0" dependencies = [ "arrayref", - "codspeed-divan-compat", "fastlanes", "itertools 0.14.0", "lending-iterator", "num-traits", "prost 0.14.3", "rand 0.10.0", - "rstest", - "vortex-alp", "vortex-array", "vortex-buffer", "vortex-error", - "vortex-fastlanes", "vortex-mask", "vortex-session", ] @@ -10542,12 +9764,10 @@ dependencies = [ "object_store 0.13.1", "paste", "prost 0.14.3", - "tempfile", "tracing", "tracing-subscriber", "url", "vortex", - "vortex-array", ] [[package]] @@ -10608,11 +9828,9 @@ dependencies = [ name = "vortex-fsst" version = "0.1.0" dependencies = [ - "codspeed-divan-compat", "fsst-rs", "prost 0.14.3", "rand 0.10.0", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10650,7 +9868,6 @@ dependencies = [ name = "vortex-io" version = "0.1.0" dependencies = [ - "anyhow", "async-fs", "async-stream", "async-trait", @@ -10660,15 +9877,12 @@ dependencies = [ "getrandom 0.4.2", "glob", "handle", - "itertools 0.14.0", "kanal", "object_store 0.13.1", "oneshot 0.2.1", "parking_lot", "pin-project-lite", - "rstest", "smol", - "tempfile", "tokio", "tracing", "vortex-array", @@ -10688,7 +9902,6 @@ dependencies = [ "futures", "itertools 0.14.0", "pin-project-lite", - "tokio", "vortex-array", "vortex-buffer", "vortex-error", @@ -10737,7 +9950,6 @@ dependencies = [ "paste", "pin-project-lite", "prost 0.14.3", - "rstest", "rustc-hash", "sketches-ddsketch 0.4.0", "termtree", @@ -10762,9 +9974,7 @@ name = "vortex-mask" version = "0.1.0" dependencies = [ "arrow-buffer 58.0.0", - "codspeed-divan-compat", "itertools 0.14.0", - "rstest", "serde", "vortex-buffer", "vortex-error", @@ -10784,9 +9994,9 @@ name = "vortex-nvcomp" version = "0.1.0" dependencies = [ "bindgen", - "libloading 0.8.9", + "libloading", "liblzma", - "reqwest 0.12.28", + "reqwest", "tar", "vortex-cuda-macros", ] @@ -10802,7 +10012,6 @@ dependencies = [ "parquet-variant", "parquet-variant-compute", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10817,7 +10026,6 @@ version = "0.1.0" dependencies = [ "pco", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10862,13 +10070,9 @@ version = "0.1.0" dependencies = [ "arbitrary", "arrow-array 58.0.0", - "arrow-schema 58.0.0", - "codspeed-divan-compat", "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rand 0.10.0", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10895,10 +10099,8 @@ dependencies = [ name = "vortex-sequence" version = "0.1.0" dependencies = [ - "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10925,7 +10127,6 @@ dependencies = [ "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10961,7 +10162,6 @@ dependencies = [ "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rstest", "vortex", ] @@ -11041,7 +10241,6 @@ dependencies = [ name = "vortex-zigzag" version = "0.1.0" dependencies = [ - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -11054,10 +10253,8 @@ dependencies = [ name = "vortex-zstd" version = "0.1.0" dependencies = [ - "codspeed-divan-compat", "itertools 0.14.0", "prost 0.14.3", - "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -11235,15 +10432,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-root-certs" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "webpki-roots" version = "1.0.6" @@ -11253,15 +10441,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "which" -version = "8.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" -dependencies = [ - "libc", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/encodings/alp/Cargo.toml b/encodings/alp/Cargo.toml index 13ccc67d63b..7d1c457fed0 100644 --- a/encodings/alp/Cargo.toml +++ b/encodings/alp/Cargo.toml @@ -29,12 +29,6 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } vortex-utils = { workspace = true } -[dev-dependencies] -divan = { workspace = true } -rand = { workspace = true } -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [[bench]] name = "alp_compress" harness = false diff --git a/encodings/bytebool/Cargo.toml b/encodings/bytebool/Cargo.toml index bca2ed71aa9..05cf5b82ce2 100644 --- a/encodings/bytebool/Cargo.toml +++ b/encodings/bytebool/Cargo.toml @@ -23,7 +23,3 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/datetime-parts/Cargo.toml b/encodings/datetime-parts/Cargo.toml index 52894fc7c07..6f7742f773d 100644 --- a/encodings/datetime-parts/Cargo.toml +++ b/encodings/datetime-parts/Cargo.toml @@ -24,8 +24,3 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } -vortex-error = { workspace = true } diff --git a/encodings/decimal-byte-parts/Cargo.toml b/encodings/decimal-byte-parts/Cargo.toml index 4934ec4fa27..f60b5590445 100644 --- a/encodings/decimal-byte-parts/Cargo.toml +++ b/encodings/decimal-byte-parts/Cargo.toml @@ -24,7 +24,3 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { path = "../../vortex-array", features = ["_test-harness"] } diff --git a/encodings/fastlanes/Cargo.toml b/encodings/fastlanes/Cargo.toml index ac30b2032b4..e043555f2fa 100644 --- a/encodings/fastlanes/Cargo.toml +++ b/encodings/fastlanes/Cargo.toml @@ -30,15 +30,6 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } -[dev-dependencies] -divan = { workspace = true } -itertools = { workspace = true } -rand = { workspace = true } -rstest = { workspace = true } -vortex-alp = { path = "../alp" } -vortex-array = { workspace = true, features = ["_test-harness"] } -vortex-fastlanes = { path = ".", features = ["_test-harness"] } - [features] _test-harness = ["dep:rand"] diff --git a/encodings/fsst/Cargo.toml b/encodings/fsst/Cargo.toml index b95eeb1f444..5ac799a8813 100644 --- a/encodings/fsst/Cargo.toml +++ b/encodings/fsst/Cargo.toml @@ -29,12 +29,6 @@ vortex-session = { workspace = true } [features] _test-harness = ["dep:rand", "vortex-array/_test-harness"] -[dev-dependencies] -divan = { workspace = true } -rand = { workspace = true } -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [[bench]] name = "fsst_compress" harness = false diff --git a/encodings/parquet-variant/Cargo.toml b/encodings/parquet-variant/Cargo.toml index cc554c85d10..9f6ec204d4a 100644 --- a/encodings/parquet-variant/Cargo.toml +++ b/encodings/parquet-variant/Cargo.toml @@ -32,9 +32,5 @@ vortex-mask = { workspace = true } vortex-proto = { workspace = true } vortex-session = { workspace = true } -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [package.metadata.cargo-machete] ignored = ["getrandom_v03"] diff --git a/encodings/pco/Cargo.toml b/encodings/pco/Cargo.toml index bc222bf1dcc..278a15f40e1 100644 --- a/encodings/pco/Cargo.toml +++ b/encodings/pco/Cargo.toml @@ -25,9 +25,5 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [package.metadata.cargo-machete] ignored = ["getrandom_v03"] diff --git a/encodings/runend/Cargo.toml b/encodings/runend/Cargo.toml index 01a5b8d7a3e..4b709802530 100644 --- a/encodings/runend/Cargo.toml +++ b/encodings/runend/Cargo.toml @@ -28,15 +28,6 @@ vortex-session = { workspace = true } [lints] workspace = true -[dev-dependencies] -arrow-array = { workspace = true } -arrow-schema = { workspace = true } -divan = { workspace = true } -itertools = { workspace = true } -rand = { workspace = true } -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [features] arbitrary = ["dep:arbitrary", "vortex-array/arbitrary"] arrow = ["dep:arrow-array"] diff --git a/encodings/sequence/Cargo.toml b/encodings/sequence/Cargo.toml index 06f695cb989..1ef12c62df7 100644 --- a/encodings/sequence/Cargo.toml +++ b/encodings/sequence/Cargo.toml @@ -23,10 +23,5 @@ vortex-mask = { workspace = true } vortex-proto = { workspace = true } vortex-session = { workspace = true } -[dev-dependencies] -itertools = { workspace = true } -rstest = { workspace = true } -vortex-array = { path = "../../vortex-array", features = ["_test-harness"] } - [lints] workspace = true diff --git a/encodings/sparse/Cargo.toml b/encodings/sparse/Cargo.toml index 53a25ad535a..3c1f34868cb 100644 --- a/encodings/sparse/Cargo.toml +++ b/encodings/sparse/Cargo.toml @@ -25,8 +25,3 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } - -[dev-dependencies] -itertools = { workspace = true } -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/zigzag/Cargo.toml b/encodings/zigzag/Cargo.toml index 83e6d324ed1..4f21527c915 100644 --- a/encodings/zigzag/Cargo.toml +++ b/encodings/zigzag/Cargo.toml @@ -21,9 +21,5 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } zigzag = { workspace = true } -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [lints] workspace = true diff --git a/encodings/zstd/Cargo.toml b/encodings/zstd/Cargo.toml index 4383f765a2f..4cfab63e2fb 100644 --- a/encodings/zstd/Cargo.toml +++ b/encodings/zstd/Cargo.toml @@ -30,11 +30,6 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } zstd = { workspace = true } -[dev-dependencies] -divan = { workspace = true } -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [[bench]] name = "listview_rebuild" harness = false diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index dbdfd5a2cd9..d00e89db1b7 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -84,17 +84,6 @@ table-display = ["dep:tabled"] _test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"] serde = ["dep:serde", "vortex-buffer/serde", "vortex-mask/serde"] -[dev-dependencies] -arrow-cast = { workspace = true } -divan = { workspace = true } -futures = { workspace = true, features = ["executor"] } -insta = { workspace = true } -rand_distr = { workspace = true } -rstest = { workspace = true } -serde_json = { workspace = true } -serde_test = { workspace = true } -vortex-array = { path = ".", features = ["_test-harness", "table-display"] } - [[bench]] name = "search_sorted" harness = false diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs index 21c80e7bd45..920052f575c 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs @@ -16,11 +16,11 @@ pub(super) fn check_primitive_sorted(array: &PrimitiveArray, strict: bool) -> Vo } fn compute_is_sorted(array: &PrimitiveArray, strict: bool) -> VortexResult { + let values = array.to_buffer::(); match array.validity_mask()? { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { - let slice = array.as_slice::(); - let iter = slice.iter().copied().map(NativeValue); + let iter = values.iter().copied().map(NativeValue); Ok(if strict { iter.is_strict_sorted() @@ -32,7 +32,7 @@ fn compute_is_sorted(array: &PrimitiveArray, strict: bool) -> Vo let iter = mask_values .bit_buffer() .iter() - .zip_eq(array.as_slice::()) + .zip_eq(values.iter()) .map(|(is_valid, value)| is_valid.then_some(NativeValue(*value))); Ok(if strict { diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 4c654a44081..6982cc82643 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -106,13 +106,13 @@ impl BoolData { /// Returns the underlying [`BitBuffer`] of the array. pub fn to_bit_buffer(&self) -> BitBuffer { - let buffer = self.bits.as_host().clone(); + let buffer = self.bits.to_host_sync(); BitBuffer::new_with_offset(buffer, self.len, self.offset) } /// Returns the underlying [`BitBuffer`] of the array pub fn into_bit_buffer(self) -> BitBuffer { - let buffer = self.bits.unwrap_host(); + let buffer = self.bits.into_host_sync(); BitBuffer::new_with_offset(buffer, self.len, self.offset) } diff --git a/vortex-array/src/arrays/primitive/vtable/operations.rs b/vortex-array/src/arrays/primitive/vtable/operations.rs index ddeaa386485..230da813e6b 100644 --- a/vortex-array/src/arrays/primitive/vtable/operations.rs +++ b/vortex-array/src/arrays/primitive/vtable/operations.rs @@ -17,7 +17,7 @@ impl OperationsVTable for Primitive { _ctx: &mut ExecutionCtx, ) -> VortexResult { Ok(match_each_native_ptype!(array.ptype(), |T| { - Scalar::primitive(array.as_slice::()[index], array.dtype().nullability()) + Scalar::primitive(array.to_buffer::()[index], array.dtype().nullability()) })) } } diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 9bbd2430f09..83629cbd2f0 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -39,12 +39,6 @@ vortex-utils = { workspace = true } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } -[dev-dependencies] -divan = { workspace = true } -rstest = { workspace = true } -test-with = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [features] # This feature enabled unstable encodings for which we don't guarantee stability. unstable_encodings = ["vortex-zstd?/unstable_encodings"] diff --git a/vortex-buffer/Cargo.toml b/vortex-buffer/Cargo.toml index ae9d7e6cc05..8cdf9d724e2 100644 --- a/vortex-buffer/Cargo.toml +++ b/vortex-buffer/Cargo.toml @@ -36,11 +36,6 @@ vortex-error = { workspace = true } [lints] workspace = true -[dev-dependencies] -divan = { workspace = true } -num-traits = { workspace = true } -rstest = { workspace = true } - [[bench]] name = "vortex_buffer" harness = false diff --git a/vortex-compressor/Cargo.toml b/vortex-compressor/Cargo.toml index 260c9c531f5..f5fa3fcc86e 100644 --- a/vortex-compressor/Cargo.toml +++ b/vortex-compressor/Cargo.toml @@ -26,9 +26,5 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-utils = { workspace = true } -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [lints] workspace = true diff --git a/vortex-cuda/Cargo.toml b/vortex-cuda/Cargo.toml index 9d21976f92b..c57528e3231 100644 --- a/vortex-cuda/Cargo.toml +++ b/vortex-cuda/Cargo.toml @@ -42,14 +42,6 @@ vortex-cuda-macros = { workspace = true } vortex-error = { workspace = true, features = ["object_store"] } vortex-nvcomp = { path = "nvcomp" } -[dev-dependencies] -criterion = { package = "codspeed-criterion-compat-walltime", version = "4.3.0" } -futures = { workspace = true, features = ["executor"] } -rstest = { workspace = true } -tokio = { workspace = true, features = ["rt", "macros"] } -vortex-array = { workspace = true, features = ["_test-harness"] } -vortex-cuda = { path = ".", features = ["_test-harness"] } - [build-dependencies] bindgen = { workspace = true } fastlanes = { workspace = true } diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index 2215bd4db8e..ee02a06028a 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -37,14 +37,5 @@ tracing = { workspace = true, features = ["std", "attributes"] } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } vortex-utils = { workspace = true, features = ["dashmap"] } -[dev-dependencies] -anyhow = { workspace = true } -datafusion = { workspace = true } -insta = { workspace = true } -rstest = { workspace = true } -tempfile = { workspace = true } -tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "fs"] } -url = { workspace = true } - [lints] workspace = true diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml index 2d095c1c3e5..9b17172d87d 100644 --- a/vortex-duckdb/Cargo.toml +++ b/vortex-duckdb/Cargo.toml @@ -40,15 +40,6 @@ url = { workspace = true } vortex = { workspace = true, features = ["files", "tokio", "object_store"] } vortex-utils = { workspace = true, features = ["dashmap"] } -[dev-dependencies] -anyhow = { workspace = true } -jiff = { workspace = true } -rstest = { workspace = true } -tempfile = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } -vortex-runend = { workspace = true } -vortex-sequence = { workspace = true } - [lints] workspace = true diff --git a/vortex-error/Cargo.toml b/vortex-error/Cargo.toml index 2856a2724ae..7e106689603 100644 --- a/vortex-error/Cargo.toml +++ b/vortex-error/Cargo.toml @@ -27,9 +27,5 @@ object_store = { workspace = true, optional = true } prost = { workspace = true } tokio = { workspace = true, features = ["rt"], optional = true } -[dev-dependencies] -serial_test = { version = "3.3.1" } -temp-env = "0.3" - [lints] workspace = true diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index 27f2c7b6140..6378d8fbe59 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -32,10 +32,6 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true, features = [] } vortex = { workspace = true, features = ["object_store"] } -[dev-dependencies] -tempfile = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - [build-dependencies] cbindgen = { workspace = true } diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index d568328bb52..45042990436 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -58,13 +58,6 @@ vortex-utils = { workspace = true, features = ["dashmap"] } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } -[dev-dependencies] -tokio = { workspace = true, features = ["full"] } -vortex-array = { workspace = true, features = ["_test-harness"] } -vortex-io = { workspace = true, features = ["tokio"] } -vortex-layout = { workspace = true, features = ["_test-harness"] } -vortex-scan = { workspace = true } - [lints] workspace = true diff --git a/vortex-file/src/segments/source.rs b/vortex-file/src/segments/source.rs index ba2fbc6fb51..e51e2ceacc1 100644 --- a/vortex-file/src/segments/source.rs +++ b/vortex-file/src/segments/source.rs @@ -47,7 +47,10 @@ pub enum ReadEvent { fn apply_ranges(buffer: BufferHandle, ranges: &[Range]) -> VortexResult { match ranges { [] => buffer.filter(&[]), - [range] => Ok(buffer.slice(range.clone())), + [range] if range.start.is_multiple_of(*buffer.alignment()) => { + Ok(buffer.slice(range.clone())) + } + [range] => buffer.filter(std::slice::from_ref(range)), _ => buffer.filter(ranges), } } @@ -180,7 +183,9 @@ impl FileSegmentSource { impl SegmentSource for FileSegmentSource { fn segment_len(&self, id: SegmentId) -> Option { - self.segments.get(*id as usize).map(|spec| spec.length as usize) + self.segments + .get(*id as usize) + .map(|spec| spec.length as usize) } fn request(&self, id: SegmentId) -> SegmentFuture { @@ -377,7 +382,9 @@ impl BufferSegmentSource { impl SegmentSource for BufferSegmentSource { fn segment_len(&self, id: SegmentId) -> Option { - self.segments.get(*id as usize).map(|spec| spec.length as usize) + self.segments + .get(*id as usize) + .map(|spec| spec.length as usize) } fn request(&self, id: SegmentId) -> SegmentFuture { diff --git a/vortex-io/Cargo.toml b/vortex-io/Cargo.toml index a73baec63c6..2ab06d1ba67 100644 --- a/vortex-io/Cargo.toml +++ b/vortex-io/Cargo.toml @@ -54,13 +54,6 @@ smol = { workspace = true } [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] wasm-bindgen-futures = { workspace = true } -[dev-dependencies] -anyhow = { workspace = true } -itertools = { workspace = true } -rstest = { workspace = true } -tempfile = { workspace = true } -tokio = { workspace = true, features = ["full"] } - [features] object_store = ["dep:object_store", "vortex-error/object_store"] tokio = ["tokio/fs", "tokio/rt-multi-thread"] diff --git a/vortex-ipc/Cargo.toml b/vortex-ipc/Cargo.toml index eda6b7bc852..5d044aedb05 100644 --- a/vortex-ipc/Cargo.toml +++ b/vortex-ipc/Cargo.toml @@ -25,9 +25,5 @@ vortex-error = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["ipc"] } vortex-session = { workspace = true } -[dev-dependencies] -tokio = { workspace = true, features = ["full"] } -vortex-array = { workspace = true, features = ["_test-harness"] } - [lints] workspace = true diff --git a/vortex-jni/Cargo.toml b/vortex-jni/Cargo.toml index d6901514b49..f07dbad79f1 100644 --- a/vortex-jni/Cargo.toml +++ b/vortex-jni/Cargo.toml @@ -32,9 +32,6 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true } vortex = { workspace = true, features = ["object_store", "files", "tokio"] } -[dev-dependencies] -jni = { version = "0.21.1", features = ["invocation"] } - [lib] crate-type = ["staticlib", "cdylib"] diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml index 8bbccf09b91..a9371448a76 100644 --- a/vortex-layout/Cargo.toml +++ b/vortex-layout/Cargo.toml @@ -52,14 +52,6 @@ vortex-sequence = { workspace = true } vortex-session = { workspace = true } vortex-utils = { workspace = true, features = ["dashmap"] } -[dev-dependencies] -futures = { workspace = true, features = ["executor"] } -rstest = { workspace = true } -tokio = { workspace = true, features = ["rt", "macros"] } -vortex-array = { path = "../vortex-array", features = ["_test-harness"] } -vortex-io = { path = "../vortex-io", features = ["tokio"] } -vortex-utils = { workspace = true, features = ["_test-harness"] } - [features] _test-harness = [] tokio = ["dep:tokio", "vortex-error/tokio"] diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs index dcc35152531..8f67ac1621b 100644 --- a/vortex-layout/src/buffer.rs +++ b/vortex-layout/src/buffer.rs @@ -18,8 +18,8 @@ use vortex_array::buffer::DeviceBuffer; use vortex_array::serde::ArrayParts; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; -use vortex_error::vortex_err; use vortex_error::VortexResult; +use vortex_error::vortex_err; use crate::segments::SegmentId; use crate::segments::SegmentSource; @@ -332,8 +332,9 @@ pub fn create_lazy_array_parts( let buffer_len = fb_buf.length() as usize; let alignment = Alignment::from_exponent(fb_buf.alignment_exponent()); - let lazy = LazyBufferHandle::new(Arc::clone(&source), segment_id, segment_len, alignment) - .slice(offset..offset + buffer_len); + let lazy = + LazyBufferHandle::new(Arc::clone(&source), segment_id, segment_len, alignment) + .slice(offset..offset + buffer_len); offset += buffer_len; BufferHandle::new_device(Arc::new(lazy)) @@ -377,7 +378,7 @@ pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { } // 3. Materialize lazy buffers, ensuring proper alignment. - let materialized = try_join_all(handles.iter().cloned().map(|handle| async move { + let materialized = try_join_all(handles.iter().map(|handle| async move { if let Some(lazy) = handle .as_device_opt() .and_then(|d| d.as_any().downcast_ref::()) @@ -386,7 +387,7 @@ pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { let buf = lazy.materialize().await?; buf.ensure_aligned(lazy.alignment()) } else { - Ok(handle) + Ok(handle.clone()) } })) .await?; diff --git a/vortex-layout/src/segments/shared.rs b/vortex-layout/src/segments/shared.rs index bd676c1c009..35f688c53a2 100644 --- a/vortex-layout/src/segments/shared.rs +++ b/vortex-layout/src/segments/shared.rs @@ -88,7 +88,7 @@ impl SegmentSource for SharedSegmentSource { Entry::Vacant(e) => { let future = self .inner - .request_ranges(id, ranges.clone()) + .request_ranges(id, ranges) .map_err(Arc::new) .boxed() .shared(); diff --git a/vortex-layout/src/segments/source.rs b/vortex-layout/src/segments/source.rs index b28abc99ff8..dd1cbb7c6c5 100644 --- a/vortex-layout/src/segments/source.rs +++ b/vortex-layout/src/segments/source.rs @@ -19,7 +19,10 @@ pub(crate) fn apply_ranges( ) -> VortexResult { match ranges { [] => buffer.filter(&[]), - [range] => Ok(buffer.slice(range.clone())), + [range] if range.start.is_multiple_of(*buffer.alignment()) => { + Ok(buffer.slice(range.clone())) + } + [range] => buffer.filter(std::slice::from_ref(range)), _ => buffer.filter(ranges), } } diff --git a/vortex-mask/Cargo.toml b/vortex-mask/Cargo.toml index 037ffdb0d9d..96512b438cd 100644 --- a/vortex-mask/Cargo.toml +++ b/vortex-mask/Cargo.toml @@ -27,10 +27,6 @@ serde = { workspace = true, optional = true, features = ["rc"] } vortex-buffer = { workspace = true, features = ["arrow"] } vortex-error = { workspace = true } -[dev-dependencies] -divan = { workspace = true } -rstest = { workspace = true } - [[bench]] name = "intersect_by_rank" harness = false diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index 6f4fe4511af..a2eb9acdaf0 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -22,6 +22,3 @@ vortex = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } prost = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index d8dc89882b0..57b183dd0d4 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -48,20 +48,6 @@ vortex-utils = { workspace = true } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } -[dev-dependencies] -anyhow = { workspace = true } -arrow-array = { workspace = true } -divan = { workspace = true } -fastlanes = { workspace = true } -mimalloc = { workspace = true } -parquet = { workspace = true } -rand = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true, features = ["full"] } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } -vortex = { path = ".", features = ["tokio"] } - [features] default = ["files", "zstd"] files = ["dep:vortex-file"] From 5753fc1f6ca0eb63595c3cbd9f11132cbe2db17b Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 17:30:45 -0400 Subject: [PATCH 09/10] Follow up Signed-off-by: Nicholas Gates --- Cargo.lock | 909 ++++++++++++++++++++++-- encodings/alp/Cargo.toml | 6 + encodings/bytebool/Cargo.toml | 4 + encodings/datetime-parts/Cargo.toml | 5 + encodings/decimal-byte-parts/Cargo.toml | 4 + encodings/fastlanes/Cargo.toml | 9 + encodings/fsst/Cargo.toml | 6 + encodings/parquet-variant/Cargo.toml | 4 + encodings/pco/Cargo.toml | 4 + encodings/runend/Cargo.toml | 9 + encodings/sequence/Cargo.toml | 5 + encodings/sparse/Cargo.toml | 5 + encodings/zigzag/Cargo.toml | 4 + encodings/zstd/Cargo.toml | 5 + vortex-array/Cargo.toml | 11 + vortex-btrblocks/Cargo.toml | 6 + vortex-buffer/Cargo.toml | 5 + vortex-compressor/Cargo.toml | 4 + vortex-cuda/Cargo.toml | 8 + vortex-datafusion/Cargo.toml | 9 + vortex-duckdb/Cargo.toml | 9 + vortex-error/Cargo.toml | 4 + vortex-ffi/Cargo.toml | 4 + vortex-file/Cargo.toml | 7 + vortex-io/Cargo.toml | 7 + vortex-ipc/Cargo.toml | 4 + vortex-jni/Cargo.toml | 3 + vortex-layout/Cargo.toml | 8 + vortex-mask/Cargo.toml | 4 + vortex/Cargo.toml | 14 + 30 files changed, 1042 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9264cbd373..e0d6ac23021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,17 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -72,6 +83,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.21" @@ -308,7 +325,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-buffer 57.3.0", "arrow-data 57.3.0", "arrow-schema 57.3.0", @@ -327,7 +344,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-buffer 58.0.0", "arrow-data 58.0.0", "arrow-schema 58.0.0", @@ -624,7 +641,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array 57.3.0", "arrow-buffer 57.3.0", "arrow-data 57.3.0", @@ -638,7 +655,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array 58.0.0", "arrow-buffer 58.0.0", "arrow-data 58.0.0", @@ -885,6 +902,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "base64" version = "0.22.1" @@ -1063,6 +1102,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "brotli" version = "8.0.2" @@ -1107,6 +1170,40 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" +[[package]] +name = "byte-unit" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6d47a4e2961fb8721bcfc54feae6455f2f64e7054f9bc67e875f0e77f4c58d" +dependencies = [ + "rust_decimal", + "schemars 1.2.1", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytecount" version = "0.6.9" @@ -1173,6 +1270,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -1281,6 +1384,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1299,7 +1429,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.9", ] [[package]] @@ -1322,6 +1452,7 @@ dependencies = [ "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] @@ -1342,6 +1473,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -1353,12 +1493,109 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "codspeed" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b684e94583e85a5ca7e1a6454a89d76a5121240f2fb67eb564129d9bafdb9db0" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.2.17", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96389aaa4bbb872ea4924dc0335b2bb181bcf28d6eedbe8fea29afcc5bde36a6" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "codspeed", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "codspeed-divan-compat" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e4bf8c7793c170fd0fcf3be97b9032b2ae39c2b9e8818aba3cc10ca0f0c6c0" +dependencies = [ + "clap", + "codspeed", + "codspeed-divan-compat-macros", + "codspeed-divan-compat-walltime", + "regex", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78aae02f2a278588e16e8ca62ea1915b8ab30f8230a09926671bba19ede801a4" +dependencies = [ + "divan-macros", + "itertools 0.14.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "codspeed-divan-compat-walltime" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ffd32c0c59ab8b674b15be65ba7c59aebac047036cfa7fa1e11bc2c178b81f" +dependencies = [ + "cfg-if", + "clap", + "codspeed", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -1445,6 +1682,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "console" version = "0.15.11" @@ -1593,6 +1836,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1708,7 +1961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3aa12038120eb13347a6ae2ffab1d34efe78150125108627fd85044dd4d6ff1e" dependencies = [ "half", - "libloading", + "libloading 0.8.9", ] [[package]] @@ -2068,7 +2321,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "arrow-ipc 57.3.0", "chrono", @@ -2090,7 +2343,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" dependencies = [ - "ahash", + "ahash 0.8.12", "apache-avro", "arrow 58.0.0", "arrow-ipc 58.0.0", @@ -2580,7 +2833,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-doc 51.0.0", @@ -2601,7 +2854,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-doc 53.0.0", @@ -2623,7 +2876,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr-common 51.0.0", @@ -2636,7 +2889,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-expr-common 53.0.0", @@ -2846,7 +3099,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr 51.0.0", @@ -2868,7 +3121,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 58.0.0", "datafusion-common 53.0.0", "datafusion-expr 53.0.0", @@ -2922,7 +3175,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "datafusion-common 51.0.0", "datafusion-expr-common 51.0.0", @@ -2936,7 +3189,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 58.0.0", "chrono", "datafusion-common 53.0.0", @@ -2990,7 +3243,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 57.3.0", "arrow-ord 57.3.0", "arrow-schema 57.3.0", @@ -3021,7 +3274,7 @@ version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow 58.0.0", "arrow-ord 58.0.0", "arrow-schema 58.0.0", @@ -3319,6 +3572,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "document-features" version = "0.2.12" @@ -3355,6 +3619,12 @@ dependencies = [ "vortex-duckdb", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -3685,6 +3955,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsst" version = "2.0.1" @@ -4125,13 +4401,22 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.12", "allocator-api2", ] @@ -4573,6 +4858,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "insta" +version = "1.47.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +dependencies = [ + "console 0.16.3", + "once_cell", + "similar", + "tempfile", +] + [[package]] name = "instability" version = "0.3.12" @@ -4617,12 +4914,32 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -4665,6 +4982,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "java-locator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f" +dependencies = [ + "glob", +] + [[package]] name = "jiff" version = "0.2.23" @@ -4715,7 +5041,9 @@ dependencies = [ "cesu8", "cfg-if", "combine", + "java-locator", "jni-sys 0.3.1", + "libloading 0.7.4", "log", "thiserror 1.0.69", "walkdir", @@ -5251,7 +5579,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2acdba67f84190067532fce07b51a435dd390d7cdc1129a05003e5cb3274cf0" dependencies = [ - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_repr", @@ -5419,6 +5747,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libloading" version = "0.8.9" @@ -5832,6 +6170,18 @@ version = "6.6.666" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6" +[[package]] +name = "nix" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6123,7 +6473,7 @@ dependencies = [ "percent-encoding", "quick-xml", "rand 0.9.2", - "reqwest", + "reqwest 0.12.28", "ring", "rustls-pki-types", "serde", @@ -6165,6 +6515,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -6195,7 +6551,7 @@ dependencies = [ "bytes", "http", "opentelemetry", - "reqwest", + "reqwest 0.12.28", ] [[package]] @@ -6210,7 +6566,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost 0.14.3", - "reqwest", + "reqwest 0.12.28", "thiserror 2.0.18", "tracing", ] @@ -6356,7 +6712,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array 57.3.0", "arrow-buffer 57.3.0", "arrow-cast 57.3.0", @@ -6390,7 +6746,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array 58.0.0", "arrow-buffer 58.0.0", "arrow-data 58.0.0", @@ -6616,6 +6972,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ping" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" +dependencies = [ + "rand 0.9.2", + "socket2", + "thiserror 2.0.18", +] + [[package]] name = "piper" version = "0.2.5" @@ -6640,17 +7007,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] -name = "polling" -version = "3.11.0" +name = "plotters" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix 1.1.4", - "windows-sys 0.61.2", + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", ] [[package]] @@ -6734,6 +7129,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -6827,6 +7244,26 @@ dependencies = [ "cc", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "public-api" version = "0.51.0" @@ -7001,6 +7438,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -7201,7 +7639,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" dependencies = [ - "ahash", + "ahash 0.8.12", "brotli", "paste", "rand 0.9.2", @@ -7368,6 +7806,26 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "regex" version = "1.12.3" @@ -7419,6 +7877,15 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -7466,6 +7933,46 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -7480,6 +7987,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "roaring" version = "0.10.12" @@ -7567,6 +8103,23 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -7630,6 +8183,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -7660,12 +8214,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -7692,6 +8274,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.29" @@ -7713,6 +8304,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars_derive" version = "0.8.22" @@ -7743,6 +8346,18 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "3.7.0" @@ -7866,6 +8481,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_test" +version = "1.0.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" +dependencies = [ + "serde", +] + [[package]] name = "serde_tokenstream" version = "0.2.3" @@ -7903,6 +8527,32 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "sha1" version = "0.10.6" @@ -8223,6 +8873,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "std_prelude" version = "0.2.12" @@ -8330,7 +8990,7 @@ dependencies = [ "prost-build", "prost-types", "regress", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", @@ -8632,6 +9292,15 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "temp-env" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" +dependencies = [ + "parking_lot", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -8654,12 +9323,43 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "termtree" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d1330fe7f7f872cd05165130b10602d667b205fd85be09be2814b115d4ced9" +[[package]] +name = "test-with" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27838d769fa9bf364bf4a352ec88862b6e6cb96a8e6705cc78fbb16ff26ee8b9" +dependencies = [ + "byte-unit", + "chrono", + "num_cpus", + "ping", + "proc-macro-error2", + "proc-macro2", + "quote", + "regex", + "reqwest 0.13.2", + "syn 2.0.117", + "sysinfo", + "uzers", + "which", +] + [[package]] name = "testing_table" version = "0.3.0" @@ -8801,6 +9501,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" @@ -9174,7 +9884,7 @@ dependencies = [ "proc-macro2", "quote", "regress", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", @@ -9191,7 +9901,7 @@ checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" dependencies = [ "proc-macro2", "quote", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", @@ -9296,6 +10006,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -9320,6 +10036,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uzers" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d283dc7e8c901e79e32d077866eaf599156cbf427fffa8289aecc52c5c3f63" +dependencies = [ + "libc", + "log", +] + [[package]] name = "valuable" version = "0.1.1" @@ -9336,6 +10062,18 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vortex" version = "0.1.0" dependencies = [ + "anyhow", + "arrow-array 58.0.0", + "codspeed-divan-compat", + "fastlanes", + "mimalloc", + "parquet 58.0.0", + "rand 0.10.0", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "vortex", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -9370,9 +10108,12 @@ dependencies = [ name = "vortex-alp" version = "0.1.0" dependencies = [ + "codspeed-divan-compat", "itertools 0.14.0", "num-traits", "prost 0.14.3", + "rand 0.10.0", + "rstest", "rustc-hash", "vortex-array", "vortex-buffer", @@ -9401,6 +10142,7 @@ dependencies = [ "async-lock", "bytes", "cfg-if", + "codspeed-divan-compat", "cudarc", "enum-iterator", "flatbuffers", @@ -9409,6 +10151,7 @@ dependencies = [ "goldenfile", "half", "humansize", + "insta", "inventory", "itertools 0.14.0", "jiff", @@ -9421,16 +10164,20 @@ dependencies = [ "primitive-types", "prost 0.14.3", "rand 0.10.0", + "rand_distr 0.6.0", "rstest", "rstest_reuse", "rustc-hash", "serde", + "serde_json", + "serde_test", "simdutf8", "static_assertions", "tabled", "termtree", "tracing", "uuid", + "vortex-array", "vortex-buffer", "vortex-error", "vortex-flatbuffers", @@ -9465,7 +10212,7 @@ dependencies = [ "parquet 58.0.0", "rand 0.10.0", "regex", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sysinfo", @@ -9488,12 +10235,15 @@ dependencies = [ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ + "codspeed-divan-compat", "getrandom 0.4.2", "itertools 0.14.0", "num-traits", "pco", "rand 0.10.0", + "rstest", "rustc-hash", + "test-with", "tracing", "vortex-alp", "vortex-array", @@ -9521,8 +10271,11 @@ dependencies = [ "arrow-buffer 58.0.0", "bitvec", "bytes", + "codspeed-divan-compat", "itertools 0.14.0", "memmap2", + "num-traits", + "rstest", "serde", "simdutf8", "tracing", @@ -9534,6 +10287,7 @@ name = "vortex-bytebool" version = "0.1.0" dependencies = [ "num-traits", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -9551,7 +10305,7 @@ dependencies = [ "clap", "futures", "parquet 58.0.0", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sha2", @@ -9574,6 +10328,7 @@ dependencies = [ "num-traits", "parking_lot", "rand 0.10.0", + "rstest", "rustc-hash", "tracing", "vortex-array", @@ -9588,7 +10343,7 @@ name = "vortex-cub" version = "0.1.0" dependencies = [ "bindgen", - "libloading", + "libloading 0.8.9", "paste", "vortex-array", "vortex-cuda-macros", @@ -9602,6 +10357,7 @@ dependencies = [ "async-trait", "bindgen", "bytes", + "codspeed-criterion-compat-walltime", "cudarc", "fastlanes", "futures", @@ -9610,11 +10366,13 @@ dependencies = [ "object_store 0.13.1", "parking_lot", "prost 0.14.3", + "rstest", "tokio", "tracing", "vortex", "vortex-array", "vortex-cub", + "vortex-cuda", "vortex-cuda-macros", "vortex-error", "vortex-nvcomp", @@ -9647,8 +10405,10 @@ dependencies = [ name = "vortex-datafusion" version = "0.1.0" dependencies = [ + "anyhow", "arrow-schema 58.0.0", "async-trait", + "datafusion 53.0.0", "datafusion-catalog 53.0.0", "datafusion-common 53.0.0", "datafusion-common-runtime 53.0.0", @@ -9662,11 +10422,15 @@ dependencies = [ "datafusion-physical-plan 53.0.0", "datafusion-pruning 53.0.0", "futures", + "insta", "itertools 0.14.0", "object_store 0.13.1", + "rstest", + "tempfile", "tokio", "tokio-stream", "tracing", + "url", "vortex", "vortex-utils", ] @@ -9677,6 +10441,7 @@ version = "0.1.0" dependencies = [ "num-traits", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -9690,6 +10455,7 @@ version = "0.1.0" dependencies = [ "num-traits", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -9701,6 +10467,7 @@ dependencies = [ name = "vortex-duckdb" version = "0.1.0" dependencies = [ + "anyhow", "async-fs", "async-trait", "bindgen", @@ -9710,15 +10477,21 @@ dependencies = [ "custom-labels", "futures", "itertools 0.14.0", + "jiff", "kanal", "num-traits", "object_store 0.13.1", "parking_lot", "paste", - "reqwest", + "reqwest 0.12.28", + "rstest", + "tempfile", "tracing", "url", "vortex", + "vortex-array", + "vortex-runend", + "vortex-sequence", "vortex-utils", "zip", ] @@ -9732,6 +10505,8 @@ dependencies = [ "jiff", "object_store 0.13.1", "prost 0.14.3", + "serial_test", + "temp-env", "tokio", ] @@ -9740,15 +10515,19 @@ name = "vortex-fastlanes" version = "0.1.0" dependencies = [ "arrayref", + "codspeed-divan-compat", "fastlanes", "itertools 0.14.0", "lending-iterator", "num-traits", "prost 0.14.3", "rand 0.10.0", + "rstest", + "vortex-alp", "vortex-array", "vortex-buffer", "vortex-error", + "vortex-fastlanes", "vortex-mask", "vortex-session", ] @@ -9765,10 +10544,12 @@ dependencies = [ "object_store 0.13.1", "paste", "prost 0.14.3", + "tempfile", "tracing", "tracing-subscriber", "url", "vortex", + "vortex-array", ] [[package]] @@ -9830,9 +10611,11 @@ dependencies = [ name = "vortex-fsst" version = "0.1.0" dependencies = [ + "codspeed-divan-compat", "fsst-rs", "prost 0.14.3", "rand 0.10.0", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -9870,6 +10653,7 @@ dependencies = [ name = "vortex-io" version = "0.1.0" dependencies = [ + "anyhow", "async-fs", "async-stream", "async-trait", @@ -9879,12 +10663,15 @@ dependencies = [ "getrandom 0.4.2", "glob", "handle", + "itertools 0.14.0", "kanal", "object_store 0.13.1", "oneshot 0.2.1", "parking_lot", "pin-project-lite", + "rstest", "smol", + "tempfile", "tokio", "tracing", "vortex-array", @@ -9904,6 +10691,7 @@ dependencies = [ "futures", "itertools 0.14.0", "pin-project-lite", + "tokio", "vortex-array", "vortex-buffer", "vortex-error", @@ -9952,6 +10740,7 @@ dependencies = [ "paste", "pin-project-lite", "prost 0.14.3", + "rstest", "rustc-hash", "sketches-ddsketch 0.4.0", "termtree", @@ -9976,7 +10765,9 @@ name = "vortex-mask" version = "0.1.0" dependencies = [ "arrow-buffer 58.0.0", + "codspeed-divan-compat", "itertools 0.14.0", + "rstest", "serde", "vortex-buffer", "vortex-error", @@ -9996,9 +10787,9 @@ name = "vortex-nvcomp" version = "0.1.0" dependencies = [ "bindgen", - "libloading", + "libloading 0.8.9", "liblzma", - "reqwest", + "reqwest 0.12.28", "tar", "vortex-cuda-macros", ] @@ -10014,6 +10805,7 @@ dependencies = [ "parquet-variant", "parquet-variant-compute", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10028,6 +10820,7 @@ version = "0.1.0" dependencies = [ "pco", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10072,9 +10865,13 @@ version = "0.1.0" dependencies = [ "arbitrary", "arrow-array 58.0.0", + "arrow-schema 58.0.0", + "codspeed-divan-compat", "itertools 0.14.0", "num-traits", "prost 0.14.3", + "rand 0.10.0", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10101,8 +10898,10 @@ dependencies = [ name = "vortex-sequence" version = "0.1.0" dependencies = [ + "itertools 0.14.0", "num-traits", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10129,6 +10928,7 @@ dependencies = [ "itertools 0.14.0", "num-traits", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10247,6 +11047,7 @@ dependencies = [ name = "vortex-zigzag" version = "0.1.0" dependencies = [ + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10259,8 +11060,10 @@ dependencies = [ name = "vortex-zstd" version = "0.1.0" dependencies = [ + "codspeed-divan-compat", "itertools 0.14.0", "prost 0.14.3", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", @@ -10438,6 +11241,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -10447,6 +11259,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +dependencies = [ + "libc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/encodings/alp/Cargo.toml b/encodings/alp/Cargo.toml index 7d1c457fed0..13ccc67d63b 100644 --- a/encodings/alp/Cargo.toml +++ b/encodings/alp/Cargo.toml @@ -29,6 +29,12 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } vortex-utils = { workspace = true } +[dev-dependencies] +divan = { workspace = true } +rand = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [[bench]] name = "alp_compress" harness = false diff --git a/encodings/bytebool/Cargo.toml b/encodings/bytebool/Cargo.toml index 05cf5b82ce2..bca2ed71aa9 100644 --- a/encodings/bytebool/Cargo.toml +++ b/encodings/bytebool/Cargo.toml @@ -23,3 +23,7 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/datetime-parts/Cargo.toml b/encodings/datetime-parts/Cargo.toml index 6f7742f773d..52894fc7c07 100644 --- a/encodings/datetime-parts/Cargo.toml +++ b/encodings/datetime-parts/Cargo.toml @@ -24,3 +24,8 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-error = { workspace = true } diff --git a/encodings/decimal-byte-parts/Cargo.toml b/encodings/decimal-byte-parts/Cargo.toml index f60b5590445..4934ec4fa27 100644 --- a/encodings/decimal-byte-parts/Cargo.toml +++ b/encodings/decimal-byte-parts/Cargo.toml @@ -24,3 +24,7 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { path = "../../vortex-array", features = ["_test-harness"] } diff --git a/encodings/fastlanes/Cargo.toml b/encodings/fastlanes/Cargo.toml index e043555f2fa..ac30b2032b4 100644 --- a/encodings/fastlanes/Cargo.toml +++ b/encodings/fastlanes/Cargo.toml @@ -30,6 +30,15 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } +[dev-dependencies] +divan = { workspace = true } +itertools = { workspace = true } +rand = { workspace = true } +rstest = { workspace = true } +vortex-alp = { path = "../alp" } +vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-fastlanes = { path = ".", features = ["_test-harness"] } + [features] _test-harness = ["dep:rand"] diff --git a/encodings/fsst/Cargo.toml b/encodings/fsst/Cargo.toml index 5ac799a8813..b95eeb1f444 100644 --- a/encodings/fsst/Cargo.toml +++ b/encodings/fsst/Cargo.toml @@ -29,6 +29,12 @@ vortex-session = { workspace = true } [features] _test-harness = ["dep:rand", "vortex-array/_test-harness"] +[dev-dependencies] +divan = { workspace = true } +rand = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [[bench]] name = "fsst_compress" harness = false diff --git a/encodings/parquet-variant/Cargo.toml b/encodings/parquet-variant/Cargo.toml index 9f6ec204d4a..cc554c85d10 100644 --- a/encodings/parquet-variant/Cargo.toml +++ b/encodings/parquet-variant/Cargo.toml @@ -32,5 +32,9 @@ vortex-mask = { workspace = true } vortex-proto = { workspace = true } vortex-session = { workspace = true } +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [package.metadata.cargo-machete] ignored = ["getrandom_v03"] diff --git a/encodings/pco/Cargo.toml b/encodings/pco/Cargo.toml index 278a15f40e1..bc222bf1dcc 100644 --- a/encodings/pco/Cargo.toml +++ b/encodings/pco/Cargo.toml @@ -25,5 +25,9 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [package.metadata.cargo-machete] ignored = ["getrandom_v03"] diff --git a/encodings/runend/Cargo.toml b/encodings/runend/Cargo.toml index 4b709802530..01a5b8d7a3e 100644 --- a/encodings/runend/Cargo.toml +++ b/encodings/runend/Cargo.toml @@ -28,6 +28,15 @@ vortex-session = { workspace = true } [lints] workspace = true +[dev-dependencies] +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +divan = { workspace = true } +itertools = { workspace = true } +rand = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [features] arbitrary = ["dep:arbitrary", "vortex-array/arbitrary"] arrow = ["dep:arrow-array"] diff --git a/encodings/sequence/Cargo.toml b/encodings/sequence/Cargo.toml index 1ef12c62df7..06f695cb989 100644 --- a/encodings/sequence/Cargo.toml +++ b/encodings/sequence/Cargo.toml @@ -23,5 +23,10 @@ vortex-mask = { workspace = true } vortex-proto = { workspace = true } vortex-session = { workspace = true } +[dev-dependencies] +itertools = { workspace = true } +rstest = { workspace = true } +vortex-array = { path = "../../vortex-array", features = ["_test-harness"] } + [lints] workspace = true diff --git a/encodings/sparse/Cargo.toml b/encodings/sparse/Cargo.toml index 3c1f34868cb..53a25ad535a 100644 --- a/encodings/sparse/Cargo.toml +++ b/encodings/sparse/Cargo.toml @@ -25,3 +25,8 @@ vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } + +[dev-dependencies] +itertools = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/zigzag/Cargo.toml b/encodings/zigzag/Cargo.toml index 4f21527c915..83e6d324ed1 100644 --- a/encodings/zigzag/Cargo.toml +++ b/encodings/zigzag/Cargo.toml @@ -21,5 +21,9 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } zigzag = { workspace = true } +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [lints] workspace = true diff --git a/encodings/zstd/Cargo.toml b/encodings/zstd/Cargo.toml index 4cfab63e2fb..4383f765a2f 100644 --- a/encodings/zstd/Cargo.toml +++ b/encodings/zstd/Cargo.toml @@ -30,6 +30,11 @@ vortex-mask = { workspace = true } vortex-session = { workspace = true } zstd = { workspace = true } +[dev-dependencies] +divan = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [[bench]] name = "listview_rebuild" harness = false diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index d00e89db1b7..dbdfd5a2cd9 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -84,6 +84,17 @@ table-display = ["dep:tabled"] _test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"] serde = ["dep:serde", "vortex-buffer/serde", "vortex-mask/serde"] +[dev-dependencies] +arrow-cast = { workspace = true } +divan = { workspace = true } +futures = { workspace = true, features = ["executor"] } +insta = { workspace = true } +rand_distr = { workspace = true } +rstest = { workspace = true } +serde_json = { workspace = true } +serde_test = { workspace = true } +vortex-array = { path = ".", features = ["_test-harness", "table-display"] } + [[bench]] name = "search_sorted" harness = false diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 83629cbd2f0..9bbd2430f09 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -39,6 +39,12 @@ vortex-utils = { workspace = true } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } +[dev-dependencies] +divan = { workspace = true } +rstest = { workspace = true } +test-with = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [features] # This feature enabled unstable encodings for which we don't guarantee stability. unstable_encodings = ["vortex-zstd?/unstable_encodings"] diff --git a/vortex-buffer/Cargo.toml b/vortex-buffer/Cargo.toml index 8cdf9d724e2..ae9d7e6cc05 100644 --- a/vortex-buffer/Cargo.toml +++ b/vortex-buffer/Cargo.toml @@ -36,6 +36,11 @@ vortex-error = { workspace = true } [lints] workspace = true +[dev-dependencies] +divan = { workspace = true } +num-traits = { workspace = true } +rstest = { workspace = true } + [[bench]] name = "vortex_buffer" harness = false diff --git a/vortex-compressor/Cargo.toml b/vortex-compressor/Cargo.toml index f5fa3fcc86e..260c9c531f5 100644 --- a/vortex-compressor/Cargo.toml +++ b/vortex-compressor/Cargo.toml @@ -26,5 +26,9 @@ vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-utils = { workspace = true } +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [lints] workspace = true diff --git a/vortex-cuda/Cargo.toml b/vortex-cuda/Cargo.toml index c57528e3231..9d21976f92b 100644 --- a/vortex-cuda/Cargo.toml +++ b/vortex-cuda/Cargo.toml @@ -42,6 +42,14 @@ vortex-cuda-macros = { workspace = true } vortex-error = { workspace = true, features = ["object_store"] } vortex-nvcomp = { path = "nvcomp" } +[dev-dependencies] +criterion = { package = "codspeed-criterion-compat-walltime", version = "4.3.0" } +futures = { workspace = true, features = ["executor"] } +rstest = { workspace = true } +tokio = { workspace = true, features = ["rt", "macros"] } +vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-cuda = { path = ".", features = ["_test-harness"] } + [build-dependencies] bindgen = { workspace = true } fastlanes = { workspace = true } diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index ee02a06028a..2215bd4db8e 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -37,5 +37,14 @@ tracing = { workspace = true, features = ["std", "attributes"] } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } vortex-utils = { workspace = true, features = ["dashmap"] } +[dev-dependencies] +anyhow = { workspace = true } +datafusion = { workspace = true } +insta = { workspace = true } +rstest = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "fs"] } +url = { workspace = true } + [lints] workspace = true diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml index 9b17172d87d..2d095c1c3e5 100644 --- a/vortex-duckdb/Cargo.toml +++ b/vortex-duckdb/Cargo.toml @@ -40,6 +40,15 @@ url = { workspace = true } vortex = { workspace = true, features = ["files", "tokio", "object_store"] } vortex-utils = { workspace = true, features = ["dashmap"] } +[dev-dependencies] +anyhow = { workspace = true } +jiff = { workspace = true } +rstest = { workspace = true } +tempfile = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-runend = { workspace = true } +vortex-sequence = { workspace = true } + [lints] workspace = true diff --git a/vortex-error/Cargo.toml b/vortex-error/Cargo.toml index 7e106689603..2856a2724ae 100644 --- a/vortex-error/Cargo.toml +++ b/vortex-error/Cargo.toml @@ -27,5 +27,9 @@ object_store = { workspace = true, optional = true } prost = { workspace = true } tokio = { workspace = true, features = ["rt"], optional = true } +[dev-dependencies] +serial_test = { version = "3.3.1" } +temp-env = "0.3" + [lints] workspace = true diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index 6378d8fbe59..27f2c7b6140 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -32,6 +32,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true, features = [] } vortex = { workspace = true, features = ["object_store"] } +[dev-dependencies] +tempfile = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + [build-dependencies] cbindgen = { workspace = true } diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 465c6c9004f..e812ab228c6 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -59,6 +59,13 @@ vortex-utils = { workspace = true, features = ["dashmap"] } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } +[dev-dependencies] +tokio = { workspace = true, features = ["full"] } +vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-io = { workspace = true, features = ["tokio"] } +vortex-layout = { workspace = true, features = ["_test-harness"] } +vortex-scan = { workspace = true } + [lints] workspace = true diff --git a/vortex-io/Cargo.toml b/vortex-io/Cargo.toml index 2ab06d1ba67..a73baec63c6 100644 --- a/vortex-io/Cargo.toml +++ b/vortex-io/Cargo.toml @@ -54,6 +54,13 @@ smol = { workspace = true } [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] wasm-bindgen-futures = { workspace = true } +[dev-dependencies] +anyhow = { workspace = true } +itertools = { workspace = true } +rstest = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["full"] } + [features] object_store = ["dep:object_store", "vortex-error/object_store"] tokio = ["tokio/fs", "tokio/rt-multi-thread"] diff --git a/vortex-ipc/Cargo.toml b/vortex-ipc/Cargo.toml index 5d044aedb05..eda6b7bc852 100644 --- a/vortex-ipc/Cargo.toml +++ b/vortex-ipc/Cargo.toml @@ -25,5 +25,9 @@ vortex-error = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["ipc"] } vortex-session = { workspace = true } +[dev-dependencies] +tokio = { workspace = true, features = ["full"] } +vortex-array = { workspace = true, features = ["_test-harness"] } + [lints] workspace = true diff --git a/vortex-jni/Cargo.toml b/vortex-jni/Cargo.toml index f07dbad79f1..d6901514b49 100644 --- a/vortex-jni/Cargo.toml +++ b/vortex-jni/Cargo.toml @@ -32,6 +32,9 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true } vortex = { workspace = true, features = ["object_store", "files", "tokio"] } +[dev-dependencies] +jni = { version = "0.21.1", features = ["invocation"] } + [lib] crate-type = ["staticlib", "cdylib"] diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml index a9371448a76..8bbccf09b91 100644 --- a/vortex-layout/Cargo.toml +++ b/vortex-layout/Cargo.toml @@ -52,6 +52,14 @@ vortex-sequence = { workspace = true } vortex-session = { workspace = true } vortex-utils = { workspace = true, features = ["dashmap"] } +[dev-dependencies] +futures = { workspace = true, features = ["executor"] } +rstest = { workspace = true } +tokio = { workspace = true, features = ["rt", "macros"] } +vortex-array = { path = "../vortex-array", features = ["_test-harness"] } +vortex-io = { path = "../vortex-io", features = ["tokio"] } +vortex-utils = { workspace = true, features = ["_test-harness"] } + [features] _test-harness = [] tokio = ["dep:tokio", "vortex-error/tokio"] diff --git a/vortex-mask/Cargo.toml b/vortex-mask/Cargo.toml index 96512b438cd..037ffdb0d9d 100644 --- a/vortex-mask/Cargo.toml +++ b/vortex-mask/Cargo.toml @@ -27,6 +27,10 @@ serde = { workspace = true, optional = true, features = ["rc"] } vortex-buffer = { workspace = true, features = ["arrow"] } vortex-error = { workspace = true } +[dev-dependencies] +divan = { workspace = true } +rstest = { workspace = true } + [[bench]] name = "intersect_by_rank" harness = false diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 7b449b8a17a..896ec139251 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -49,6 +49,20 @@ vortex-utils = { workspace = true } vortex-zigzag = { workspace = true } vortex-zstd = { workspace = true, optional = true } +[dev-dependencies] +anyhow = { workspace = true } +arrow-array = { workspace = true } +divan = { workspace = true } +fastlanes = { workspace = true } +mimalloc = { workspace = true } +parquet = { workspace = true } +rand = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +vortex = { path = ".", features = ["tokio"] } + [features] default = ["files", "zstd"] files = ["dep:vortex-file"] From f7c78b1dde4fd93c9366daf0623b6957f8b38c9f Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 18:03:45 -0400 Subject: [PATCH 10/10] Follow up Signed-off-by: Nicholas Gates --- vortex-layout/src/buffer.rs | 24 ++++++++++++------------ vortex-layout/src/segments/cache.rs | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs index 8f67ac1621b..07f9eca1993 100644 --- a/vortex-layout/src/buffer.rs +++ b/vortex-layout/src/buffer.rs @@ -472,11 +472,12 @@ fn validate_filter_ranges(ranges: &[Range], len: usize) { #[cfg(test)] mod tests { + use std::iter; use std::ops::Range; use std::sync::Arc; - use std::sync::Mutex; use futures::FutureExt; + use parking_lot::Mutex; use vortex_array::buffer::BufferHandle; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; @@ -489,10 +490,14 @@ mod tests { use crate::segments::SegmentSource; use crate::segments::apply_ranges; + type RangeRequest = Vec>; + type RangeRequestLog = Vec; + type SharedRangeRequestLog = Arc>; + /// A trivial in-memory segment source for tests. struct SingleSegment { buffer: BufferHandle, - ranged_requests: Arc>>>>, + ranged_requests: SharedRangeRequestLog, } impl SegmentSource for SingleSegment { @@ -506,10 +511,7 @@ mod tests { } fn request_ranges(&self, _id: SegmentId, ranges: Vec>) -> SegmentFuture { - self.ranged_requests - .lock() - .expect("range request log poisoned") - .push(ranges.clone()); + self.ranged_requests.lock().push(ranges.clone()); let handle = self.buffer.clone(); async move { apply_ranges(handle, &ranges) }.boxed() } @@ -519,7 +521,7 @@ mod tests { lazy_with_requests(data).0 } - fn lazy_with_requests(data: &[u8]) -> (LazyBufferHandle, Arc>>>>) { + fn lazy_with_requests(data: &[u8]) -> (LazyBufferHandle, SharedRangeRequestLog) { let buf = BufferHandle::new_host(ByteBuffer::copy_from(data)); let ranged_requests = Arc::new(Mutex::new(Vec::new())); ( @@ -656,13 +658,11 @@ mod tests { block_on(|_| async { let (lazy, ranged_requests) = lazy_with_requests(&[1, 2, 3, 4, 5, 6]); let handle = lazy.slice(1..5).materialize().await?; + let expected_ranges: RangeRequest = iter::once(1..5).collect(); assert_eq!(handle.unwrap_host().as_slice(), &[2, 3, 4, 5]); assert_eq!( - ranged_requests - .lock() - .expect("range request log poisoned") - .as_slice(), - &[vec![1..5]] + ranged_requests.lock().as_slice(), + std::slice::from_ref(&expected_ranges) ); Ok(()) }) diff --git a/vortex-layout/src/segments/cache.rs b/vortex-layout/src/segments/cache.rs index 078136e116c..02807249412 100644 --- a/vortex-layout/src/segments/cache.rs +++ b/vortex-layout/src/segments/cache.rs @@ -237,8 +237,9 @@ mod tests { let full = adapter.request(SegmentId::from(0)).await.unwrap(); assert_eq!(full.unwrap_host().as_slice(), &[1, 2, 3, 4]); + let requested_ranges = std::iter::once(1..3).collect(); let ranges = adapter - .request_ranges(SegmentId::from(0), vec![1..3]) + .request_ranges(SegmentId::from(0), requested_ranges) .await .unwrap(); assert_eq!(ranges.unwrap_host().as_slice(), &[2, 3]);