Skip to content

wip: chunked iter exe #7263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joseph-isaacs wants to merge 2 commits into
base: develop
Choose a base branch
from
Draft

wip: chunked iter exe #7263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b80f796
wip: chunked iter exe
joseph-isaacs Apr 2, 2026
7d9fc3d
wip: chunked iter exe
joseph-isaacs Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 32 additions & 8 deletions vortex-array/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3364,7 +3364,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:

pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -3416,6 +3416,10 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner

impl vortex_array::arrays::patched::PatchedArray

pub fn vortex_array::arrays::patched::PatchedArray::into_parts(self) -> vortex_array::arrays::patched::PatchedArrayParts

impl vortex_array::arrays::patched::PatchedArray

pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef

impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray
Expand Down Expand Up @@ -3444,6 +3448,22 @@ impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray

pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef

pub struct vortex_array::arrays::patched::PatchedArrayParts

pub vortex_array::arrays::patched::PatchedArrayParts::indices: vortex_array::ArrayRef

pub vortex_array::arrays::patched::PatchedArrayParts::inner: vortex_array::ArrayRef

pub vortex_array::arrays::patched::PatchedArrayParts::lane_offsets: vortex_array::ArrayRef

pub vortex_array::arrays::patched::PatchedArrayParts::len: usize

pub vortex_array::arrays::patched::PatchedArrayParts::n_lanes: usize

pub vortex_array::arrays::patched::PatchedArrayParts::offset: usize

pub vortex_array::arrays::patched::PatchedArrayParts::values: vortex_array::ArrayRef

pub struct vortex_array::arrays::patched::PatchedMetadata

impl core::clone::Clone for vortex_array::arrays::patched::PatchedMetadata
Expand Down Expand Up @@ -4148,7 +4168,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v

pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -7332,7 +7352,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:

pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -7384,6 +7404,10 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner

impl vortex_array::arrays::patched::PatchedArray

pub fn vortex_array::arrays::patched::PatchedArray::into_parts(self) -> vortex_array::arrays::patched::PatchedArrayParts

impl vortex_array::arrays::patched::PatchedArray

pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef

impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray
Expand Down Expand Up @@ -7942,7 +7966,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v

pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -22324,7 +22348,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:

pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -22444,7 +22468,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v

pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -23768,7 +23792,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:

pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down Expand Up @@ -23888,7 +23912,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v

pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType

pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>

pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>

Expand Down
24 changes: 13 additions & 11 deletions vortex-array/src/arrays/chunked/vtable/canonical.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ use crate::Canonical;
use crate::DynArray;
use crate::ExecutionCtx;
use crate::IntoArray;
use crate::arrays::ListView;
use crate::arrays::ListViewArray;
use crate::arrays::PrimitiveArray;
use crate::arrays::Struct;
use crate::arrays::StructArray;
use crate::arrays::chunked::vtable::ChunkedArray;
use crate::arrays::listview::ListViewRebuildMode;
Expand Down Expand Up @@ -41,7 +43,6 @@ pub(super) fn _canonicalize(
&owned_chunks,
Validity::copy_from_array(&array.clone().into_array())?,
struct_dtype,
ctx,
)?;
Canonical::Struct(struct_array)
}
Expand All @@ -62,24 +63,22 @@ pub(super) fn _canonicalize(
/// Packs many [`StructArray`]s to instead be a single [`StructArray`], where the [`DynArray`] for each
/// field is a [`ChunkedArray`].
///
/// The caller guarantees there are at least 2 chunks.
/// The caller guarantees there are at least 2 chunks, and that all chunks are already
/// canonicalized to [`StructArray`] by iterative execution.
fn pack_struct_chunks(
chunks: &[ArrayRef],
validity: Validity,
struct_dtype: &StructFields,
ctx: &mut ExecutionCtx,
) -> VortexResult<StructArray> {
let len = chunks.iter().map(|chunk| chunk.len()).sum();
let mut field_arrays = Vec::new();

let executed_chunks: Vec<StructArray> = chunks
.iter()
.map(|c| c.clone().execute::<StructArray>(ctx))
.collect::<VortexResult<_>>()?;

for (field_idx, field_dtype) in struct_dtype.fields().enumerate() {
let mut field_chunks = Vec::with_capacity(chunks.len());
for struct_array in &executed_chunks {
for chunk in chunks {
let struct_array = chunk
.as_opt::<Struct>()
.vortex_expect("struct chunk pre-canonicalized by iterative execution");
let field = struct_array.unmasked_field(field_idx).to_array();
field_chunks.push(field);
}
Expand All @@ -99,7 +98,8 @@ fn pack_struct_chunks(
///
/// We use the existing arrays (chunks) to form a chunked array of `elements` (the child array).
///
/// The caller guarantees there are at least 2 chunks.
/// The caller guarantees there are at least 2 chunks, and that all chunks are already
/// canonicalized to [`ListViewArray`] by iterative execution.
fn swizzle_list_chunks(
chunks: &[ArrayRef],
validity: Validity,
Expand Down Expand Up @@ -131,7 +131,9 @@ fn swizzle_list_chunks(
let mut sizes = BufferMut::<u64>::with_capacity(len);

for chunk in chunks {
let chunk_array = chunk.clone().execute::<ListViewArray>(ctx)?;
let chunk_array = chunk
.as_opt::<ListView>()
.vortex_expect("list chunk pre-canonicalized by iterative execution");
// By rebuilding as zero-copy to `List` and trimming all elements (to prevent gaps), we make
// the final output `ListView` also zero-copyable to `List`.
let chunk_array = chunk_array.rebuild(ListViewRebuildMode::MakeExact)?;
Expand Down
11 changes: 11 additions & 0 deletions vortex-array/src/arrays/chunked/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use vortex_error::vortex_err;
use vortex_error::vortex_panic;
use vortex_session::VortexSession;

use crate::AnyCanonical;
use crate::ArrayRef;
use crate::Canonical;
use crate::EmptyMetadata;
Expand Down Expand Up @@ -202,6 +203,16 @@ impl VTable for Chunked {
}

fn execute(array: Arc<Array<Self>>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
// Iteratively request execution of each chunk until it reaches canonical form.
// This gives the scheduler visibility into child execution and enables
// cross-step optimizations between chunk decoding steps.
for i in 0..array.nchunks() {
if !array.chunk(i).is::<AnyCanonical>() {
return Ok(ExecutionResult::execute_slot::<AnyCanonical>(array, i + 1));
}
}

// All chunks are now canonical — combine them.
Ok(ExecutionResult::done(
_canonicalize(&array, ctx)?.into_array(),
))
Expand Down
13 changes: 10 additions & 3 deletions vortex-array/src/arrays/filter/vtable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ use vortex_error::vortex_panic;
use vortex_mask::Mask;
use vortex_session::VortexSession;

use crate::AnyCanonical;
use crate::ArrayEq;
use crate::ArrayHash;
use crate::ArrayRef;
use crate::DynArray;
use crate::IntoArray;
use crate::Precision;
use crate::arrays::filter::array::CHILD_SLOT;
use crate::arrays::filter::array::FilterArray;
use crate::arrays::filter::array::NUM_SLOTS;
use crate::arrays::filter::array::SLOT_NAMES;
Expand All @@ -30,6 +32,7 @@ use crate::buffer::BufferHandle;
use crate::dtype::DType;
use crate::executor::ExecutionCtx;
use crate::executor::ExecutionResult;
use crate::require_child;
use crate::scalar::Scalar;
use crate::serde::ArrayChildren;
use crate::stats::StatsSetRef;
Expand Down Expand Up @@ -151,14 +154,18 @@ impl VTable for Filter {
if let Some(canonical) = execute_filter_fast_paths(&array, ctx)? {
return Ok(ExecutionResult::done(canonical));
}

let array = require_child!(array, array.child(), CHILD_SLOT => AnyCanonical);

let Mask::Values(mask_values) = &array.mask else {
unreachable!("`execute_filter_fast_paths` handles AllTrue and AllFalse")
};

// We rely on the optimization pass that runs prior to this execution for filter pushdown,
// so now we can just execute the filter without worrying.
// Child is pre-canonicalized — apply the filter directly.
debug_assert!(array.child().is_canonical());
let child = array.child().to_canonical()?;
Ok(ExecutionResult::done(
execute_filter(array.child().clone().execute(ctx)?, mask_values).into_array(),
execute_filter(child, mask_values).into_array(),
))
}

Expand Down
4 changes: 4 additions & 0 deletions vortex-array/src/arrays/list/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ use crate::IntoArray;
use crate::Precision;
use crate::ProstMetadata;
use crate::arrays::ListArray;
use crate::arrays::Primitive;
use crate::arrays::list::array::NUM_SLOTS;
use crate::arrays::list::array::OFFSETS_SLOT;
use crate::arrays::list::array::SLOT_NAMES;
use crate::arrays::list::array::VALIDITY_SLOT;
use crate::arrays::list::compute::PARENT_KERNELS;
Expand All @@ -32,6 +34,7 @@ use crate::hash::ArrayEq;
use crate::hash::ArrayHash;
use crate::metadata::DeserializeMetadata;
use crate::metadata::SerializeMetadata;
use crate::require_child;
use crate::serde::ArrayChildren;
use crate::stats::StatsSetRef;
use crate::validity::Validity;
Expand Down Expand Up @@ -193,6 +196,7 @@ impl VTable for List {
}

fn execute(array: Arc<Array<Self>>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
let array = require_child!(array, array.offsets(), OFFSETS_SLOT => Primitive);
Ok(ExecutionResult::done(
list_view_from_list(ListArray::clone(&array), ctx)?.into_array(),
))
Expand Down
9 changes: 7 additions & 2 deletions vortex-array/src/arrays/masked/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@ use vortex_error::vortex_ensure;
use vortex_error::vortex_panic;
use vortex_session::VortexSession;

use crate::AnyCanonical;
use crate::ArrayRef;
use crate::Canonical;
use crate::EmptyMetadata;
use crate::IntoArray;
use crate::Precision;
use crate::arrays::ConstantArray;
use crate::arrays::MaskedArray;
use crate::arrays::masked::array::CHILD_SLOT;
use crate::arrays::masked::array::NUM_SLOTS;
use crate::arrays::masked::array::SLOT_NAMES;
use crate::arrays::masked::compute::rules::PARENT_RULES;
Expand All @@ -30,6 +31,7 @@ use crate::executor::ExecutionCtx;
use crate::executor::ExecutionResult;
use crate::hash::ArrayEq;
use crate::hash::ArrayHash;
use crate::require_child;
use crate::scalar::Scalar;
use crate::serde::ArrayChildren;
use crate::stats::StatsSetRef;
Expand Down Expand Up @@ -162,7 +164,10 @@ impl VTable for Masked {
// While we could manually convert the dtype, `mask_validity_canonical` is already O(1) for
// `AllTrue` masks (no data copying), so there's no benefit.

let child = array.child().clone().execute::<Canonical>(ctx)?;
let array = require_child!(array, array.child(), CHILD_SLOT => AnyCanonical);

debug_assert!(array.child().is_canonical());
let child = array.child().to_canonical()?;
Ok(ExecutionResult::done(
mask_validity_canonical(child, &validity_mask, ctx)?.into_array(),
))
Expand Down
41 changes: 41 additions & 0 deletions vortex-array/src/arrays/patched/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,47 @@ impl PatchedArray {
}
}

/// The owned parts of a [`PatchedArray`], produced by [`PatchedArray::into_parts`].
pub struct PatchedArrayParts {
/// The inner array being patched.
pub inner: ArrayRef,
/// The lane offsets array (u32).
pub lane_offsets: ArrayRef,
/// The patch indices array (u16).
pub indices: ArrayRef,
/// The patch values array.
pub values: ArrayRef,
/// Number of lanes.
pub n_lanes: usize,
/// Offset into the first chunk.
pub offset: usize,
/// Logical length.
pub len: usize,
}

impl PatchedArray {
/// Consume this array into its owned parts.
pub fn into_parts(mut self) -> PatchedArrayParts {
PatchedArrayParts {
inner: self.slots[INNER_SLOT]
.take()
.vortex_expect("PatchedArray inner slot"),
lane_offsets: self.slots[LANE_OFFSETS_SLOT]
.take()
.vortex_expect("PatchedArray lane_offsets slot"),
indices: self.slots[INDICES_SLOT]
.take()
.vortex_expect("PatchedArray indices slot"),
values: self.slots[VALUES_SLOT]
.take()
.vortex_expect("PatchedArray values slot"),
n_lanes: self.n_lanes,
offset: self.offset,
len: self.len,
}
}
}

impl PatchedArray {
/// Returns a reference to the base array being patched.
#[inline]
Expand Down
Loading
Loading