From 799e9081fa1f166f4da0ce62490363efa8c650f6 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 18 Aug 2026 08:29:47 +0200 Subject: [PATCH 1/6] Add ImportTensorVersioned --- cpp/src/arrow/c/dlpack.cc | 223 +++++++++++++++++++++ cpp/src/arrow/c/dlpack.h | 29 +++ cpp/src/arrow/c/dlpack_test.cc | 349 +++++++++++++++++++++++++++++++++ cpp/src/arrow/tensor.cc | 36 +++- cpp/src/arrow/tensor.h | 9 + 5 files changed, 641 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 4e25d50bb56..521248b06c1 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -18,22 +18,32 @@ #include "arrow/c/dlpack.h" #include +#include #include +#include #include #include #include "arrow/array/array_base.h" +#include "arrow/array/util.h" #include "arrow/buffer.h" #include "arrow/c/dlpack_abi.h" #include "arrow/device.h" #include "arrow/tensor.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/logging_internal.h" +#include "arrow/util/small_vector.h" namespace arrow::dlpack { namespace { +/*************** + * Producers * + ***************/ + Result GetDLDataType(const DataType& type) { auto dtype = DLDataType{}; dtype.lanes = 1; @@ -248,4 +258,217 @@ Result ExportDevice(const std::shared_ptr& t) { return ExportDeviceImpl(t); } +/*************** + * Consumers * + ***************/ + +namespace { + +class CppDLTensor { + public: + using value_type = DLManagedTensorVersioned; + using pointer_type = value_type*; + + static Result TakeOwnership(pointer_type ptr) { + if (ptr == nullptr) { + return Status::Invalid("Received null pointer."); + } + // Create the wrapper before checking the version as the spec mandates that the + // deleter MUST be called on version major mismatch. + auto out = CppDLTensor(ptr); + if (out.ptr_->version.major != DLPACK_MAJOR_VERSION) { + return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, + ", expected ", DLPACK_MAJOR_VERSION); + } + return out; + } + + const DLTensor& tensor() const { return ptr_->dl_tensor; } + + int64_t ndim() const { + DCHECK_GE(tensor().ndim, 0); + return tensor().ndim; + } + + template + T* data_as() { + return static_cast(tensor().data); + } + + std::span shape() const { + return {tensor().shape, static_cast(ndim())}; + } + + std::span strides() const { + return {tensor().strides, static_cast(ndim())}; + } + + bool flag_is_set(uint8_t bits) const { return (ptr_->flags & bits) == bits; } + + bool is_readonly() const { return flag_is_set(DLPACK_FLAG_BITMASK_READ_ONLY); } + + int32_t byte_width() const { return tensor().dtype.bits / 8; } + + private: + struct Deleter { + void operator()(pointer_type ptr) { + // Null is valid in DLPack spec + if (auto del = ptr->deleter) { + del(ptr); + } + } + }; + + /// Make a safe wrapper that will delete the resource in case of exception. + std::unique_ptr ptr_; + + explicit CppDLTensor(pointer_type ptr) : ptr_(ptr) {} +}; + +Result> DataTypeFromDLPack(DLDataType dtype) { + if (dtype.lanes != 1) { + return Status::TypeError("Only type with one lane are supported."); + } + + auto constexpr as_fw = [](auto dt) { + return std::static_pointer_cast(std::move(dt)); + }; + + switch (dtype.code) { + case kDLInt: { + switch (dtype.bits) { + case 8: + return as_fw(int8()); + case 16: + return as_fw(int16()); + case 32: + return as_fw(int32()); + case 64: + return as_fw(int64()); + default: + return Status::Invalid("unsupported integer bit width ", + static_cast(dtype.bits)); + } + } + case kDLUInt: { + switch (dtype.bits) { + case 8: + return as_fw(uint8()); + case 16: + return as_fw(uint16()); + case 32: + return as_fw(uint32()); + case 64: + return as_fw(uint64()); + default: + return Status::Invalid("unsupported unsigned integer bit width ", + static_cast(dtype.bits)); + } + } + case kDLFloat: { + switch (dtype.bits) { + case 16: + return as_fw(float16()); + case 32: + return as_fw(float32()); + case 64: + return as_fw(float64()); + default: + return Status::Invalid("unsupported float bit width ", + static_cast(dtype.bits)); + } + } + default: { + return Status::Invalid("unsupported DLPack type ", static_cast(dtype.code)); + } + } +} + +inline std::vector StridesInBytes(std::span strides, + int64_t byte_width) { + std::vector out{}; + out.reserve(strides.size()); + for (const auto& s : strides) { + out.push_back(s * byte_width); + } + return out; +} + +Result> ImportBuffer(CppDLTensor&& dl, bool copy) { + // DLPack strides are in number of elements, so is the size we compute from them. + ARROW_ASSIGN_OR_RAISE(const auto nelements, + internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); + const auto nbytes = nelements * dl.byte_width(); + // DLPack mandates a null data pointer when the tensor holds no element, so there is + // neither anything to share nor to copy. + uint8_t* data = + (nbytes == 0) ? nullptr : dl.data_as() + dl.tensor().byte_offset; + + std::shared_ptr buffer = nullptr; + if (nbytes == 0) { + // DLPack data pointer may be null on empty tensors + buffer = std::make_shared(data, nbytes); + } else if (copy) { + ARROW_ASSIGN_OR_RAISE(buffer, MutableBuffer::CopyNonOwned( + {data, nbytes}, default_cpu_memory_manager())); + } else { + const bool readonly = dl.is_readonly(); + // Trick to keep DLPack data alive taken from `Buffer::FromVector`. + auto deleter = [dl = std::move(dl)](auto* buffer) { delete buffer; }; + if (readonly) { + buffer = {new Buffer{data, nbytes}, std::move(deleter)}; + } else { + buffer = std::shared_ptr{ + new MutableBuffer{data, nbytes}, + std::move(deleter), + }; + } + } + + return buffer; +} + +} // namespace + +Result> ImportArrayVersioned(DLManagedTensorVersioned* unmanaged, + bool copy) { + ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); + + if (dl.tensor().device.device_type != kDLCPU) { + return Status::NotImplemented( + "DLPack support is implemented only for buffers on CPU device."); + } + + if (dl.ndim() != 1 || dl.strides().front() != 1) { + return Status::NotImplemented( + "Only contiguous one dimensional tensor can be imported as arrays." + " Try importing to Tensor first."); + } + + ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); + const auto nelements = dl.shape().front(); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + auto data = ArrayData::Make(type, nelements, {nullptr, std::move(buffer)}); + return MakeArray(std::move(data)); +} + +Result> ImportTensorVersioned(DLManagedTensorVersioned* unmanaged, + bool copy) { + ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); + + if (dl.tensor().device.device_type != kDLCPU) { + return Status::NotImplemented( + "DLPack support is implemented only for buffers on CPU device."); + } + + ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); + auto shape = std::vector(dl.shape().begin(), dl.shape().end()); + auto strides = std::vector(dl.strides().begin(), dl.strides().end()); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + const auto byte_width = type->byte_width(); + + return Tensor::Make(std::move(type), std::move(buffer), std::move(shape), + StridesInBytes(std::move(strides), byte_width)); +} + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 8a9084f36c7..152dd8dbbad 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -105,4 +105,33 @@ Result ExportDevice(const std::shared_ptr& arr); ARROW_EXPORT Result ExportDevice(const std::shared_ptr& t); +/// \brief Import a DLPack tensor as an Arrow Array. +/// +/// Same restrictions on data types as `ExportArrayVersioned`, and only row-major +/// tensors are supported. Dimensions beyond the first are imported as nested +/// fixed size lists. +/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// +/// \param[in] raw DLPack tensor +/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack +/// producer. +/// \return An Arrow Array +ARROW_EXPORT +Result> ImportArrayVersioned(DLManagedTensorVersioned* raw, + bool copy); + +/// \brief Import a DLPack tensor as an Arrow Tensor. +/// +/// Same restrictions on data types as `ExportTensorVersioned`. +/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// If the DLPack input is marked as readonly, this will produce an immutable tensor. +/// +/// \param[in] raw Arrow array +/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack +/// producer. +/// \return An Arrow Tensor +ARROW_EXPORT +Result> ImportTensorVersioned(DLManagedTensorVersioned* raw, + bool copy); + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 05de22237a9..6f320e24b25 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -329,4 +329,353 @@ TYPED_TEST(TestExportTensor, TestTensorStrided) { f_dlpack_strides); } +/*************** + * Consumers * + ***************/ + +/// A DLPack tensor as a foreign library would produce it. +struct ForeignTensor { + DLDataType dtype = {.code = kDLFloat, .bits = 32, .lanes = 1}; + std::vector shape = {}; + /// In number of elements, as mandated by DLPack. + std::vector strides = {}; + std::vector data = {}; + DLDevice device = {.device_type = kDLCPU, .device_id = 0}; + uint64_t byte_offset = 0; + uint64_t flags = 0; + /// Incremented when the consumer releases the tensor. + std::shared_ptr deleted = std::make_shared(0); + + DLManagedTensorVersioned managed = {}; +}; + +template +std::vector ToBytes(const std::vector& values) { + std::vector bytes(values.size() * sizeof(T)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + return bytes; +} + +/// Hand out a DLPack tensor owning ``foreign``, releasing it through its deleter. +DLManagedTensorVersioned* Produce(ForeignTensor foreign) { + auto owned = std::make_unique(std::move(foreign)); + owned->managed = { + .version = {.major = DLPACK_MAJOR_VERSION, .minor = DLPACK_MINOR_VERSION}, + .manager_ctx = owned.get(), + .deleter = + [](DLManagedTensorVersioned* self) { + auto* ctx = static_cast(self->manager_ctx); + ++(*ctx->deleted); + delete ctx; + }, + .flags = owned->flags, + .dl_tensor = + { + .data = owned->data.data(), + .device = owned->device, + .ndim = static_cast(owned->shape.size()), + .dtype = owned->dtype, + .shape = owned->shape.data(), + .strides = owned->strides.data(), + .byte_offset = owned->byte_offset, + }, + }; + return &owned.release()->managed; +} + +template +struct TensorConsumer { + using Imported = std::shared_ptr; + static constexpr bool copy = kCopy; + static constexpr const char* name = copy ? "TensorCopied" : "TensorShared"; + + static Result Import(DLManagedTensorVersioned* raw) { + return ImportTensorVersioned(raw, copy); + } + static std::shared_ptr ValueType(const Imported& t) { return t->type(); } + static const uint8_t* RawData(const Imported& t) { return t->raw_data(); } + static bool IsMutable(const Imported& t) { return t->is_mutable(); } + static int64_t Size(const Imported& t) { return t->size(); } +}; + +template +struct ArrayConsumer { + using Imported = std::shared_ptr; + static constexpr bool copy = kCopy; + static constexpr const char* name = copy ? "ArrayCopied" : "ArrayShared"; + + static Result Import(DLManagedTensorVersioned* raw) { + return ImportArrayVersioned(raw, copy); + } + static std::shared_ptr ValueType(const Imported& arr) { return arr->type(); } + static const uint8_t* RawData(const Imported& arr) { + return arr->data()->buffers[1]->data() + arr->offset() * arr->type()->byte_width(); + } + static bool IsMutable(const Imported& arr) { + return arr->data()->buffers[1]->is_mutable(); + } + static int64_t Size(const Imported& arr) { return arr->length(); } +}; + +struct ConsumerNames { + template + static std::string GetName(int) { + return Consumer::name; + } +}; + +using ConsumerTypes = ::testing::Types, TensorConsumer, + ArrayConsumer, ArrayConsumer>; +using TensorConsumerTypes = ::testing::Types, TensorConsumer>; +using ArrayConsumerTypes = ::testing::Types, ArrayConsumer>; + +/// Tests sharing the same expectations for Arrow Tensor and Array imports. +template +class TestImport : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImport, ConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImport, Basic) { + auto foreign = ForeignTensor{ + .shape = {6}, + .strides = {1}, + .data = ToBytes(std::vector{0, 0, 1, 2, 3, 4, 5, 6}), + .byte_offset = 2 * sizeof(float), + .flags = DLPACK_FLAG_BITMASK_READ_ONLY, + }; + const auto deleted = foreign.deleted; + const auto expected = std::vector{1, 2, 3, 4, 5, 6}; + const auto* values = foreign.data.data() + foreign.byte_offset; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + + AssertTypeEqual(*float32(), *TypeParam::ValueType(imported)); + ASSERT_EQ(6, TypeParam::Size(imported)); + // A copy is ours to mutate, whatever the producer flagged + ASSERT_EQ(TypeParam::copy, TypeParam::IsMutable(imported)); + ASSERT_EQ(0, std::memcmp(TypeParam::RawData(imported), expected.data(), + expected.size() * sizeof(float))); + + if constexpr (TypeParam::copy) { + // The producer tensor is released as soon as its data has been copied + ASSERT_EQ(1, *deleted); + } else { + ASSERT_EQ(TypeParam::RawData(imported), values); + // The producer tensor is kept alive by the imported data + ASSERT_EQ(0, *deleted); + imported.reset(); + ASSERT_EQ(1, *deleted); + } +} + +TYPED_TEST(TestImport, Mutable) { + auto foreign = ForeignTensor{ + .shape = {4}, + .strides = {1}, + .data = ToBytes(std::vector{1, 2, 3, 4}), + .flags = 0, + }; + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_TRUE(TypeParam::IsMutable(imported)); +} + +TYPED_TEST(TestImport, NullDeleter) { + // The DLPack spec allows producers not to set a deleter + auto* managed = + Produce({.shape = {2}, .strides = {1}, .data = std::vector(8)}); + auto* foreign = static_cast(managed->manager_ctx); + managed->deleter = nullptr; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + imported.reset(); + delete foreign; +} + +TYPED_TEST(TestImport, DataTypes) { + const std::vector>> cases = { + {{kDLInt, 8, 1}, int8()}, {{kDLInt, 16, 1}, int16()}, + {{kDLInt, 32, 1}, int32()}, {{kDLInt, 64, 1}, int64()}, + {{kDLUInt, 8, 1}, uint8()}, {{kDLUInt, 16, 1}, uint16()}, + {{kDLUInt, 32, 1}, uint32()}, {{kDLUInt, 64, 1}, uint64()}, + {{kDLFloat, 16, 1}, float16()}, {{kDLFloat, 32, 1}, float32()}, + {{kDLFloat, 64, 1}, float64()}}; + + for (const auto& [dtype, expected] : cases) { + ARROW_SCOPED_TRACE("dtype ", expected->ToString()); + ASSERT_OK_AND_ASSIGN( + auto imported, TypeParam::Import(Produce( + {.dtype = dtype, + .shape = {3}, + .strides = {1}, + .data = std::vector(3 * expected->byte_width())}))); + AssertTypeEqual(*expected, *TypeParam::ValueType(imported)); + } +} + +TYPED_TEST(TestImport, Empty) { + // DLPack mandates a null data pointer when the tensor holds no element + auto* managed = Produce({.shape = {0}, .strides = {1}}); + managed->dl_tensor.data = nullptr; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + ASSERT_EQ(0, TypeParam::Size(imported)); +} + +TYPED_TEST(TestImport, Errors) { + auto check = [](ForeignTensor foreign, const std::string& message) { + const auto deleted = foreign.deleted; + const auto status = TypeParam::Import(Produce(std::move(foreign))).status(); + EXPECT_EQ(message, status.ToStringWithoutContextLines()); + // Ownership is taken even when the import fails + EXPECT_EQ(1, *deleted); + }; + + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Received null pointer.", + TypeParam::Import(nullptr)); + check({.shape = {2}, + .strides = {1}, + .data = std::vector(8), + .device = {.device_type = kDLCUDA, .device_id = 0}}, + "NotImplemented: DLPack support is implemented only for buffers on CPU device."); + check({.dtype = {kDLFloat, 32, 2}, .shape = {2}, .strides = {1}}, + "Type error: Only type with one lane are supported."); + check({.dtype = {kDLInt, 4, 1}, .shape = {2}, .strides = {1}}, + "Invalid: unsupported integer bit width 4"); + check({.dtype = {kDLBool, 8, 1}, .shape = {2}, .strides = {1}}, + "Invalid: unsupported DLPack type " + std::to_string(kDLBool)); +} + +TYPED_TEST(TestImport, UnsupportedVersion) { + auto* managed = + Produce({.shape = {2}, .strides = {1}, .data = std::vector(8)}); + const auto deleted = static_cast(managed->manager_ctx)->deleted; + const auto major = DLPACK_MAJOR_VERSION + 1; + managed->version.major = major; + + ASSERT_RAISES_WITH_MESSAGE(Invalid, + "Invalid: Unsupported DLPack major version " + + std::to_string(major) + ", expected " + + std::to_string(DLPACK_MAJOR_VERSION), + TypeParam::Import(managed)); + // The spec mandates the deleter to be called on major version mismatch + ASSERT_EQ(1, *deleted); +} + +template +class TestImportTensor : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImportTensor, TensorConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImportTensor, ShapeAndStrides) { + auto foreign = ForeignTensor{ + .shape = {2, 3}, + .strides = {3, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), + }; + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(foreign)))); + + ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(2, 3)); + // Arrow strides are in bytes, DLPack strides in elements + ASSERT_THAT(tensor->strides(), + ::testing::ElementsAre(3 * sizeof(float), sizeof(float))); +} + +TYPED_TEST(TestImportTensor, Empty) { + auto* managed = Produce({.shape = {0, 3}, .strides = {3, 1}}); + managed->dl_tensor.data = nullptr; + + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(0, 3)); +} + +TYPED_TEST(TestImportTensor, Strided) { + auto column_major = ForeignTensor{ + .shape = {2, 3}, + .strides = {1, 2}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), + }; + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(column_major)))); + ASSERT_TRUE(tensor->is_column_major()); + + // A 2x2 window over every other row of a 4x2 buffer + auto non_contiguous = ForeignTensor{ + .shape = {2, 2}, + .strides = {4, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6, 7, 8}), + }; + ASSERT_OK_AND_ASSIGN(tensor, TypeParam::Import(Produce(std::move(non_contiguous)))); + ASSERT_FALSE(tensor->is_contiguous()); + ASSERT_EQ(6, tensor->template Value({1, 1})); +} + +TYPED_TEST(TestImportTensor, NegativeStrides) { + auto foreign = ForeignTensor{ + .shape = {2, 2}, + .strides = {-2, 1}, + .data = std::vector(16), + }; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: negative strides not supported", + TypeParam::Import(Produce(std::move(foreign)))); +} + +TYPED_TEST(TestImportTensor, RoundTrip) { + const auto original = TensorFromJSON(float64(), "[1, 2, 3, 4, 5, 6]", {3, 2}); + + ASSERT_OK_AND_ASSIGN(auto* managed, ExportTensorVersioned(original, /*copy=*/false)); + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + + ASSERT_TRUE(tensor->Equals(*original)); + if constexpr (!TypeParam::copy) { + ASSERT_EQ(original->raw_data(), tensor->raw_data()); + } +} + +template +class TestImportArray : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImportArray, ArrayConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImportArray, OneDimension) { + auto foreign = ForeignTensor{ + .dtype = {.code = kDLInt, .bits = 32, .lanes = 1}, + .shape = {4}, + .strides = {1}, + .data = ToBytes(std::vector{1, 2, 3, 4}), + }; + ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(Produce(std::move(foreign)))); + AssertArraysEqual(*ArrayFromJSON(int32(), "[1, 2, 3, 4]"), *array); +} + +TYPED_TEST(TestImportArray, Unsupported) { + auto check = [](ForeignTensor foreign) { + ASSERT_RAISES_WITH_MESSAGE( + NotImplemented, + "NotImplemented: Only contiguous one dimensional tensor can be imported as" + " arrays. Try importing to Tensor first.", + TypeParam::Import(Produce(std::move(foreign)))); + }; + + // Only a Tensor can hold more than one dimension + check({.shape = {2, 3}, + .strides = {3, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6})}); + check({.shape = {0, 3}, .strides = {1, 1}}); + // Array values are contiguous, whatever the dimension count + check( + {.shape = {3}, .strides = {2}, .data = ToBytes(std::vector{1, 2, 3, 4, 5})}); + check({.shape = {2, 2}, .strides = {-2, 1}, .data = std::vector(16)}); +} + +TYPED_TEST(TestImportArray, RoundTrip) { + const auto original = ArrayFromJSON(float64(), "[1, 2, 3, 4, 5, 6]"); + + ASSERT_OK_AND_ASSIGN(auto* managed, ExportArrayVersioned(original, /*copy=*/false)); + ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(managed)); + + AssertArraysEqual(*original, *array); + if constexpr (!TypeParam::copy) { + ASSERT_EQ(TypeParam::RawData(original), TypeParam::RawData(array)); + } +} + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index f2ff11a4f66..d4b8ffef361 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -109,9 +109,35 @@ Status ComputeColumnMajorStrides(const FixedWidthType& type, return Status::OK(); } -} // namespace internal +Result ComputeTensorSize(std::span shape, + std::span strides, int64_t elem_size) { + // Check the largest offset can be computed without overflow + const size_t ndim = shape.size(); + int64_t largest_offset = elem_size; + for (size_t i = 0; i < ndim; ++i) { + if (shape[i] == 0) continue; + if (strides[i] < 0) { + // TODO(mrkn): Support negative strides for sharing views + return Status::Invalid("negative strides not supported"); + } -namespace { + int64_t dim_offset = 0; + if (!internal::MultiplyWithOverflow(shape[i] - 1, strides[i], &dim_offset)) { + if (!internal::AddWithOverflow(largest_offset, dim_offset, &largest_offset)) { + continue; + } + } + + return Status::Invalid( + "offsets computed from shape and strides would not fit in 64-bit integer"); + } + + // A dimension with no element means empty for which the preceding does not apply. + if (std::find(shape.begin(), shape.end(), 0) != shape.end()) { + return 0; + } + return largest_offset; +} inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, const std::vector& shape, @@ -194,7 +220,7 @@ Status CheckTensorStridesValidity(const std::shared_ptr& data, return Status::OK(); } -} // namespace +} // namespace internal namespace internal { @@ -532,11 +558,11 @@ bool Tensor::is_contiguous() const { } bool Tensor::is_row_major() const { - return IsTensorStridesRowMajor(type_, shape_, strides_); + return internal::IsTensorStridesRowMajor(type_, shape_, strides_); } bool Tensor::is_column_major() const { - return IsTensorStridesColumnMajor(type_, shape_, strides_); + return internal::IsTensorStridesColumnMajor(type_, shape_, strides_); } Type::type Tensor::type_id() const { return type_->id(); } diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h index f3270313434..2917905049c 100644 --- a/cpp/src/arrow/tensor.h +++ b/cpp/src/arrow/tensor.h @@ -71,6 +71,15 @@ bool IsTensorStridesContiguous(const std::shared_ptr& type, const std::vector& shape, const std::vector& strides); +/// Compute the size needed to store the tensor with the given strides and shape. +/// +/// If the strides are in number of element, pass `elem_size=1` to compute the buffer size +/// in the number of elements. If the strides are in bytes, pass the element size in byte +/// to `elem_size` and get the result in bytes. +ARROW_EXPORT +Result ComputeTensorSize(std::span shape, + std::span strides, int64_t elem_size); + ARROW_EXPORT Status ValidateTensorParameters(const std::shared_ptr& type, const std::shared_ptr& data, From 130542c79604530457d978275d78183ddf9f020e Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 1 Sep 2026 15:59:01 +0200 Subject: [PATCH 2/6] Bind FixedShapeTensorArray.from_tensor --- python/pyarrow/array.pxi | 26 ++++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 2b2130e992e..893e92469ce 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -4969,6 +4969,32 @@ cdef class FixedShapeTensorArray(ExtensionArray): return self.to_tensor().to_numpy() + @staticmethod + def from_tensor(Tensor tensor not None): + """ + Convert a pyarrow.Tensor to a fixed shape tensor extension array. + + The first dimension of the tensor becomes the length of the fixed shape + tensor array and the remaining dimensions the shape of the individual + tensors. If the tensor provides strides, they are used to determine the + dimension permutation, otherwise row-major layout is assumed. + + Parameters + ---------- + tensor : pyarrow.Tensor + + Returns + ------- + FixedShapeTensorArray + """ + cdef shared_ptr[CFixedShapeTensorArray] c_array + + with nogil: + c_array = GetResultValue( + CFixedShapeTensorArray.FromTensor(tensor.sp_tensor)) + + return pyarrow_wrap_array( c_array) + @staticmethod def from_numpy_ndarray(obj, dim_names=None): """ diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index ffc02ffd79a..5eac72ace8a 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3090,6 +3090,13 @@ cdef extern from "arrow/extension/fixed_shape_tensor.h" namespace "arrow::extens const vector[int64_t] permutation() const vector[c_string] dim_names() + cdef cppclass CFixedShapeTensorArray \ + " arrow::extension::FixedShapeTensorArray"(CExtensionArray): + + @staticmethod + CResult[shared_ptr[CFixedShapeTensorArray]] FromTensor( + const shared_ptr[CTensor]& tensor) + cdef extern from "arrow/extension/opaque.h" namespace "arrow::extension" nogil: cdef cppclass COpaqueType \ From c204283681e473a0b6b6c70965cd8fa8c2d49ae8 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 1 Sep 2026 16:55:58 +0200 Subject: [PATCH 3/6] Fix linkage issue --- cpp/src/arrow/tensor.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index d4b8ffef361..8b3137d66a3 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -139,9 +139,10 @@ Result ComputeTensorSize(std::span shape, return largest_offset; } -inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, - const std::vector& shape, - const std::vector& strides) { +namespace { +bool IsTensorStridesRowMajor(const std::shared_ptr& type, + const std::vector& shape, + const std::vector& strides) { std::vector c_strides; const auto& fw_type = checked_cast(*type); if (internal::ComputeRowMajorStrides(fw_type, shape, &c_strides).ok()) { @@ -151,9 +152,9 @@ inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, } } -inline bool IsTensorStridesColumnMajor(const std::shared_ptr& type, - const std::vector& shape, - const std::vector& strides) { +bool IsTensorStridesColumnMajor(const std::shared_ptr& type, + const std::vector& shape, + const std::vector& strides) { std::vector f_strides; const auto& fw_type = checked_cast(*type); if (internal::ComputeColumnMajorStrides(fw_type, shape, &f_strides).ok()) { @@ -163,9 +164,9 @@ inline bool IsTensorStridesColumnMajor(const std::shared_ptr& type, } } -inline Status CheckTensorValidity(const std::shared_ptr& type, - const std::shared_ptr& data, - const std::vector& shape) { +Status CheckTensorValidity(const std::shared_ptr& type, + const std::shared_ptr& data, + const std::vector& shape) { if (!type) { return Status::Invalid("Null type is supplied"); } @@ -219,7 +220,7 @@ Status CheckTensorStridesValidity(const std::shared_ptr& data, } return Status::OK(); } - +} // namespace } // namespace internal namespace internal { From dad3defffb37d03a31699d39a121b6cfe5b092ce Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 3 Sep 2026 17:28:58 +0200 Subject: [PATCH 4/6] Add from_dlpack --- cpp/src/arrow/c/dlpack.cc | 11 ++-- cpp/src/arrow/c/dlpack.h | 3 ++ python/pyarrow/array.pxi | 49 +++++++++++++++++- python/pyarrow/includes/libarrow.pxd | 13 +++++ python/pyarrow/tensor.pxi | 47 ++++++++++++++++- python/pyarrow/tests/test_dlpack.py | 77 ++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 521248b06c1..a0a45c09ed5 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -38,6 +38,11 @@ namespace arrow::dlpack { +extern const DLPackVersion VERSION = { + .major = DLPACK_MAJOR_VERSION, + .minor = DLPACK_MINOR_VERSION, +}; + namespace { /*************** @@ -118,7 +123,7 @@ DT* ExportBuffer(ExportBufferParams&& p) { // Strides must be non-null when ndim > 0 ctx->tensor.dl_tensor.strides = ctx->strides.data(); if constexpr (std::is_same_v) { - ctx->tensor.version = {.major = DLPACK_MAJOR_VERSION, .minor = DLPACK_MINOR_VERSION}; + ctx->tensor.version = VERSION; ctx->tensor.flags = p.flags; } @@ -276,9 +281,9 @@ class CppDLTensor { // Create the wrapper before checking the version as the spec mandates that the // deleter MUST be called on version major mismatch. auto out = CppDLTensor(ptr); - if (out.ptr_->version.major != DLPACK_MAJOR_VERSION) { + if (out.ptr_->version.major != VERSION.major) { return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, - ", expected ", DLPACK_MAJOR_VERSION); + ", expected ", VERSION.major); } return out; } diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 152dd8dbbad..93b5534c0cc 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -25,6 +25,9 @@ namespace arrow::dlpack { +/// The DLPack version used during compilation. +extern const DLPackVersion VERSION; + /// \brief Export Arrow array as DLPack tensor. /// /// DLMangedTensor is produced as defined by the DLPack protocol, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 893e92469ce..add5d2d3f4d 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -15,7 +15,12 @@ # specific language governing permissions and limitations # under the License. -from cpython.pycapsule cimport PyCapsule_CheckExact, PyCapsule_GetPointer, PyCapsule_New +from cpython.pycapsule cimport ( + PyCapsule_CheckExact, + PyCapsule_GetPointer, + PyCapsule_New, + PyCapsule_SetName, +) from collections.abc import Sequence import os @@ -2269,6 +2274,48 @@ cdef class Array(_PandasConvertible): return pyarrow_wrap_array(array) + def from_dlpack(x, /, *, device=None, copy=None): + """ + Construct an Array from an object implementing the DLPack protocol. + + Parameters + ---------- + x : object + The input object containing array data, following the DLPack + protocol (has a ``__dlpack__`` method). + device : tuple[enum.Enum, int], optional + Designates where the resulting Array should reside, in the + format returned by :meth:`Array.__dlpack_device__`. When None, + the output Array occupies the same device as the source. + Default: None. + copy : bool, optional + Controls duplication behavior. True mandates copying; False + prohibits copying and raises ``BufferError`` if unavoidable; + None duplicates only when necessary. Default: None. + + Returns + ------- + Array + An Array housing the data from the input object, potentially + as a copy or view. + """ + version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) + pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( + pycapsule, "dltensor_versioned") + if ptr == NULL: + raise ValueError( + 'DLPack producer did not produce a "dltensor_versioned" PyCapsule') + # Mark the capsule as consumed so its destructor does not also invoke the deleter. + # ImportArrayVersionedFromDLPack will take ownership even if it errors (calling + # the deleter in that case). + PyCapsule_SetName(pycapsule, "used_dltensor_versioned") + with nogil: + # Copy handled on producer side + result = ImportArrayVersionedFromDLPack(ptr, False) + carray = GetResultValue(result) + return pyarrow_wrap_array(carray) + def __dlpack__(self, stream=None, max_version=None, dl_device=None, copy=None): """ Export a primitive array as a DLPack capsule. diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 5eac72ace8a..5b857b33c08 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1460,6 +1460,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef extern from "arrow/c/dlpack_abi.h" nogil: + ctypedef struct DLPackVersion: + uint32_t major + uint32_t minor + ctypedef enum DLDeviceType: kDLCPU = 1 @@ -1475,6 +1479,8 @@ cdef extern from "arrow/c/dlpack_abi.h" nogil: cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: + const DLPackVersion DLPACK_VERSION" arrow::dlpack::VERSION" + CResult[DLManagedTensor*] ExportArrayToDLPack" arrow::dlpack::ExportArray"( const shared_ptr[CArray]& arr) CResult[DLManagedTensor*] ExportTensorToDLPack" arrow::dlpack::ExportTensor"( @@ -1490,6 +1496,13 @@ cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: CResult[DLDevice] ExportDevice(const shared_ptr[CArray]& arr) CResult[DLDevice] ExportDevice(const shared_ptr[CTensor]& tensor) + CResult[shared_ptr[CArray]] \ + ImportArrayVersionedFromDLPack" arrow::dlpack::ImportArrayVersioned"( + DLManagedTensorVersioned* raw, c_bool copy) + CResult[shared_ptr[CTensor]] \ + ImportTensorVersionedFromDLPack" arrow::dlpack::ImportTensorVersioned"( + DLManagedTensorVersioned* raw, c_bool copy) + cdef extern from "arrow/builder.h" namespace "arrow" nogil: diff --git a/python/pyarrow/tensor.pxi b/python/pyarrow/tensor.pxi index 521ee0c3f44..656da9f260a 100644 --- a/python/pyarrow/tensor.pxi +++ b/python/pyarrow/tensor.pxi @@ -18,6 +18,8 @@ # Avoid name clash with `pa.struct` function import struct as _struct +from cpython.pycapsule cimport PyCapsule_SetName, PyCapsule_GetPointer + cdef class Tensor(_Weakrefable): """ @@ -300,7 +302,50 @@ strides: {self.strides}""" buffer.strides = cp.PyBytes_AsString(self._ssize_t_strides) buffer.suboffsets = NULL - def __dlpack__(self, stream=None, max_version=None, dl_device=None, copy=None): + def from_dlpack(x, /, *, device=None, copy=None): + """ + Construct a Tensor from an object implementing the DLPack protocol. + + Parameters + ---------- + x : object + The input object containing array data, following the DLPack + protocol (has a ``__dlpack__`` method) or the array API's + ``__array_namespace__`` protocol. + device : tuple[enum.Enum, int], optional + Designates where the resulting Tensor should reside, in the + format returned by :meth:`Tensor.__dlpack_device__`. When None, + the output Tensor occupies the same device as the source. + Default: None. + copy : bool, optional + Controls duplication behavior. True mandates copying; False + prohibits copying and raises ``BufferError`` if unavoidable; + None duplicates only when necessary. Default: None. + + Returns + ------- + Tensor + A Tensor housing the data from the input object, potentially + as a copy or view. + """ + version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) + pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( + pycapsule, "dltensor_versioned") + if ptr == NULL: + raise ValueError( + 'DLPack producer did not produce a "dltensor_versioned" PyCapsule') + # Mark the capsule as consumed so its destructor does not also invoke the deleter. + # ImportTensorVersionedFromDLPack will take ownership even if it errors (calling + # the deleter in that case). + PyCapsule_SetName(pycapsule, "used_dltensor_versioned") + with nogil: + # Copy handled on producer side + result = ImportTensorVersionedFromDLPack(ptr, False) + ctensor = GetResultValue(result) + return pyarrow_wrap_tensor(ctensor) + + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): """ Export a Tensor as a DLPack capsule. diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index f9aac892ced..33045373f29 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -339,3 +339,80 @@ def test_dlpack_cuda_not_supported(): with pytest.raises(NotImplementedError, match="DLPack support is implemented " "only for buffers on CPU device."): carr.__dlpack_device__() + + +@check_bytes_allocated +@pytest.mark.parametrize('np_type', + [np.uint8, np.uint16, np.uint32, np.uint64, + np.int8, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64]) +def test_tensor_from_dlpack(np_type): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + # Non-contiguous, strided slice: DLPack carries explicit strides, so this + # should not need a copy on export. + base = np.arange(24, dtype=np_type).reshape((4, 6)) + expected = base[::2, 1::2] + assert not expected.flags['C_CONTIGUOUS'] + tensor = pa.Tensor.from_dlpack(expected) + assert isinstance(tensor, pa.Tensor) + np.testing.assert_array_equal(tensor.to_numpy(), expected, strict=True) + + +@check_bytes_allocated +@pytest.mark.parametrize('np_type', + [np.uint8, np.uint16, np.uint32, np.uint64, + np.int8, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64]) +def test_array_from_dlpack(np_type): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3, 4, 5], dtype=np_type) + arr = pa.Array.from_dlpack(expected) + assert isinstance(arr, pa.Array) + np.testing.assert_array_equal(arr.to_numpy(), expected, strict=True) + + +@check_bytes_allocated +def test_from_dlpack_zero_copy(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3], dtype=np.int64) + tensor = pa.Tensor.from_dlpack(expected) + result = tensor.to_numpy() + expected[0] = 100 + # Zero-copy import: mutating the source is visible through the tensor. + assert result[0] == 100 + + +@check_bytes_allocated +def test_from_dlpack_explicit_copy(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3], dtype=np.int64) + tensor = pa.Tensor.from_dlpack(expected, copy=True) + result = tensor.to_numpy() + expected[0] = 100 + # The data was copied, so mutating the source is not visible. + assert result[0] == 1 + + +def test_from_dlpack_no_dlpack_method(): + with pytest.raises(AttributeError): + pa.Tensor.from_dlpack(object()) + + +@check_bytes_allocated +def test_array_from_dlpack_multi_dim_not_supported(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.arange(6, dtype=np.int32).reshape((2, 3)) + with pytest.raises(pa.ArrowNotImplementedError, + match="Only contiguous one dimensional tensor can be " + "imported as arrays"): + pa.Array.from_dlpack(expected) From 2fe859d5c4b9a82bfa536f146580c6bdcadd8785 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 3 Sep 2026 18:15:53 +0200 Subject: [PATCH 5/6] Add missing export --- cpp/src/arrow/c/dlpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 93b5534c0cc..6b552cf818f 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -26,7 +26,7 @@ namespace arrow::dlpack { /// The DLPack version used during compilation. -extern const DLPackVersion VERSION; +ARROW_EXPORT extern const DLPackVersion VERSION; /// \brief Export Arrow array as DLPack tensor. /// From 496604c64b51d6c6ca12363a233816d3daf122cb Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 4 Sep 2026 09:53:04 +0200 Subject: [PATCH 6/6] Apply review comments --- cpp/src/arrow/c/dlpack.cc | 43 +++++++++++++++++++++-------- cpp/src/arrow/c/dlpack.h | 9 +++--- cpp/src/arrow/c/dlpack_test.cc | 4 +-- python/pyarrow/array.pxi | 3 ++ python/pyarrow/tensor.pxi | 12 ++++++-- python/pyarrow/tests/test_dlpack.py | 7 +++-- 6 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index a0a45c09ed5..1a990ebd9d5 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -18,9 +18,7 @@ #include "arrow/c/dlpack.h" #include -#include #include -#include #include #include @@ -32,9 +30,9 @@ #include "arrow/tensor.h" #include "arrow/type.h" #include "arrow/type_traits.h" -#include "arrow/util/checked_cast.h" +#include "arrow/util/int_util_overflow.h" #include "arrow/util/logging_internal.h" -#include "arrow/util/small_vector.h" +#include "arrow/util/macros.h" namespace arrow::dlpack { @@ -275,16 +273,27 @@ class CppDLTensor { using pointer_type = value_type*; static Result TakeOwnership(pointer_type ptr) { - if (ptr == nullptr) { + if (ARROW_PREDICT_FALSE(ptr == nullptr)) { return Status::Invalid("Received null pointer."); } // Create the wrapper before checking the version as the spec mandates that the // deleter MUST be called on version major mismatch. auto out = CppDLTensor(ptr); - if (out.ptr_->version.major != VERSION.major) { + if (ARROW_PREDICT_FALSE(out.ptr_->version.major != VERSION.major)) { return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, ", expected ", VERSION.major); } + if (ARROW_PREDICT_FALSE(out.tensor().ndim < 0)) { + return Status::Invalid("Invalid DLPack tensor: ndim must be >= 0"); + } + if (ARROW_PREDICT_FALSE(out.tensor().ndim != 0 && out.tensor().shape == nullptr)) { + return Status::Invalid( + "Invalid DLPack tensor: shape must be non-null when ndim != 0"); + } + if (ARROW_PREDICT_FALSE(out.tensor().ndim != 0 && out.tensor().strides == nullptr)) { + return Status::Invalid( + "Invalid DLPack tensor: strides must be non-null when ndim != 0"); + } return out; } @@ -389,12 +398,17 @@ Result> DataTypeFromDLPack(DLDataType dtype) { } } -inline std::vector StridesInBytes(std::span strides, - int64_t byte_width) { +Result> StridesInBytes(std::span strides, + int64_t byte_width) { std::vector out{}; out.reserve(strides.size()); for (const auto& s : strides) { - out.push_back(s * byte_width); + int64_t stride_bytes = 0; + if (ARROW_PREDICT_FALSE( + internal::MultiplyWithOverflow(s, byte_width, &stride_bytes))) { + return Status::Invalid("Overflow computing DLPack tensor stride in bytes."); + } + out.push_back(stride_bytes); } return out; } @@ -403,7 +417,11 @@ Result> ImportBuffer(CppDLTensor&& dl, bool copy) { // DLPack strides are in number of elements, so is the size we compute from them. ARROW_ASSIGN_OR_RAISE(const auto nelements, internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); - const auto nbytes = nelements * dl.byte_width(); + int64_t nbytes = 0; + if (ARROW_PREDICT_FALSE(internal::MultiplyWithOverflow( + nelements, static_cast(dl.byte_width()), &nbytes))) { + return Status::Invalid("Overflow computing DLPack tensor size in bytes."); + } // DLPack mandates a null data pointer when the tensor holds no element, so there is // neither anything to share nor to copy. uint8_t* data = @@ -445,7 +463,7 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* un } if (dl.ndim() != 1 || dl.strides().front() != 1) { - return Status::NotImplemented( + return Status::Invalid( "Only contiguous one dimensional tensor can be imported as arrays." " Try importing to Tensor first."); } @@ -471,9 +489,10 @@ Result> ImportTensorVersioned(DLManagedTensorVersioned* auto strides = std::vector(dl.strides().begin(), dl.strides().end()); ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); const auto byte_width = type->byte_width(); + ARROW_ASSIGN_OR_RAISE(auto strides_bytes, StridesInBytes(strides, byte_width)); return Tensor::Make(std::move(type), std::move(buffer), std::move(shape), - StridesInBytes(std::move(strides), byte_width)); + std::move(strides_bytes)); } } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 6b552cf818f..4c7f7a41ab2 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -110,10 +110,9 @@ Result ExportDevice(const std::shared_ptr& t); /// \brief Import a DLPack tensor as an Arrow Array. /// -/// Same restrictions on data types as `ExportArrayVersioned`, and only row-major -/// tensors are supported. Dimensions beyond the first are imported as nested -/// fixed size lists. -/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// Same restrictions on data types as `ExportArrayVersioned`, only row-major +/// tensors are supported. Takes ownership of the `DLManagedTensorVersioned` in +/// an error-safe fashion. /// /// \param[in] raw DLPack tensor /// \param[in] copy Whether to copy the data instead of sharing it with the DLPack @@ -126,7 +125,7 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* ra /// \brief Import a DLPack tensor as an Arrow Tensor. /// /// Same restrictions on data types as `ExportTensorVersioned`. -/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// Takes ownership of the `DLManagedTensorVersioned` in an error-safe fashion. /// If the DLPack input is marked as readonly, this will produce an immutable tensor. /// /// \param[in] raw Arrow array diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 6f320e24b25..4f22c5c527a 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -649,8 +649,8 @@ TYPED_TEST(TestImportArray, OneDimension) { TYPED_TEST(TestImportArray, Unsupported) { auto check = [](ForeignTensor foreign) { ASSERT_RAISES_WITH_MESSAGE( - NotImplemented, - "NotImplemented: Only contiguous one dimensional tensor can be imported as" + Invalid, + "Invalid: Only contiguous one dimensional tensor can be imported as" " arrays. Try importing to Tensor first.", TypeParam::Import(Produce(std::move(foreign)))); }; diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index add5d2d3f4d..a4c30deedb4 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2274,6 +2274,7 @@ cdef class Array(_PandasConvertible): return pyarrow_wrap_array(array) + @staticmethod def from_dlpack(x, /, *, device=None, copy=None): """ Construct an Array from an object implementing the DLPack protocol. @@ -2301,6 +2302,8 @@ cdef class Array(_PandasConvertible): """ version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + if not PyCapsule_CheckExact(pycapsule): + raise TypeError("DLPack producer did not return a PyCapsule") cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( pycapsule, "dltensor_versioned") if ptr == NULL: diff --git a/python/pyarrow/tensor.pxi b/python/pyarrow/tensor.pxi index 656da9f260a..a4044986699 100644 --- a/python/pyarrow/tensor.pxi +++ b/python/pyarrow/tensor.pxi @@ -18,7 +18,11 @@ # Avoid name clash with `pa.struct` function import struct as _struct -from cpython.pycapsule cimport PyCapsule_SetName, PyCapsule_GetPointer +from cpython.pycapsule cimport ( + PyCapsule_CheckExact, + PyCapsule_GetPointer, + PyCapsule_SetName, +) cdef class Tensor(_Weakrefable): @@ -302,6 +306,7 @@ strides: {self.strides}""" buffer.strides = cp.PyBytes_AsString(self._ssize_t_strides) buffer.suboffsets = NULL + @staticmethod def from_dlpack(x, /, *, device=None, copy=None): """ Construct a Tensor from an object implementing the DLPack protocol. @@ -310,8 +315,7 @@ strides: {self.strides}""" ---------- x : object The input object containing array data, following the DLPack - protocol (has a ``__dlpack__`` method) or the array API's - ``__array_namespace__`` protocol. + protocol (has a ``__dlpack__`` method). device : tuple[enum.Enum, int], optional Designates where the resulting Tensor should reside, in the format returned by :meth:`Tensor.__dlpack_device__`. When None, @@ -330,6 +334,8 @@ strides: {self.strides}""" """ version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + if not PyCapsule_CheckExact(pycapsule): + raise TypeError("DLPack producer did not return a PyCapsule") cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( pycapsule, "dltensor_versioned") if ptr == NULL: diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 33045373f29..34ffa0733b1 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -412,7 +412,8 @@ def test_array_from_dlpack_multi_dim_not_supported(): pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") expected = np.arange(6, dtype=np.int32).reshape((2, 3)) - with pytest.raises(pa.ArrowNotImplementedError, - match="Only contiguous one dimensional tensor can be " - "imported as arrays"): + with pytest.raises( + pa.ArrowInvalid, + match="Only contiguous one dimensional tensor can be imported as arrays", + ): pa.Array.from_dlpack(expected)