diff --git a/docs/guide.md b/docs/guide.md index 62c17017..64d7a9cd 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -733,6 +733,8 @@ int main() { return condy::sync_wait(co_main()); } ### Buffer Registration +TODO: Document extended buffer registration and dma-buf backed buffers. + Similar to files, each io_uring async operation needs to acquire a reference to memory pages. Pre-registering memory regions with the kernel can avoid this overhead and improve performance. Condy abstracts buffer registration as operations on the `condy::BufferTable` type. Each `condy::Runtime` has a `condy::BufferTable` object, accessible via `condy::Runtime::buffer_table()`. diff --git a/include/condy/condy_uring.hpp b/include/condy/condy_uring.hpp index 429d6e03..d3c38876 100644 --- a/include/condy/condy_uring.hpp +++ b/include/condy/condy_uring.hpp @@ -7,6 +7,7 @@ #include // IWYU pragma: export #include +#include #include #include @@ -24,6 +25,39 @@ (IO_URING_VERSION_MAJOR > (major) || \ (IO_URING_VERSION_MAJOR == (major) && IO_URING_VERSION_MINOR >= (minor))) +// TODO: Wait for these Linux 7.4 UAPI definitions to be synced into liburing; +// remove this temporary compatibility block before merging. +struct io_uring_rsrc_update2_tmp { + __u32 offset; + __u32 flags; + __aligned_u64 data; + __aligned_u64 tags; + __u32 nr; + __u32 resv2; +}; + +enum io_uring_rsrc_reg_flags { + IORING_RSRC_UPDATE_EXTENDED = 1U << 1, +}; + +enum io_uring_regbuf_type { + IO_REGBUF_TYPE_EMPTY, + IO_REGBUF_TYPE_UADDR, + IO_REGBUF_TYPE_DMABUF, + + __IO_REGBUF_TYPE_MAX, +}; + +struct io_uring_regbuf_desc { + __u32 type; // enum io_uring_regbuf_type + __u32 flags; + uint64_t size; + uint64_t uaddr; + __s32 dmabuf_fd; + __s32 target_fd; + uint64_t __resv[6]; +}; + // Polyfill for io_uring_prep_uring_cmd (added in liburing 2.13) // Opcode exists since 2.3, only the helper function is missing #if !CONDY_URING_VERSION_GE(2, 13) // < 2.13 diff --git a/include/condy/ring_settings.hpp b/include/condy/ring_settings.hpp index d94210a1..f973e11f 100644 --- a/include/condy/ring_settings.hpp +++ b/include/condy/ring_settings.hpp @@ -130,6 +130,29 @@ class BufferTable { nullptr, nr_vecs); } +#if CONDY_URING_VERSION_GE(2, 16) // >= 2.16 + /** + * @brief Update registered buffers using extended kernel descriptors. + * @param index_base First buffer-table index to update. + * @param descs Array of extended buffer descriptors. + * @param nr Number of descriptors in the array. + * @return Number of buffers updated, or a negative errno on failure. + * @details Requires Linux 7.4. Descriptor types and flags follow the + * kernel UAPI; support is determined by the running kernel. + */ + int update(unsigned index_base, const io_uring_regbuf_desc *descs, + unsigned nr) noexcept { + io_uring_rsrc_update2_tmp update{}; + update.offset = index_base; + update.flags = IORING_RSRC_UPDATE_EXTENDED; + update.data = reinterpret_cast(descs); + update.nr = nr; + return io_uring_register(ring_.ring_fd, IORING_REGISTER_BUFFERS_UPDATE, + &update, sizeof(update)); + } + +#endif + #if CONDY_URING_VERSION_GE(2, 10) // >= 2.10 /** @@ -277,4 +300,4 @@ class RingSettings { io_uring_probe *probe_ = nullptr; }; -} // namespace condy \ No newline at end of file +} // namespace condy diff --git a/tests/test_dmabuf.cpp b/tests/test_dmabuf.cpp new file mode 100644 index 00000000..88cdc8f0 --- /dev/null +++ b/tests/test_dmabuf.cpp @@ -0,0 +1,216 @@ +#include "condy.hpp" + +#if CONDY_URING_VERSION_GE(2, 16) // >= 2.16 +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class DMABufResources { +public: + DMABufResources() { + bool ok = false; + auto cleanup = condy::detail::defer([&] { + if (!ok) { + release_(); + } + }); + heap_ = open("/dev/dma_heap/system", O_RDWR | O_CLOEXEC); + REQUIRE(heap_ >= 0); + dma_heap_allocation_data alloc{}; + alloc.len = size_; + alloc.fd_flags = O_RDWR | O_CLOEXEC; + REQUIRE(ioctl(heap_, DMA_HEAP_IOCTL_ALLOC, &alloc) == 0); + dma_ = alloc.fd; + mapping_ = + mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, dma_, 0); + REQUIRE(mapping_ != MAP_FAILED); + ok = true; + } + + CONDY_DELETE_COPY_MOVE(DMABufResources); + + ~DMABufResources() { release_(); } + + int dma_fd() const noexcept { return dma_; } + void *data() const noexcept { return mapping_; } + size_t size() const noexcept { return size_; } + + void sync(uint64_t flags) const { + dma_buf_sync sync{.flags = flags}; + REQUIRE(ioctl(dma_, DMA_BUF_IOCTL_SYNC, &sync) == 0); + } + +private: + void release_() noexcept { + if (mapping_ != MAP_FAILED) { + munmap(mapping_, size_); + } + for (int fd : {heap_, dma_}) { + if (fd >= 0) { + close(fd); + } + } + } + + int heap_ = -1; + int dma_ = -1; + void *mapping_ = MAP_FAILED; + size_t size_ = 8192; +}; + +} // namespace + +TEST_CASE("test dmabuf - invalid registration preserves ordinary buffers") { + auto task = []() -> condy::Coro { + auto &table = condy::current_runtime().buffer_table(); + REQUIRE(table.init(2) == 0); + // Old kernels reject extended updates; new kernels reject these fds. + io_uring_regbuf_desc desc{}; + desc.type = IO_REGBUF_TYPE_DMABUF; + desc.dmabuf_fd = -1; + desc.target_fd = -1; + REQUIRE(table.update(0, &desc, 1) < 0); + char data[32] = {}; + iovec vec{.iov_base = data, .iov_len = sizeof(data)}; + REQUIRE(table.update(0, &vec, 1) == 1); + REQUIRE(table.destroy() == 0); + co_return; + }; + condy::sync_wait(task()); +} + +TEST_CASE("test buffer_table - extended descriptors") { + auto task = []() -> condy::Coro { + auto &table = condy::current_runtime().buffer_table(); + REQUIRE(table.init(2) == 0); + char bytes[32]{}; + io_uring_regbuf_desc descs[2]{}; + descs[0].type = IO_REGBUF_TYPE_UADDR; + descs[0].size = sizeof(bytes); + descs[0].uaddr = reinterpret_cast(bytes); + int r = table.update(0, descs, 2); + if (r == -EINVAL) { + MESSAGE("Kernel does not support extended buffer updates"); + co_return; + } + REQUIRE(r == 2); + descs[1].type = ~0U; + REQUIRE(table.update(0, descs, 2) == 1); + REQUIRE(table.update(0, descs + 1, 1) == -EINVAL); + io_uring_regbuf_desc empty{}; + REQUIRE(table.update(0, &empty, 1) == 1); + REQUIRE(table.destroy() == 0); + co_return; + }; + condy::sync_wait(task()); +} + +// This test overwrites the first 4KiB of an explicitly supplied scratch NVMe +// device. Run in QEMU with CONDY_TEST_NVME_DEVICE_PATH=/dev/nvme0n1. +TEST_CASE("test dmabuf - NVMe read write and clone") { + const char *device = std::getenv("CONDY_TEST_NVME_DEVICE_PATH"); + if (device == nullptr || device[0] == '\0') { + MESSAGE("CONDY_TEST_NVME_DEVICE_PATH not set, skipping"); + return; + } + + auto task = [&]() -> condy::Coro { + auto &runtime = condy::current_runtime(); + auto &table = runtime.buffer_table(); + REQUIRE(table.init(2) == 0); + io_uring_regbuf_desc desc{}; + desc.type = IO_REGBUF_TYPE_DMABUF; + desc.dmabuf_fd = -1; + desc.target_fd = -1; + // A supported dma-buf registration reaches fd lookup and returns + // EBADF. Probe before allocating resources or opening the device. + int r = table.update(0, &desc, 1); + if (r == -EINVAL || r == -EOPNOTSUPP || r == -ENOSYS) { + MESSAGE("Kernel does not support dma-buf registration, skipping"); + co_return; + } + REQUIRE(r == -EBADF); + + DMABufResources resources; + int target = open(device, O_RDWR | O_DIRECT | O_CLOEXEC); + REQUIRE(target >= 0); + auto close_target = condy::detail::defer([&] { close(target); }); + int other = open(device, O_RDWR | O_DIRECT | O_CLOEXEC); + REQUIRE(other >= 0); + auto close_other = condy::detail::defer([&] { close(other); }); + desc.dmabuf_fd = resources.dma_fd(); + desc.target_fd = target; + r = table.update(0, &desc, 1); + if (r == -EOPNOTSUPP || r == -ENODEV) { + MESSAGE("Device does not support dma-buf I/O, skipping"); + co_return; + } + REQUIRE(r == 1); + REQUIRE(table.update(2, &desc, 1) == -EINVAL); + + condy::Runtime poll_runtime(condy::RuntimeOptions().enable_iopoll()); + auto &poll_table = poll_runtime.buffer_table(); + REQUIRE(poll_table.init(2) == 0); + REQUIRE(poll_table.update(0, &desc, 1) == -EOPNOTSUPP); + REQUIRE(poll_table.clone_buffers(table, 0, 0, 1) == -EOPNOTSUPP); + REQUIRE(poll_table.destroy() == 0); + + auto *bytes = static_cast(resources.data()); + resources.sync(DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE); + for (size_t i = 0; i < resources.size(); ++i) { + bytes[i] = static_cast(i * 37 + i / 256); + } + resources.sync(DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE); + + // A nonzero buffer offset is encoded in sqe->addr, not a CPU address. + int index = 0; + auto range = condy::fixed( + index, + condy::buffer(reinterpret_cast(uintptr_t{4096}), 4096)); + REQUIRE(co_await condy::async_write(target, range, 0) == 4096); + REQUIRE(co_await condy::async_read(other, range, 0) == -EBADF); + REQUIRE(co_await condy::async_read( + target, + condy::fixed(index, condy::buffer(reinterpret_cast( + resources.size()), + 4096)), + 0) == -EFAULT); + + // Clone to an independent table, release the original entry, and + // clone back at a new index. This also checks shared kernel lifetime. + condy::Runtime clone_runtime; + auto &cloned = clone_runtime.buffer_table(); + REQUIRE(cloned.clone_buffers(table, 0, 0, 1) == 0); + iovec empty{}; + REQUIRE(table.update(0, &empty, 1) == 1); + REQUIRE(table.clone_buffers(cloned, 1, 0, 1) == 0); + REQUIRE(cloned.destroy() == 0); + index = 1; + + int target_fds[] = {target}; + REQUIRE(runtime.fd_table().init(target_fds, 1) == 0); + range = condy::fixed(index, + condy::buffer(static_cast(nullptr), 4096)); + REQUIRE(co_await condy::async_read(condy::fixed(0), range, 0) == 4096); + resources.sync(DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ); + CHECK(std::memcmp(bytes, bytes + 4096, 4096) == 0); + resources.sync(DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ); + + // Replacing a dma-buf with ordinary memory must release it normally. + char data[32]{}; + iovec vec{.iov_base = data, .iov_len = sizeof(data)}; + REQUIRE(table.update(index, &vec, 1) == 1); + REQUIRE(table.destroy() == 0); + REQUIRE(runtime.fd_table().destroy() == 0); + co_return; + }; + condy::sync_wait(task()); +} + +#endif