From 83bfffe64d5080a26fa160b0c8520f456ea06a13 Mon Sep 17 00:00:00 2001 From: Cheng Lingfei Date: Mon, 15 Jun 2026 10:01:49 +0800 Subject: [PATCH] lkl: Add MSI and MSI-X support Add PCI MSI/MSI-X interrupt support for LKL PCI devices. Implement LKL arch MSI hooks so PCI drivers can allocate MSI and MSI-X vectors through the generic PCI MSI fallback path. MSI vectors are backed by contiguous LKL IRQ blocks, while MSI-X vectors preserve sparse hardware vector indexes. The setup path associates MSI descriptors with LKL IRQs, installs an MSI irq_chip, programs the MSI multiple-message enable value, and unwinds IRQ state on failures. Extend the LKL PCI host ABI with MSI/MSI-X setup and teardown callbacks, and teach the VFIO PCI host backend to wire VFIO MSI/MSI-X eventfds to LKL IRQs. When MSI/MSI-X is enabled, the VFIO backend disables the INTx path, polls the MSI eventfds, triggers the matching LKL IRQs, and restores INTx when MSI/MSI-X is torn down. Also fix LKL ioremap handling for BAR offsets and add KUnit coverage for MSI IRQ allocation, sparse MSI-X vector indexes, and failure unwind paths. Enable PCI_MSI in the LKL defconfig and include the MSI KUnit suite in the LKL test configuration. Signed-off-by: Cheng Lingfei --- arch/lkl/Kconfig | 8 + arch/lkl/configs/defconfig | 1 + arch/lkl/drivers/Makefile | 2 + arch/lkl/drivers/pci-msi-test.c | 350 +++++++++++++++ arch/lkl/drivers/pci-msi.c | 235 ++++++++++ arch/lkl/include/asm/irq.h | 2 + arch/lkl/include/uapi/asm/host_ops.h | 12 + arch/lkl/kernel/irq.c | 35 ++ tools/lkl/Makefile.autoconf | 3 + tools/lkl/lib/iomem.c | 5 +- tools/lkl/lib/vfio_pci.c | 461 ++++++++++++++++++-- tools/lkl/scripts/qemu-linux-make-images.sh | 5 + tools/lkl/tests/boot.c | 31 +- tools/lkl/tests/disk-vfio-pci.c | 89 +++- tools/lkl/tests/memory-report | 0 15 files changed, 1191 insertions(+), 48 deletions(-) create mode 100644 arch/lkl/drivers/pci-msi-test.c create mode 100644 arch/lkl/drivers/pci-msi.c create mode 100644 tools/lkl/tests/memory-report diff --git a/arch/lkl/Kconfig b/arch/lkl/Kconfig index 5a04bfdb99424b..ffb706aa78d8cc 100644 --- a/arch/lkl/Kconfig +++ b/arch/lkl/Kconfig @@ -125,6 +125,7 @@ config PCI select GENERIC_PCI_IOMAP select HAS_DMA select ARCH_HAS_DMA_OPS + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI default y config LKL_PCI_KUNIT_TEST @@ -139,6 +140,13 @@ config LKL_PCI_KUNIT_TEST memory returned to callers is unmapped and released using the correct CPU address. +config LKL_PCI_MSI_KUNIT_TEST + bool "KUnit tests for LKL PCI MSI" + depends on KUNIT && PCI_MSI + default n + help + KUnit tests for LKL's PCI MSI and MSI-X IRQ allocation hooks. + config RAID6_PQ_BENCHMARK bool default n diff --git a/arch/lkl/configs/defconfig b/arch/lkl/configs/defconfig index fc42217354ea8d..45332b692f5d59 100644 --- a/arch/lkl/configs/defconfig +++ b/arch/lkl/configs/defconfig @@ -29,6 +29,7 @@ CONFIG_NET_SCH_FQ=y # CONFIG_WIRELESS is not set # CONFIG_FW_LOADER is not set CONFIG_VIRTIO_BLK=y +CONFIG_PCI_MSI=y CONFIG_BLK_DEV_NVME=y CONFIG_NETDEVICES=y CONFIG_VIRTIO_NET=y diff --git a/arch/lkl/drivers/Makefile b/arch/lkl/drivers/Makefile index 6b0ee1ce448c8a..584603104b4825 100644 --- a/arch/lkl/drivers/Makefile +++ b/arch/lkl/drivers/Makefile @@ -2,3 +2,5 @@ obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_LKL_PCI_KUNIT_TEST) += pci_test.o +obj-$(CONFIG_PCI_MSI) += pci-msi.o +obj-$(CONFIG_LKL_PCI_MSI_KUNIT_TEST) += pci-msi-test.o diff --git a/arch/lkl/drivers/pci-msi-test.c b/arch/lkl/drivers/pci-msi-test.c new file mode 100644 index 00000000000000..e2cc8e16d0a026 --- /dev/null +++ b/arch/lkl/drivers/pci-msi-test.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#define LKL_PCI_MSI_TEST_MAX_VECTORS 8 + +struct lkl_pci_msi_test_ctx { + struct pci_dev pdev; + struct pci_bus bus; + u8 config[PCI_CFG_SPACE_SIZE]; + bool dev_added; + struct lkl_dev_pci_ops ops; + struct lkl_dev_pci_ops *saved_ops; + unsigned long fake_dev; + + int fail_msi_init; + int init_calls; + int teardown_calls; + int last_type; + int last_nvec; + int last_irqs[LKL_PCI_MSI_TEST_MAX_VECTORS]; + struct lkl_pci_dev *last_dev; +}; + +static int lkl_pci_msi_test_config_bounds(unsigned int devfn, int where, + int size) +{ + if (devfn != 0 || where < 0 || size <= 0 || size > 4) + return PCIBIOS_BAD_REGISTER_NUMBER; + if ((unsigned int)where > PCI_CFG_SPACE_SIZE - size) + return PCIBIOS_BAD_REGISTER_NUMBER; + + return PCIBIOS_SUCCESSFUL; +} + +static int lkl_pci_msi_test_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct lkl_pci_msi_test_ctx *ctx = bus->sysdata; + int ret; + + ret = lkl_pci_msi_test_config_bounds(devfn, where, size); + if (ret) + return ret; + + *val = 0; + memcpy(val, &ctx->config[where], size); + return PCIBIOS_SUCCESSFUL; +} + +static int lkl_pci_msi_test_config_write(struct pci_bus *bus, + unsigned int devfn, int where, + int size, u32 val) +{ + struct lkl_pci_msi_test_ctx *ctx = bus->sysdata; + int ret; + + ret = lkl_pci_msi_test_config_bounds(devfn, where, size); + if (ret) + return ret; + + memcpy(&ctx->config[where], &val, size); + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops lkl_pci_msi_test_pci_ops = { + .read = lkl_pci_msi_test_config_read, + .write = lkl_pci_msi_test_config_write, +}; + +static void lkl_pci_msi_test_config_write_word(struct lkl_pci_msi_test_ctx *ctx, + int where, u16 val) +{ + memcpy(&ctx->config[where], &val, sizeof(val)); +} + +static u16 lkl_pci_msi_test_config_read_word(struct lkl_pci_msi_test_ctx *ctx, + int where) +{ + u16 val; + + memcpy(&val, &ctx->config[where], sizeof(val)); + return val; +} + +static void lkl_pci_msi_test_release(struct device *dev) +{ +} + +static int lkl_pci_msi_test_init_op(struct lkl_pci_dev *dev, int type, + int nvec, int *irqs) +{ + struct lkl_pci_msi_test_ctx *ctx = (void *)dev; + int i; + + ctx->init_calls++; + ctx->last_dev = dev; + ctx->last_type = type; + ctx->last_nvec = nvec; + memset(ctx->last_irqs, 0, sizeof(ctx->last_irqs)); + + for (i = 0; i < nvec && i < LKL_PCI_MSI_TEST_MAX_VECTORS; i++) + ctx->last_irqs[i] = irqs[i]; + + return ctx->fail_msi_init; +} + +static void lkl_pci_msi_test_teardown_op(struct lkl_pci_dev *dev, int type) +{ + struct lkl_pci_msi_test_ctx *ctx = (void *)dev; + + ctx->teardown_calls++; + ctx->last_type = type; +} + +static int lkl_pci_msi_test_suite_init(struct kunit *test) +{ + struct lkl_pci_msi_test_ctx *ctx; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, lkl_ops); + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + + ctx->saved_ops = lkl_ops->pci_ops; + if (ctx->saved_ops) + ctx->ops = *ctx->saved_ops; + ctx->ops.msi_init = lkl_pci_msi_test_init_op; + ctx->ops.msi_teardown = lkl_pci_msi_test_teardown_op; + lkl_ops->pci_ops = &ctx->ops; + + ctx->bus.ops = &lkl_pci_msi_test_pci_ops; + ctx->bus.sysdata = ctx; + + device_initialize(&ctx->pdev.dev); + ctx->pdev.dev.bus = &pci_bus_type; + ctx->pdev.dev.release = lkl_pci_msi_test_release; + ctx->pdev.bus = &ctx->bus; + ctx->pdev.devfn = 0; + ctx->pdev.msi_cap = 0x50; + ctx->pdev.sysdata = ctx; + lkl_pci_msi_test_config_write_word( + ctx, ctx->pdev.msi_cap + PCI_MSI_FLAGS, + FIELD_PREP(PCI_MSI_FLAGS_QMASK, 3)); + + ret = dev_set_name(&ctx->pdev.dev, "lkl-pci-msi-%p", ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = device_add(&ctx->pdev.dev); + KUNIT_ASSERT_EQ(test, ret, 0); + ctx->dev_added = true; + + ret = msi_setup_device_data(&ctx->pdev.dev); + KUNIT_ASSERT_EQ(test, ret, 0); + + test->priv = ctx; + return 0; +} + +static void lkl_pci_msi_test_suite_exit(struct kunit *test) +{ + struct lkl_pci_msi_test_ctx *ctx = test->priv; + + if (ctx->pdev.dev.msi.data) { + msi_lock_descs(&ctx->pdev.dev); + arch_teardown_msi_irqs(&ctx->pdev); + msi_free_msi_descs(&ctx->pdev.dev); + msi_unlock_descs(&ctx->pdev.dev); + } + + if (ctx->dev_added) + device_del(&ctx->pdev.dev); + + lkl_ops->pci_ops = ctx->saved_ops; + put_device(&ctx->pdev.dev); +} + +static void lkl_pci_msi_insert_desc(struct kunit *test, + struct lkl_pci_msi_test_ctx *ctx, + unsigned int index, unsigned int nvec, + bool is_msix) +{ + struct msi_desc desc; + int ret; + + memset(&desc, 0, sizeof(desc)); + desc.msi_index = index; + desc.nvec_used = nvec; + desc.pci.msi_attrib.is_msix = is_msix; + desc.pci.msi_attrib.multiple = order_base_2(nvec); + + ret = msi_insert_msi_desc(&ctx->pdev.dev, &desc); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +static void lkl_pci_msi_expect_irq_chip_ready(struct kunit *test, + unsigned int irq) +{ + struct irq_data *data = irq_get_irq_data(irq); + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, data->chip); + KUNIT_EXPECT_TRUE(test, data->chip->irq_ack != NULL); + KUNIT_EXPECT_TRUE(test, data->chip->irq_mask != NULL); + KUNIT_EXPECT_TRUE(test, data->chip->irq_unmask != NULL); +} + +static void lkl_pci_msi_allocates_contiguous_irqs(struct kunit *test) +{ + struct lkl_pci_msi_test_ctx *ctx = test->priv; + struct msi_desc *desc; + u16 control; + int ret, i, base; + + msi_lock_descs(&ctx->pdev.dev); + lkl_pci_msi_insert_desc(test, ctx, 0, 4, false); + + ret = arch_setup_msi_irqs(&ctx->pdev, 4, PCI_CAP_ID_MSI); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, ctx->init_calls, 1); + KUNIT_EXPECT_EQ(test, ctx->last_type, LKL_PCI_IRQ_MSI); + KUNIT_EXPECT_EQ(test, ctx->last_nvec, 4); + KUNIT_EXPECT_PTR_EQ(test, ctx->last_dev, (struct lkl_pci_dev *)ctx); + + base = ctx->last_irqs[0]; + KUNIT_EXPECT_GT(test, base, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, ctx->last_irqs[i], base + i); + lkl_pci_msi_expect_irq_chip_ready(test, base); + + desc = msi_first_desc(&ctx->pdev.dev, MSI_DESC_ASSOCIATED); + KUNIT_ASSERT_NOT_NULL(test, desc); + KUNIT_EXPECT_EQ(test, desc->irq, base); + control = lkl_pci_msi_test_config_read_word( + ctx, ctx->pdev.msi_cap + PCI_MSI_FLAGS); + KUNIT_EXPECT_EQ(test, (int)FIELD_GET(PCI_MSI_FLAGS_QSIZE, control), + (int)desc->pci.msi_attrib.multiple); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, irq_get_msi_desc(base + i), desc); + + arch_teardown_msi_irqs(&ctx->pdev); + KUNIT_EXPECT_EQ(test, ctx->teardown_calls, 1); + KUNIT_EXPECT_EQ(test, ctx->last_type, LKL_PCI_IRQ_MSI); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(base + i)); + + msi_free_msi_descs(&ctx->pdev.dev); + msi_unlock_descs(&ctx->pdev.dev); +} + +static void lkl_pci_msix_preserves_vector_indexes(struct kunit *test) +{ + struct lkl_pci_msi_test_ctx *ctx = test->priv; + struct msi_desc *desc0, *desc2, *desc5; + int ret, irq0, irq2, irq5; + + msi_lock_descs(&ctx->pdev.dev); + lkl_pci_msi_insert_desc(test, ctx, 0, 1, true); + lkl_pci_msi_insert_desc(test, ctx, 2, 1, true); + lkl_pci_msi_insert_desc(test, ctx, 5, 1, true); + + ret = arch_setup_msi_irqs(&ctx->pdev, 3, PCI_CAP_ID_MSIX); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, ctx->init_calls, 1); + KUNIT_EXPECT_EQ(test, ctx->last_type, LKL_PCI_IRQ_MSIX); + KUNIT_EXPECT_EQ(test, ctx->last_nvec, 6); + + irq0 = ctx->last_irqs[0]; + irq2 = ctx->last_irqs[2]; + irq5 = ctx->last_irqs[5]; + KUNIT_EXPECT_GT(test, irq0, 0); + KUNIT_EXPECT_EQ(test, ctx->last_irqs[1], 0); + KUNIT_EXPECT_GT(test, irq2, 0); + KUNIT_EXPECT_EQ(test, ctx->last_irqs[3], 0); + KUNIT_EXPECT_EQ(test, ctx->last_irqs[4], 0); + KUNIT_EXPECT_GT(test, irq5, 0); + lkl_pci_msi_expect_irq_chip_ready(test, irq0); + lkl_pci_msi_expect_irq_chip_ready(test, irq2); + lkl_pci_msi_expect_irq_chip_ready(test, irq5); + + desc0 = irq_get_msi_desc(irq0); + desc2 = irq_get_msi_desc(irq2); + desc5 = irq_get_msi_desc(irq5); + KUNIT_ASSERT_NOT_NULL(test, desc0); + KUNIT_ASSERT_NOT_NULL(test, desc2); + KUNIT_ASSERT_NOT_NULL(test, desc5); + KUNIT_EXPECT_EQ(test, desc0->msi_index, 0); + KUNIT_EXPECT_EQ(test, desc2->msi_index, 2); + KUNIT_EXPECT_EQ(test, desc5->msi_index, 5); + + arch_teardown_msi_irqs(&ctx->pdev); + KUNIT_EXPECT_EQ(test, ctx->teardown_calls, 1); + KUNIT_EXPECT_EQ(test, ctx->last_type, LKL_PCI_IRQ_MSIX); + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(irq0)); + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(irq2)); + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(irq5)); + + msi_free_msi_descs(&ctx->pdev.dev); + msi_unlock_descs(&ctx->pdev.dev); +} + +static void lkl_pci_msi_failure_unwinds_associations(struct kunit *test) +{ + struct lkl_pci_msi_test_ctx *ctx = test->priv; + struct msi_desc *desc; + int ret; + + ctx->fail_msi_init = -EIO; + + msi_lock_descs(&ctx->pdev.dev); + lkl_pci_msi_insert_desc(test, ctx, 0, 1, true); + lkl_pci_msi_insert_desc(test, ctx, 1, 1, true); + + ret = arch_setup_msi_irqs(&ctx->pdev, 2, PCI_CAP_ID_MSIX); + KUNIT_EXPECT_EQ(test, ret, -EIO); + KUNIT_EXPECT_EQ(test, ctx->init_calls, 1); + KUNIT_EXPECT_EQ(test, ctx->teardown_calls, 0); + + desc = msi_first_desc(&ctx->pdev.dev, MSI_DESC_ASSOCIATED); + KUNIT_EXPECT_NULL(test, desc); + if (ctx->last_irqs[0] > 0) + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(ctx->last_irqs[0])); + if (ctx->last_irqs[1] > 0) + KUNIT_EXPECT_NULL(test, irq_get_msi_desc(ctx->last_irqs[1])); + + msi_free_msi_descs(&ctx->pdev.dev); + msi_unlock_descs(&ctx->pdev.dev); +} + +static struct kunit_case lkl_pci_msi_test_cases[] = { + KUNIT_CASE(lkl_pci_msi_allocates_contiguous_irqs), + KUNIT_CASE(lkl_pci_msix_preserves_vector_indexes), + KUNIT_CASE(lkl_pci_msi_failure_unwinds_associations), + {} +}; + +static struct kunit_suite lkl_pci_msi_test_suite = { + .name = "lkl_pci_msi", + .init = lkl_pci_msi_test_suite_init, + .exit = lkl_pci_msi_test_suite_exit, + .test_cases = lkl_pci_msi_test_cases, +}; + +kunit_test_suite(lkl_pci_msi_test_suite); diff --git a/arch/lkl/drivers/pci-msi.c b/arch/lkl/drivers/pci-msi.c new file mode 100644 index 00000000000000..101d6bbe5b2150 --- /dev/null +++ b/arch/lkl/drivers/pci-msi.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LKL_MSI_USER "pci-msi" +#define LKL_MSIX_USER "pci-msix" + +static void lkl_pci_msi_ack(struct irq_data *data) +{ +} + +static struct irq_chip lkl_pci_msi_chip = { + .name = "LKL-PCI-MSI", + .irq_ack = lkl_pci_msi_ack, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, + .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_ONESHOT_SAFE, +}; + +static int lkl_pci_setup_msi_desc(struct msi_desc *desc, unsigned int irq) +{ + int i, ret; + + for (i = 0; i < desc->nvec_used; i++) { + irq_set_chip_and_handler(irq + i, &lkl_pci_msi_chip, + handle_edge_irq); + ret = irq_set_msi_desc_off(irq, i, desc); + if (ret) { + while (--i >= 0) + irq_set_msi_desc_off(irq, i, NULL); + desc->irq = 0; + return ret; + } + } + + return 0; +} + +static int lkl_pci_enable_multi_msi(struct pci_dev *dev, struct msi_desc *desc) +{ + u16 control; + int ret; + + ret = pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + if (ret) + return -EIO; + + control &= ~PCI_MSI_FLAGS_QSIZE; + control |= FIELD_PREP(PCI_MSI_FLAGS_QSIZE, + desc->pci.msi_attrib.multiple); + + ret = pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); + if (ret) + return -EIO; + + ret = pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + if (ret || FIELD_GET(PCI_MSI_FLAGS_QSIZE, control) != + desc->pci.msi_attrib.multiple) + return -EIO; + + return 0; +} + +static void lkl_pci_clear_msi_desc(struct msi_desc *desc) +{ + unsigned int irq = desc->irq; + int i; + + for (i = 0; i < desc->nvec_used; i++) + irq_set_msi_desc_off(irq, i, NULL); + + desc->irq = 0; +} + +static int lkl_pci_setup_msi_irqs(struct pci_dev *dev, int nvec) +{ + struct msi_desc *desc; + int *irqs, base, i, ret; + + desc = msi_first_desc(&dev->dev, MSI_DESC_NOTASSOCIATED); + if (!desc || desc->nvec_used != nvec) + return -EINVAL; + + base = lkl_get_free_irq_block(LKL_MSI_USER, nvec); + if (base < 0) + return base; + + irqs = kcalloc(nvec, sizeof(*irqs), GFP_KERNEL); + if (!irqs) { + lkl_put_irq_block(base, nvec, LKL_MSI_USER); + return -ENOMEM; + } + + for (i = 0; i < nvec; i++) + irqs[i] = base + i; + + ret = lkl_pci_setup_msi_desc(desc, base); + if (ret) + goto err_free_irq; + + ret = lkl_ops->pci_ops->msi_init(dev->sysdata, LKL_PCI_IRQ_MSI, + nvec, irqs); + if (ret) + goto err_clear_desc; + + /* + * VFIO only accepts an MME value after VFIO_DEVICE_SET_IRQS has + * established the available MSI vector count. + */ + ret = lkl_pci_enable_multi_msi(dev, desc); + if (ret) { + lkl_ops->pci_ops->msi_teardown(dev->sysdata, LKL_PCI_IRQ_MSI); + goto err_clear_desc; + } + + kfree(irqs); + return 0; + +err_clear_desc: + lkl_pci_clear_msi_desc(desc); +err_free_irq: + lkl_put_irq_block(base, nvec, LKL_MSI_USER); + kfree(irqs); + return ret; +} + +static int lkl_pci_setup_msix_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int *irqs, irq, nvec = 0, ret; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_NOTASSOCIATED) + nvec = max_t(int, nvec, desc->msi_index + 1); + + if (!nvec) + return -EINVAL; + + irqs = kcalloc(nvec, sizeof(*irqs), GFP_KERNEL); + if (!irqs) + return -ENOMEM; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_NOTASSOCIATED) { + irq = lkl_get_free_irq(LKL_MSIX_USER); + if (irq < 0) { + ret = irq; + goto err_clear; + } + + ret = lkl_pci_setup_msi_desc(desc, irq); + if (ret) { + lkl_put_irq(irq, LKL_MSIX_USER); + goto err_clear; + } + + irqs[desc->msi_index] = irq; + } + + ret = lkl_ops->pci_ops->msi_init(dev->sysdata, LKL_PCI_IRQ_MSIX, + nvec, irqs); + if (ret) + goto err_clear; + + kfree(irqs); + return 0; + +err_clear: + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ASSOCIATED) { + irq = desc->irq; + lkl_pci_clear_msi_desc(desc); + lkl_put_irq(irq, LKL_MSIX_USER); + } + kfree(irqs); + return ret; +} + +int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + if (!lkl_ops->pci_ops || !lkl_ops->pci_ops->msi_init) + return -ENODEV; + + switch (type) { + case PCI_CAP_ID_MSI: + return lkl_pci_setup_msi_irqs(dev, nvec); + case PCI_CAP_ID_MSIX: + return lkl_pci_setup_msix_irqs(dev); + default: + return -EINVAL; + } +} + +void arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + bool teardown_called = false; + + /* + * The generic core clears dev->msi_enabled/msix_enabled in + * pci_msi_shutdown()/pci_msix_shutdown() before calling this + * function, so derive the MSI type from the descriptors instead. + */ + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ASSOCIATED) { + if (!teardown_called) { + teardown_called = true; + if (lkl_ops->pci_ops && lkl_ops->pci_ops->msi_teardown) + lkl_ops->pci_ops->msi_teardown( + dev->sysdata, + desc->pci.msi_attrib.is_msix ? + LKL_PCI_IRQ_MSIX : + LKL_PCI_IRQ_MSI); + } + if (desc->pci.msi_attrib.is_msix) { + unsigned int irq = desc->irq; + + lkl_pci_clear_msi_desc(desc); + lkl_put_irq(irq, LKL_MSIX_USER); + } else { + unsigned int irq = desc->irq; + + lkl_pci_clear_msi_desc(desc); + lkl_put_irq_block(irq, desc->nvec_used, LKL_MSI_USER); + } + } +} + +bool arch_restore_msi_irqs(struct pci_dev *dev) +{ + return false; +} diff --git a/arch/lkl/include/asm/irq.h b/arch/lkl/include/asm/irq.h index 975894e1c00794..e2fbe1a5c645ff 100644 --- a/arch/lkl/include/asm/irq.h +++ b/arch/lkl/include/asm/irq.h @@ -6,6 +6,8 @@ void run_irqs(void); void set_irq_pending(int irq); +int lkl_get_free_irq_block(const char *user, int nr_irqs); +void lkl_put_irq_block(int irq, int nr_irqs, const char *user); #include diff --git a/arch/lkl/include/uapi/asm/host_ops.h b/arch/lkl/include/uapi/asm/host_ops.h index c6fb872e3ad82a..673a059512108a 100644 --- a/arch/lkl/include/uapi/asm/host_ops.h +++ b/arch/lkl/include/uapi/asm/host_ops.h @@ -11,6 +11,11 @@ struct lkl_jmp_buf { }; struct lkl_pci_dev; +enum lkl_pci_irq_type { + LKL_PCI_IRQ_MSI = 1, + LKL_PCI_IRQ_MSIX = 2, +}; + /** * lkl_dev_pci_ops - PCI host operations * @@ -20,6 +25,10 @@ struct lkl_pci_dev; * @add - add a new PCI device; returns a handler or NULL if fails * @remove - release resources * @init_irq - allocate resources for interrupts + * @msi_init - allocate resources for MSI/MSI-X interrupts. @irqs contains + * one LKL IRQ per hardware vector, with non-positive entries left + * unassigned. + * @msi_teardown - release MSI/MSI-X interrupt resources * @read - read the PCI Configuration Space * @write - write the PCI Configuration Space * @resource_alloc - map BARx and return the mapped address. x is resource_index @@ -33,6 +42,9 @@ struct lkl_dev_pci_ops { unsigned long ram_size); void (*remove)(struct lkl_pci_dev *dev); int (*irq_init)(struct lkl_pci_dev *dev, int irq); + int (*msi_init)(struct lkl_pci_dev *dev, int type, int nvec, + int *irqs); + void (*msi_teardown)(struct lkl_pci_dev *dev, int type); int (*read)(struct lkl_pci_dev *dev, int where, int size, void *val); int (*write)(struct lkl_pci_dev *dev, int where, int size, void *val); void *(*resource_alloc)(struct lkl_pci_dev *dev, diff --git a/arch/lkl/kernel/irq.c b/arch/lkl/kernel/irq.c index f4fce2b96c1255..fdbabc3e6a46e1 100644 --- a/arch/lkl/kernel/irq.c +++ b/arch/lkl/kernel/irq.c @@ -156,6 +156,33 @@ int lkl_get_free_irq(const char *user) return ret; } +int lkl_get_free_irq_block(const char *user, int nr_irqs) +{ + int i, j; + + if (nr_irqs <= 0 || nr_irqs >= NR_IRQS) + return -EINVAL; + + for (i = 1; i + nr_irqs <= NR_IRQS; i++) { + for (j = 0; j < nr_irqs; j++) + if (irqs[i + j].user) + break; + if (j != nr_irqs) + continue; + + for (j = 0; j < nr_irqs; j++) { + irqs[i + j].user = user; + if (lkl_is_running()) + irq_set_chip_and_handler(i + j, &dummy_irq_chip, + handle_simple_irq); + } + + return i; + } + + return -EBUSY; +} + void lkl_put_irq(int i, const char *user) { if (!irqs[i].user || strcmp(irqs[i].user, user) != 0) { @@ -166,6 +193,14 @@ void lkl_put_irq(int i, const char *user) irqs[i].user = NULL; } +void lkl_put_irq_block(int irq, int nr_irqs, const char *user) +{ + int i; + + for (i = 0; i < nr_irqs; i++) + lkl_put_irq(irq + i, user); +} + unsigned long arch_local_save_flags(void) { return irqs_enabled; diff --git a/tools/lkl/Makefile.autoconf b/tools/lkl/Makefile.autoconf index b665f75bfefb9c..b8129d55a231a9 100644 --- a/tools/lkl/Makefile.autoconf +++ b/tools/lkl/Makefile.autoconf @@ -258,8 +258,11 @@ endef define kunit_test_enable $(call set_autoconf_var,LKL_PCI_KUNIT_TEST,y) + $(call set_autoconf_var,LKL_PCI_MSI_KUNIT_TEST,y) $(call set_kernel_config,KUNIT,y) $(call set_kernel_config,LKL_PCI_KUNIT_TEST,y) + $(call set_kernel_config,PCI_MSI,y) + $(call set_kernel_config,LKL_PCI_MSI_KUNIT_TEST,y) $(if $(filter $(LD_FMT),$(KASAN_HOSTS)),$(call set_kernel_config,KASAN,y)) $(if $(filter $(LD_FMT),$(KASAN_HOSTS)),$(call kasan_test_enable)) endef diff --git a/tools/lkl/lib/iomem.c b/tools/lkl/lib/iomem.c index 90545b901165df..f7dfe094ac10fe 100644 --- a/tools/lkl/lib/iomem.c +++ b/tools/lkl/lib/iomem.c @@ -56,13 +56,14 @@ void unregister_iomem(void *base) void *lkl_ioremap(long addr, int size) { int index = IOMEM_ADDR_TO_INDEX(addr); + int offset = IOMEM_ADDR_TO_OFFSET(addr); struct iomem_region *iomem = &iomem_regions[index]; if (index >= MAX_IOMEM_REGIONS) return NULL; - if (iomem->ops && size <= iomem->size) - return IOMEM_INDEX_TO_ADDR(index); + if (iomem->ops && offset + size <= iomem->size) + return (void *)addr; return NULL; } diff --git a/tools/lkl/lib/vfio_pci.c b/tools/lkl/lib/vfio_pci.c index 1d76793420ae18..f93c8d30e62d6e 100644 --- a/tools/lkl/lib/vfio_pci.c +++ b/tools/lkl/lib/vfio_pci.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -9,20 +10,35 @@ #include #include #include +#include #include #include #include #include +#include #include #include "iomem.h" +struct vfio_pci_msi_vector { + int irq; + int fd; +}; + struct lkl_pci_dev { struct lkl_sem *thread_init_sem; + int thread_init_status; int irq; lkl_thread_t int_thread; - int quit; + int int_thread_running; + int intx_quit; int fd; int irq_fd; + lkl_thread_t msi_thread; + int msi_thread_running; + int msi_quit; + int msi_type; + int msi_nvec; + struct vfio_pci_msi_vector *msi_vectors; struct vfio_device_info device_info; struct vfio_region_info config_reg; struct vfio_iommu_type1_dma_map dma_map; @@ -46,7 +62,8 @@ static struct lkl_pci_dev *vfio_pci_add(const char *name, void *kernel_ram, char path[128], link[128], *l; int segn, busn, devn, funcn; int i; - int container_fd = 0, group_fd = 0; + int container_fd = -1, group_fd = -1; + const char *step = NULL; struct vfio_group_status group_status = { .argsz = sizeof( group_status) }; struct vfio_iommu_type1_info iommu_info = { .argsz = sizeof( @@ -57,53 +74,70 @@ static struct lkl_pci_dev *vfio_pci_add(const char *name, void *kernel_ram, return NULL; memset(dev, 0, sizeof(*dev)); + dev->fd = -1; + dev->irq_fd = -1; dev->device_info.argsz = sizeof(struct vfio_device_info); dev->config_reg.argsz = sizeof(struct vfio_region_info); dev->dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map); container_fd = open("/dev/vfio/vfio", O_RDWR); - if (container_fd < 0) + if (container_fd < 0) { + step = "open /dev/vfio/vfio"; goto error; + } + step = "check api version / type1 extension"; if (ioctl(container_fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION || ioctl(container_fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0) goto error; - if (sscanf(name, "vfio%x:%x:%x.%x", &segn, &busn, &devn, &funcn) != 4) + if (sscanf(name, "vfio%x:%x:%x.%x", &segn, &busn, &devn, &funcn) != 4) { + step = "parse device name"; goto error; + } snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/iommu_group", segn, busn, devn, funcn); + step = "readlink iommu_group"; i = readlink(path, link, sizeof(link) - 1); if (i < 0) goto error; link[i] = '\0'; l = strrchr(link, '/'); - if (l == NULL) + if (l == NULL) { + step = "parse iommu_group link"; goto error; + } snprintf(path, sizeof(path), "/dev/vfio%s", l); + step = "open iommu group"; group_fd = open(path, O_RDWR); if (group_fd < 0) goto error; + step = "get group status"; if (ioctl(group_fd, VFIO_GROUP_GET_STATUS, &group_status) < 0) goto error; - if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + step = "group not viable"; goto error; + } + step = "set container"; if (ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container_fd) < 0) goto error; + step = "set iommu type1"; if (ioctl(container_fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU) < 0) goto error; + step = "get iommu info"; if (ioctl(container_fd, VFIO_IOMMU_GET_INFO, &iommu_info) < 0) goto error; @@ -116,49 +150,47 @@ static struct lkl_pci_dev *vfio_pci_add(const char *name, void *kernel_ram, dev->dma_map.iova = 0; dev->dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + step = "map dma"; if (ioctl(container_fd, VFIO_IOMMU_MAP_DMA, &dev->dma_map) < 0) goto error; } snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x", segn, busn, devn, funcn); + step = "get device fd"; dev->fd = ioctl(group_fd, VFIO_GROUP_GET_DEVICE_FD, path); if (dev->fd < 0) goto error; + step = "get device info"; if (ioctl(dev->fd, VFIO_DEVICE_GET_INFO, &dev->device_info) < 0) goto error; - if (dev->device_info.num_regions <= VFIO_PCI_CONFIG_REGION_INDEX) + if (dev->device_info.num_regions <= VFIO_PCI_CONFIG_REGION_INDEX) { + step = "no config region"; goto error; + } dev->config_reg.index = VFIO_PCI_CONFIG_REGION_INDEX; + step = "get config region info"; if (ioctl(dev->fd, VFIO_DEVICE_GET_REGION_INFO, &dev->config_reg) < 0) goto error; return dev; error: - lkl_printf("lkl_vfio_pci: failed to create a PCI device for %s\n", - name); - if (container_fd > 0) + lkl_printf("lkl_vfio_pci: failed to create a PCI device for %s: %s (%d: %s)\n", + name, step ? step : "unknown", errno, strerror(errno)); + if (container_fd >= 0) close(container_fd); - if (group_fd > 0) + if (group_fd >= 0) close(group_fd); free(dev); return NULL; } -static void vfio_pci_remove(struct lkl_pci_dev *dev) -{ - dev->quit = 1; - lkl_host_ops.thread_join(dev->int_thread); - close(dev->fd); - free(dev); -} - static int check_irq_status(struct lkl_pci_dev *dev) { unsigned short status; @@ -168,28 +200,29 @@ static int check_irq_status(struct lkl_pci_dev *dev) return (status & (1 << 3)) ? 1 : 0; } -/* Currently, we only support INTx. */ +static int vfio_pci_disable_irq_index(struct lkl_pci_dev *dev, + unsigned int index) +{ + struct vfio_irq_set irq_set = { + .argsz = sizeof(irq_set), + .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, + .index = index, + .start = 0, + .count = 0, + }; + + return ioctl(dev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); +} + static void vfio_int_thread(void *_dev) { eventfd_t icount; struct lkl_pci_dev *dev = (struct lkl_pci_dev *)_dev; struct timespec req = { 0, 1000 * 1000 }; - struct vfio_irq_info irq = { .argsz = sizeof(irq) }; struct vfio_irq_set *irq_set; char irq_set_buf[sizeof(struct vfio_irq_set) + sizeof(int)]; fd_set rfds; - if (dev->device_info.num_irqs <= VFIO_PCI_INTX_IRQ_INDEX) - goto init_error; - - irq.index = VFIO_PCI_INTX_IRQ_INDEX; - - if (ioctl(dev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq)) - goto init_error; - - if (irq.count != 1) - goto init_error; - irq_set = (struct vfio_irq_set *)irq_set_buf; irq_set->argsz = sizeof(irq_set_buf); irq_set->count = 1; @@ -205,18 +238,19 @@ static void vfio_int_thread(void *_dev) if (ioctl(dev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) goto init_error; + dev->thread_init_status = 0; lkl_host_ops.sem_up(dev->thread_init_sem); while (1) { /* We should wait until the driver actually handles * an interrupt by monitoring the PCI interrupt status bit. */ - while (check_irq_status(dev) && !dev->quit) { + while (check_irq_status(dev) && !dev->intx_quit) { lkl_trigger_irq(dev->irq); nanosleep(&req, NULL); } - if (dev->quit) + if (dev->intx_quit) return; /* unmask interrupts */ @@ -247,25 +281,43 @@ static void vfio_int_thread(void *_dev) goto handling_error; else break; - else if (dev->quit) + else if (dev->intx_quit) return; } } init_error: lkl_printf("lkl_vfio_pci: failed to setup INTx for a device\n"); + dev->thread_init_status = -1; + lkl_host_ops.sem_up(dev->thread_init_sem); return; handling_error: lkl_printf("lkl_vfio_pci: unknown error in the interrupt handler\n"); } -static int vfio_pci_irq_init(struct lkl_pci_dev *dev, int irq) +static int vfio_pci_start_intx(struct lkl_pci_dev *dev) { + struct vfio_irq_info irq = { .argsz = sizeof(irq) }; + + if (dev->device_info.num_irqs <= VFIO_PCI_INTX_IRQ_INDEX) + return 0; + + irq.index = VFIO_PCI_INTX_IRQ_INDEX; + + if (ioctl(dev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq) || irq.count != 1) + /* + * Some devices (e.g. QEMU's emulated NVMe) only support + * MSI/MSI-X and have no INTx. That is fine as long as the + * driver enables MSI/MSI-X, so do not fail device setup. + */ + return 0; + dev->thread_init_sem = lkl_host_ops.sem_alloc(0); if (!dev->thread_init_sem) return -1; - dev->irq = irq; + dev->thread_init_status = -1; + dev->intx_quit = 0; dev->int_thread = lkl_host_ops.thread_create(vfio_int_thread, (void *)dev); @@ -277,9 +329,281 @@ static int vfio_pci_irq_init(struct lkl_pci_dev *dev, int irq) /* wait until the interrupt handler thread is ready */ lkl_host_ops.sem_down(dev->thread_init_sem); lkl_host_ops.sem_free(dev->thread_init_sem); + dev->thread_init_sem = NULL; + + if (dev->thread_init_status < 0) { + lkl_host_ops.thread_join(dev->int_thread); + dev->int_thread = 0; + if (dev->irq_fd >= 0) { + close(dev->irq_fd); + dev->irq_fd = -1; + } + return -1; + } + + dev->int_thread_running = 1; + return 0; +} + +static void vfio_pci_stop_intx(struct lkl_pci_dev *dev) +{ + if (dev->int_thread_running) { + dev->intx_quit = 1; + lkl_host_ops.thread_join(dev->int_thread); + dev->int_thread = 0; + dev->int_thread_running = 0; + } + + if (dev->irq_fd >= 0) { + vfio_pci_disable_irq_index(dev, VFIO_PCI_INTX_IRQ_INDEX); + close(dev->irq_fd); + dev->irq_fd = -1; + } + + dev->intx_quit = 0; +} + +static int vfio_pci_irq_init(struct lkl_pci_dev *dev, int irq) +{ + dev->irq = irq; + return vfio_pci_start_intx(dev); +} + +static unsigned int vfio_pci_msi_index(int type) +{ + return type == LKL_PCI_IRQ_MSIX ? VFIO_PCI_MSIX_IRQ_INDEX : + VFIO_PCI_MSI_IRQ_INDEX; +} + +static void vfio_msi_thread(void *_dev) +{ + struct lkl_pci_dev *dev = (struct lkl_pci_dev *)_dev; + struct pollfd *pfds; + int *vectors; + int active = 0; + int i; + + for (i = 0; i < dev->msi_nvec; i++) + if (dev->msi_vectors[i].fd >= 0) + active++; + + pfds = calloc(active, sizeof(*pfds)); + vectors = calloc(active, sizeof(*vectors)); + if (!pfds || !vectors) { + dev->thread_init_status = -1; + lkl_host_ops.sem_up(dev->thread_init_sem); + free(pfds); + free(vectors); + return; + } + + for (i = 0, active = 0; i < dev->msi_nvec; i++) { + if (dev->msi_vectors[i].fd < 0) + continue; + pfds[active].fd = dev->msi_vectors[i].fd; + pfds[active].events = POLLIN; + vectors[active] = i; + active++; + } + + dev->thread_init_status = 0; + lkl_host_ops.sem_up(dev->thread_init_sem); + + while (!dev->msi_quit) { + int rc = poll(pfds, active, 100); + + if (rc < 0) + break; + if (!rc) + continue; + + for (i = 0; i < active; i++) { + eventfd_t icount; + int vector; + + if (!(pfds[i].revents & POLLIN)) + continue; + + if (read(pfds[i].fd, &icount, sizeof(icount)) < 0) + continue; + + vector = vectors[i]; + lkl_trigger_irq(dev->msi_vectors[vector].irq); + } + } + + free(pfds); + free(vectors); +} + +static void vfio_pci_free_msi_vectors(struct lkl_pci_dev *dev) +{ + int i; + + if (!dev->msi_vectors) + return; + + for (i = 0; i < dev->msi_nvec; i++) { + if (dev->msi_vectors[i].fd >= 0) + close(dev->msi_vectors[i].fd); + } + + free(dev->msi_vectors); + dev->msi_vectors = NULL; + dev->msi_nvec = 0; + dev->msi_type = 0; +} + +static void vfio_pci_stop_msi(struct lkl_pci_dev *dev, int restart_intx) +{ + if (!dev->msi_vectors) + return; + + if (dev->msi_thread_running) { + dev->msi_quit = 1; + lkl_host_ops.thread_join(dev->msi_thread); + dev->msi_thread = 0; + dev->msi_thread_running = 0; + } + + vfio_pci_disable_irq_index(dev, vfio_pci_msi_index(dev->msi_type)); + vfio_pci_free_msi_vectors(dev); + dev->msi_quit = 0; + + if (restart_intx && dev->irq > 0) + vfio_pci_start_intx(dev); +} + +static int vfio_pci_start_msi_thread(struct lkl_pci_dev *dev) +{ + dev->thread_init_sem = lkl_host_ops.sem_alloc(0); + if (!dev->thread_init_sem) + return -1; + + dev->thread_init_status = -1; + dev->msi_quit = 0; + + dev->msi_thread = lkl_host_ops.thread_create(vfio_msi_thread, + (void *)dev); + if (!dev->msi_thread) { + lkl_host_ops.sem_free(dev->thread_init_sem); + dev->thread_init_sem = NULL; + return -1; + } + + lkl_host_ops.sem_down(dev->thread_init_sem); + lkl_host_ops.sem_free(dev->thread_init_sem); + dev->thread_init_sem = NULL; + + if (dev->thread_init_status < 0) { + lkl_host_ops.thread_join(dev->msi_thread); + dev->msi_thread = 0; + return -1; + } + + dev->msi_thread_running = 1; return 0; } +static int vfio_pci_msi_init(struct lkl_pci_dev *dev, int type, int nvec, + int *irqs) +{ + unsigned int index = vfio_pci_msi_index(type); + struct vfio_irq_info irq = { .argsz = sizeof(irq), .index = index }; + struct vfio_irq_set *irq_set = NULL; + int *fds = NULL; + int active = 0; + int ret = -1; + int i; + + if (type != LKL_PCI_IRQ_MSI && type != LKL_PCI_IRQ_MSIX) + return -1; + if (nvec <= 0 || dev->msi_vectors) + return -1; + if (dev->device_info.num_irqs <= index) + return -1; + if (ioctl(dev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq)) + return -1; + + if (!(irq.flags & VFIO_IRQ_INFO_EVENTFD) || irq.count < (unsigned int)nvec) + return -1; + + dev->msi_vectors = calloc(nvec, sizeof(*dev->msi_vectors)); + fds = malloc(sizeof(*fds) * nvec); + irq_set = malloc(sizeof(*irq_set) + sizeof(*fds) * nvec); + if (!dev->msi_vectors || !fds || !irq_set) + goto out_free; + dev->msi_nvec = nvec; + + for (i = 0; i < nvec; i++) { + dev->msi_vectors[i].irq = irqs[i]; + dev->msi_vectors[i].fd = -1; + fds[i] = -1; + + if (irqs[i] <= 0) + continue; + + dev->msi_vectors[i].fd = eventfd(0, EFD_CLOEXEC); + if (dev->msi_vectors[i].fd < 0) + goto out_free; + + fds[i] = dev->msi_vectors[i].fd; + active++; + } + + if (!active) + goto out_free; + + vfio_pci_stop_intx(dev); + + irq_set->argsz = sizeof(*irq_set) + sizeof(*fds) * nvec; + irq_set->flags = + VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = index; + irq_set->start = 0; + irq_set->count = nvec; + memcpy(irq_set->data, fds, sizeof(*fds) * nvec); + + if (ioctl(dev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) + goto out_restart_intx; + + dev->msi_type = type; + + if (vfio_pci_start_msi_thread(dev)) + goto out_disable_msi; + + ret = 0; + goto out; + +out_disable_msi: + vfio_pci_disable_irq_index(dev, index); +out_restart_intx: + if (dev->irq > 0) + vfio_pci_start_intx(dev); +out_free: + vfio_pci_free_msi_vectors(dev); +out: + free(irq_set); + free(fds); + return ret; +} + +static void vfio_pci_msi_teardown(struct lkl_pci_dev *dev, int type) +{ + if (!dev->msi_vectors || dev->msi_type != type) + return; + + vfio_pci_stop_msi(dev, 1); +} + +static void vfio_pci_remove(struct lkl_pci_dev *dev) +{ + vfio_pci_stop_msi(dev, 0); + vfio_pci_stop_intx(dev); + close(dev->fd); + free(dev); +} + static unsigned long long vfio_map_page(struct lkl_pci_dev *dev, void *vaddr, unsigned long size) { @@ -291,6 +615,13 @@ static void vfio_unmap_page(struct lkl_pci_dev *dev, { } +struct vfio_pci_resource { + struct lkl_pci_dev *dev; + void *mmap_addr; + unsigned long size; + unsigned long offset; +}; + static int vfio_pci_read(struct lkl_pci_dev *dev, int where, int size, void *val) { @@ -305,7 +636,21 @@ static int vfio_pci_write(struct lkl_pci_dev *dev, int where, int size, static int pci_resource_read(void *data, int offset, void *res, int size) { - void *addr = data + offset; + struct vfio_pci_resource *resource = data; + unsigned long end = (unsigned long)offset + (unsigned long)size; + void *addr; + + if (offset < 0 || size <= 0) + return -LKL_EINVAL; + if (end > resource->size) + return -LKL_EINVAL; + + if (!resource->mmap_addr) + return pread(resource->dev->fd, res, size, + resource->offset + offset) == size ? 0 : + -LKL_EIO; + + addr = resource->mmap_addr + offset; switch (size) { case 8: @@ -328,7 +673,21 @@ static int pci_resource_read(void *data, int offset, void *res, int size) static int pci_resource_write(void *data, int offset, void *res, int size) { - void *addr = data + offset; + struct vfio_pci_resource *resource = data; + unsigned long end = (unsigned long)offset + (unsigned long)size; + void *addr; + + if (offset < 0 || size <= 0) + return -LKL_EINVAL; + if (end > resource->size) + return -LKL_EINVAL; + + if (!resource->mmap_addr) + return pwrite(resource->dev->fd, res, size, + resource->offset + offset) == size ? 0 : + -LKL_EIO; + + addr = resource->mmap_addr + offset; switch (size) { case 8: @@ -366,6 +725,7 @@ static void *vfio_resource_alloc(struct lkl_pci_dev *dev, VFIO_PCI_BAR4_REGION_INDEX, VFIO_PCI_BAR5_REGION_INDEX, }; struct vfio_region_info reg = { .argsz = sizeof(reg) }; + struct vfio_pci_resource *resource; void *mmio_addr; if ((unsigned int)resource_index >= ARRAY_SIZE(region_index_list)) @@ -384,19 +744,36 @@ static void *vfio_resource_alloc(struct lkl_pci_dev *dev, if (reg.size < resource_size) return NULL; + resource = malloc(sizeof(*resource)); + if (!resource) + return NULL; + mmio_addr = mmap(NULL, resource_size, PROT_READ | PROT_WRITE, MAP_SHARED, dev->fd, reg.offset); - if (mmio_addr == MAP_FAILED) - return NULL; + mmio_addr = NULL; + + resource->dev = dev; + resource->mmap_addr = mmio_addr; + resource->size = resource_size; + resource->offset = reg.offset; + + mmio_addr = register_iomem(resource, resource_size, &pci_resource_ops); + if (!mmio_addr) { + if (resource->mmap_addr) + munmap(resource->mmap_addr, resource_size); + free(resource); + } - return register_iomem(mmio_addr, resource_size, &pci_resource_ops); + return mmio_addr; } struct lkl_dev_pci_ops vfio_pci_ops = { .add = vfio_pci_add, .remove = vfio_pci_remove, .irq_init = vfio_pci_irq_init, + .msi_init = vfio_pci_msi_init, + .msi_teardown = vfio_pci_msi_teardown, .read = vfio_pci_read, .write = vfio_pci_write, .resource_alloc = vfio_resource_alloc, diff --git a/tools/lkl/scripts/qemu-linux-make-images.sh b/tools/lkl/scripts/qemu-linux-make-images.sh index 4051e20b99624e..6c5cab5b9fc828 100755 --- a/tools/lkl/scripts/qemu-linux-make-images.sh +++ b/tools/lkl/scripts/qemu-linux-make-images.sh @@ -15,6 +15,11 @@ chpasswd: { expire: False } groups: sudo ssh_pwauth: True shell: /bin/bash +write_files: + - path: /etc/security/limits.d/99-lkl.conf + content: | + lkl soft memlock unlimited + lkl hard memlock unlimited EOF cloud-localds cloud.img cloud.txt rm cloud.txt diff --git a/tools/lkl/tests/boot.c b/tools/lkl/tests/boot.c index 3d727f30e52f06..f7b7d4fa0a360e 100644 --- a/tools/lkl/tests/boot.c +++ b/tools/lkl/tests/boot.c @@ -688,10 +688,11 @@ static int lkl_test_kunit_pci(void) char *log = strdup(boot_log); char *line = NULL; int n; + char c; line = strtok(log, "\n"); while (line) { - if (sscanf(line, "[ %*f] ok %d lkl_pci", &n) == 1) { + if (sscanf(line, "[ %*f] ok %d lkl_pci%c", &n, &c) == 1) { lkl_test_logf("%s", line); free(log); return TEST_SUCCESS; @@ -706,6 +707,31 @@ static int lkl_test_kunit_pci(void) } #endif // LKL_HOST_CONFIG_LKL_PCI_KUNIT_TEST +#ifdef LKL_HOST_CONFIG_LKL_PCI_MSI_KUNIT_TEST +static int lkl_test_kunit_pci_msi(void) +{ + char *log = strdup(boot_log); + char *line = NULL; + int n; + char c; + + line = strtok(log, "\n"); + while (line) { + if (sscanf(line, "[ %*f] ok %d lkl_pci_msi%c", &n, &c) == 1) { + lkl_test_logf("%s", line); + free(log); + return TEST_SUCCESS; + } + + line = strtok(NULL, "\n"); + } + + free(log); + + return TEST_FAILURE; +} +#endif // LKL_HOST_CONFIG_LKL_PCI_MSI_KUNIT_TEST + #define CMD_LINE "mem=32M loglevel=8 " static int lkl_test_start_kernel(void) @@ -778,6 +804,9 @@ struct lkl_test tests[] = { #endif #ifdef LKL_HOST_CONFIG_LKL_PCI_KUNIT_TEST LKL_TEST(kunit_pci), +#endif +#ifdef LKL_HOST_CONFIG_LKL_PCI_MSI_KUNIT_TEST + LKL_TEST(kunit_pci_msi), #endif LKL_TEST(stop_kernel), }; diff --git a/tools/lkl/tests/disk-vfio-pci.c b/tools/lkl/tests/disk-vfio-pci.c index 41d6b2bc9695af..8779f59bf2e5d5 100644 --- a/tools/lkl/tests/disk-vfio-pci.c +++ b/tools/lkl/tests/disk-vfio-pci.c @@ -2,9 +2,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -27,17 +29,86 @@ static char bootparams[128]; #define min(a, b) (a < b ? a : b) +static int is_nvme_disk(const char *name) +{ + return !strncmp(name, "nvme", 4) && !strchr(name, 'p'); +} + +static int find_nvme_blkdev(unsigned int *dev, int log_missing) +{ + char buf[4096]; + char *line, *saveptr = NULL; + int fd, err; + + fd = lkl_sys_open("/proc/partitions", LKL_O_RDONLY, 0); + if (fd < 0) + return fd; + + err = lkl_sys_read(fd, buf, sizeof(buf) - 1); + lkl_sys_close(fd); + if (err < 0) + return err; + + buf[err] = '\0'; + line = strtok_r(buf, "\n", &saveptr); + while (line) { + unsigned int major, minor; + unsigned long long blocks; + char name[32]; + + if (sscanf(line, " %u %u %llu %31s", &major, &minor, + &blocks, name) == 4 && is_nvme_disk(name)) { + *dev = LKL_MKDEV(major, minor); + lkl_test_logf("using /dev/%s (%u:%u)\n", + name, major, minor); + return 0; + } + + line = strtok_r(NULL, "\n", &saveptr); + } + + if (log_missing) + lkl_test_logf("no NVMe disk found in /proc/partitions:\n%s", + buf); + return -LKL_ENODEV; +} + +static int wait_for_nvme_blkdev(unsigned int *dev) +{ + int i, err; + + err = lkl_mount_fs("proc"); + if (err < 0) { + lkl_test_logf("mount proc failed: %s\n", lkl_strerror(err)); + return err; + } + + for (i = 0; i < 100; i++) { + err = find_nvme_blkdev(dev, i == 99); + if (!err) + return 0; + usleep(100000); + } + + return err; +} + static int lkl_test_blkdev(void) { char dev_str[] = { "/dev/xxxxxxxx" }; char buffer[64*1024]; uint64_t size, read = 0; + unsigned int dev; int err; int fd; - snprintf(dev_str, sizeof(dev_str), "/dev/%08x", LKL_MKDEV(259, 0)); + err = wait_for_nvme_blkdev(&dev); + if (err < 0) + return TEST_FAILURE; + + snprintf(dev_str, sizeof(dev_str), "/dev/%08x", dev); - err = lkl_sys_mknod(dev_str, LKL_S_IFBLK | 0600, LKL_MKDEV(259, 0)); + err = lkl_sys_mknod(dev_str, LKL_S_IFBLK | 0600, dev); if (err < 0) { lkl_test_logf("mknod failed: %s\n", lkl_strerror(err)); return TEST_FAILURE; @@ -89,8 +160,20 @@ int main(int argc, const char **argv) if (parse_args(argc, argv, args) < 0) return -1; +#ifdef RLIMIT_MEMLOCK + /* + * VFIO pins the whole LKL memory region with VFIO_IOMMU_MAP_DMA, + * which is accounted against RLIMIT_MEMLOCK. Raise it if the + * hard limit allows, otherwise the DMA map fails and no PCI + * device is created. + */ + struct rlimit rlim = { RLIM_INFINITY, RLIM_INFINITY }; + + setrlimit(RLIMIT_MEMLOCK, &rlim); +#endif + snprintf(bootparams, sizeof(bootparams), - "mem=16M loglevel=8 lkl_pci=vfio%s", cla.pciname); + "mem=128M loglevel=8 lkl_pci=vfio%s", cla.pciname); lkl_host_ops.print = lkl_test_log; diff --git a/tools/lkl/tests/memory-report b/tools/lkl/tests/memory-report new file mode 100644 index 00000000000000..e69de29bb2d1d6