From 24fa832369e10f2578f706bbbe912c54cf3c49f5 Mon Sep 17 00:00:00 2001 From: Minwoo Im Date: Mon, 9 Dec 2024 17:44:50 +0900 Subject: [PATCH] nvme: add support for CMB (Controller Memory Buffer) CMB provides device memory buffer for driver context such as SQ/CQ and data buffer. nvme_configure_cmb() initializes CMB registers in the nvme controller and nvme_discard_cmb() will disable it and destroy the cmb attributes in @ctrl->cmb. Signed-off-by: Minwoo Im --- examples/cmb-p2p.c | 65 +++-------------------------------- include/vfn/nvme/ctrl.h | 34 ++++++++++++++++++ src/nvme/core.c | 76 +++++++++++++++++++++++++++++++++++++++++ src/nvme/types.h | 41 ++++++++++++++++++++++ 4 files changed, 156 insertions(+), 60 deletions(-) diff --git a/examples/cmb-p2p.c b/examples/cmb-p2p.c index 7513752..d780722 100644 --- a/examples/cmb-p2p.c +++ b/examples/cmb-p2p.c @@ -54,70 +54,11 @@ static const struct nvme_ctrl_opts ctrl_opts = { .ncqr = 63, }; -static inline off_t bar_offset(int bar) -{ - return PCI_BASE_ADDRESS_0 + bar * 4; -} - -static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *iova, size_t *len) -{ - uint64_t cap, szu, ofst, hwaddr; - uint32_t cmbloc, cmbsz, v32; - int bir; - void *cmb; - - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); - - if (NVME_GET(cap, CAP_CMBS)) - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(0x1)); - - cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ)); - cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); - - szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); - *len = szu * NVME_GET(cmbsz, CMBSZ_SZ); - - ofst = szu * NVME_GET(cmbloc, CMBLOC_OFST); - bir = NVME_GET(cmbloc, CMBLOC_BIR); - - printf("cmb bir is %d\n", bir); - - if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir)) < 0) - err(1, "failed to read pci config"); - - hwaddr = v32 & PCI_BASE_ADDRESS_MEM_MASK; - - if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir + 1)) < 0) - err(1, "failed to read pci config"); - - hwaddr |= (uint64_t)v32 << 32; - - printf("cmb bar is mapped at physical address 0x%lx\n", hwaddr); - - /* map the cmb */ - cmb = vfio_pci_map_bar(&ctrl->pci, bir, *len, ofst, PROT_READ | PROT_WRITE); - if (!cmb) - err(1, "failed to map cmb"); - - if (iommu_map_vaddr(__iommu_ctx(ctrl), cmb, *len, iova, 0x0)) - err(1, "failed to map cmb in iommu (try VFN_IOMMU_FORCE_VFIO=1)"); - - if (NVME_GET(cap, CAP_CMBS)) { - printf("assigned cmb base address is 0x%lx\n", *iova); - - /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*iova | 0x3)); - } - - return cmb; -} - int main(int argc, char **argv) { struct nvme_ctrl src = {}, dst = {}; uint64_t iova; - size_t len; void *cmb; union nvme_cmd cmd = {}; @@ -141,7 +82,11 @@ int main(int argc, char **argv) if (nvme_init(&dst, bdfs[1], &ctrl_opts)) err(1, "failed to initialize destination nvme controller"); - cmb = nvme_configure_cmb(&dst, &iova, &len); + if (nvme_configure_cmb(&dst)) + err(1, "failed to initialize cmb to destination nvme controller"); + + iova = dst.cmb.iova; + cmb = dst.cmb.vaddr; cmd.identify = (struct nvme_cmd_identify) { .opcode = nvme_admin_identify, diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index 664263b..f0df449 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -106,6 +106,16 @@ struct nvme_ctrl { * See &enum nvme_ctrl_feature_flags. */ unsigned long flags; + + /** + * @cmb: CMB atribute initialized by nvme_configure_cmb() + */ + struct { + int bar; + void *vaddr; + uint64_t iova; + size_t size; + } cmb; }; /** @@ -340,4 +350,28 @@ void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq); */ void nvme_discard_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq); +/** + * nvme_configure_cmb - Initialize Controller Memory Buffer + * @ctrl: See &struct nvme_ctrl + * + * Enable CMB in the @ctrl and initialize @ctrl->cmb members. It first maps + * a vaddr to the CMB BAR memory region and map it to IOMMU. And the mapped + * iova will be configured as CBA(CMB Base Address) to the register. + * + * **Note** iommufd-backed IOMMU does not support BAR vaddr to be mapped to the + * IOMMU page table. vfio-backed IOMMU should be used. + * + * Return: On success, returns ``0``. On error, returns ``-1`` and sets + * ``errno``. + */ +int nvme_configure_cmb(struct nvme_ctrl *ctrl); + +/** + * nvme_discard_cmb - Disable CMB and discard cmb instance @ctrl->cmb + * @ctrl: See &struct nvme_ctrl + * + * Disable CMB in the @ctrl and discard @ctrl->cmb members. + */ +void nvme_discard_cmb(struct nvme_ctrl *ctrl); + #endif /* LIBVFN_NVME_CTRL_H */ diff --git a/src/nvme/core.c b/src/nvme/core.c index 2465191..449c1f2 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -735,6 +735,8 @@ void nvme_close(struct nvme_ctrl *ctrl) free(ctrl->cq); + nvme_discard_cmb(ctrl); + if (ctrl->dbbuf.doorbells) { struct iommu_ctx *ctx = __iommu_ctx(ctrl); size_t len; @@ -755,3 +757,77 @@ void nvme_close(struct nvme_ctrl *ctrl) memset(ctrl, 0x0, sizeof(*ctrl)); } + +int nvme_configure_cmb(struct nvme_ctrl *ctrl) +{ + uint64_t cmbmsc; + uint32_t cmbloc; + uint32_t cmbsz; + uint64_t cap; + int bar; + + cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + if (!NVME_FIELD_GET(cap, CAP_CMBS)) + return 0; + + cmbmsc = NVME_FIELD_SET(1, CMBMSC_CRE); + mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc)); + + /* + * Read CMBMSC back here to guarantee that the previous CMBMSC write to + * be flushed before reading CMBLOC register avoiding reordering. + */ + cmbmsc = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CMBMSC)); + cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); + bar = NVME_FIELD_GET(cmbloc, CMBLOC_BIR); + + cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ)); + + ctrl->cmb.bar = bar; + ctrl->cmb.size = nvme_cmb_size(cmbsz); + ctrl->cmb.vaddr = vfio_pci_map_bar(&ctrl->pci, bar, + ctrl->cmb.size, NVME_FIELD_GET(cmbloc, CMBLOC_OFST), + PROT_READ | PROT_WRITE); + if (!ctrl->cmb.vaddr) { + log_debug("could not map bar %d vaddr\n", bar); + return -1; + } + + if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->cmb.vaddr, + ctrl->cmb.size, &ctrl->cmb.iova, 0x0)) { + log_debug("could not map bar vaddr to iommu\n"); + return -1; + } + + cmbmsc |= NVME_FIELD_SET(1, CMBMSC_CMSE) | + (ctrl->cmb.iova >> NVME_CMBMSC_CBA_SHIFT) << + NVME_CMBMSC_CBA_SHIFT; + + mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc)); + + log_debug("cmb initialized (bar=%d, iova=%#lx, vaddr=%p, size=%#lx)\n", + bar, ctrl->cmb.iova, ctrl->cmb.vaddr, ctrl->cmb.size); + return 0; +} + +void nvme_discard_cmb(struct nvme_ctrl *ctrl) +{ + uint64_t cmbmsc; + uint32_t cmbloc; + + if (!ctrl->cmb.vaddr) + return; + + cmbmsc = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CMBMSC)); + cmbmsc &= ~(1 << NVME_CMBMSC_CMSE_SHIFT); + cmbmsc &= ~(NVME_CMBMSC_CBA_MASK << NVME_CMBMSC_CBA_SHIFT); + mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc)); + + iommu_unmap_vaddr(__iommu_ctx(ctrl), ctrl->cmb.vaddr, NULL); + + cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); + vfio_pci_unmap_bar(&ctrl->pci, ctrl->cmb.bar, ctrl->cmb.vaddr, + ctrl->cmb.size, NVME_FIELD_GET(cmbloc, CMBLOC_OFST)); + + memset(&ctrl->cmb, 0, sizeof(ctrl->cmb)); +} diff --git a/src/nvme/types.h b/src/nvme/types.h index 91ef66a..d3c51a2 100644 --- a/src/nvme/types.h +++ b/src/nvme/types.h @@ -27,6 +27,9 @@ enum nvme_reg { NVME_REG_AQA = 0x0024, NVME_REG_ASQ = 0x0028, NVME_REG_ACQ = 0x0030, + NVME_REG_CMBLOC = 0x0038, + NVME_REG_CMBSZ = 0x003c, + NVME_REG_CMBMSC = 0x0050, }; enum nvme_cap { @@ -42,6 +45,8 @@ enum nvme_cap { NVME_CAP_MPSMIN_MASK = 0xf, NVME_CAP_MPSMAX_SHIFT = 52, NVME_CAP_MPSMAX_MASK = 0xf, + NVME_CAP_CMBS_SHIFT = 57, + NVME_CAP_CMBS_MASK = 0x1, NVME_CAP_CSS_CSI = 1 << 6, NVME_CAP_CSS_ADMIN = 1 << 7, @@ -75,6 +80,42 @@ enum nvme_csts { NVME_CSTS_RDY_MASK = 0x1, }; +enum nvme_cmbloc { + NVME_CMBLOC_BIR_SHIFT = 0, + NVME_CMBLOC_BIR_MASK = 0x7, + NVME_CMBLOC_OFST_SHIFT = 12, + NVME_CMBLOC_OFST_MASK = 0xfffff, +}; + +enum nvme_cmbsz { + NVME_CMBSZ_SZU_SHIFT = 8, + NVME_CMBSZ_SZU_MASK = 0xf, + NVME_CMBSZ_SZ_SHIFT = 12, + NVME_CMBSZ_SZ_MASK = 0xfffff, + NVME_CMBSZ_SZU_4K = 0, + NVME_CMBSZ_SZU_64K = 1, + NVME_CMBSZ_SZU_1M = 2, + NVME_CMBSZ_SZU_16M = 3, + NVME_CMBSZ_SZU_256M = 4, + NVME_CMBSZ_SZU_4G = 5, + NVME_CMBSZ_SZU_64G = 6, +}; + +static inline __u64 nvme_cmb_size(__u32 cmbsz) +{ + return ((__u64)NVME_FIELD_GET(cmbsz, CMBSZ_SZ)) * + (1ULL << (12 + 4 * NVME_FIELD_GET(cmbsz, CMBSZ_SZU))); +} + +enum nvme_cmbmsc { + NVME_CMBMSC_CRE_SHIFT = 0, + NVME_CMBMSC_CRE_MASK = 0x1, + NVME_CMBMSC_CMSE_SHIFT = 1, + NVME_CMBMSC_CMSE_MASK = 0x1, + NVME_CMBMSC_CBA_SHIFT = 12, +}; +static const __u64 NVME_CMBMSC_CBA_MASK = 0xfffffffffffffull; + enum nvme_feat { NVME_FEAT_NRQS_NSQR_SHIFT = 0, NVME_FEAT_NRQS_NSQR_MASK = 0xffff,