From 037eb1540ae111b9d52e87ab6b8eb218bbffe4e5 Mon Sep 17 00:00:00 2001 From: Minwoo Im Date: Mon, 9 Dec 2024 18:07:17 +0900 Subject: [PATCH] nvme: export nvme_configure_[sq|cq] There are two reasons why we have these two functions in public. (1) high-level APIs that using these two are not considering interrupt event support (e.g., MSI-X) for the completion. For example, nvme_create_io[sq|cq]() APIs call __admin() internal helper to issue an admin command simply and it waits for the CQ entry by spinning on CQ by default even if device enables the interrupt and application thread might have fetched the cq entry while racing with this API. To prevent this conflicts, application should be able to choose how to reap the CQ entry, not just relying on the libvfn high-level API. (2) SQ and CQ are major driver context and they can be located in the device memory such as CMB. In this case, application might want to initialize sq and cq instances with a queue memory which application might have already allocated one. To configure sq and cq instances much more flexible, this patch exported them to public by adding @vaddr argument to give callers have a chance to configure their own queue address space. @vaddr can be given as NULL and they will allocate one by default as it was, otherwise it will just initialize sq and cq instances with the given @vaddr. Signed-off-by: Minwoo Im --- include/vfn/nvme/ctrl.h | 37 +++++++++++++++++++++++++ src/nvme/core.c | 61 ++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 19 deletions(-) diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index a28063b..9f60866 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -265,6 +265,43 @@ int nvme_reset(struct nvme_ctrl *ctrl); */ int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags); +/** + * nvme_configure_sq - Initialize a SQ instance whose qid is given @qid + * @ctrl: See &struct nvme_ctrl + * @vaddr: Queue virtual address (can be NULL) + * @qid: Queue identifier + * @qsize: Queue size + * @cq: Associated I/O Completion Queue + * @flags: See &enum nvme_create_iosq_flags + * + * Initialize SQ instance attributes and doorbell. It does not issue Create + * I/O SQ admin commmand to controller, instead it just initializes SQ + * instance. If @vaddr is given as ``NULL``, it will allocate one, otherwise + * @vaddr should already be mapped to iommu. + * + * Return: ``0`` on success, ``-1`` on error and set ``errno``. + */ +int nvme_configure_sq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize, + struct nvme_cq *cq, unsigned long flags); + +/** + * nvme_configure_cq - Initialize a CQ instance whose qid is given @qid + * @ctrl: See &struct nvme_ctrl + * @vaddr: Queue virtual address (can be NULL) + * @qid: Queue identifier + * @qsize: Queue size + * @vector: interrupt vector + * + * Initialize CQ instance attributes and doorbell. It does not issue Create + * I/O CQ admin commmand to controller, instead it just initializes CQ + * instance. If @vaddr is given as ``NULL``, it will allocate one, otherwise + * @vaddr should already be mapped to iommu. + * + * Return: ``0`` on success, ``-1`` on error and set ``errno``. + */ +int nvme_configure_cq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize, + int vector); + /** * nvme_enable - Enable controller * @ctrl: Controller to enable diff --git a/src/nvme/core.c b/src/nvme/core.c index 3348b71..aefd49b 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -110,12 +110,11 @@ int nvme_del_ctrl(struct nvme_ctrl *ctrl) return 0; } -static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) +int nvme_configure_cq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize, int vector) { struct nvme_cq *cq = &ctrl->cq[qid]; uint64_t cap; uint8_t dstrd; - size_t len; cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); @@ -149,13 +148,25 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec cq->dbbuf.eventidx = cqhdbl(ctrl->dbbuf.eventidxs, qid, dstrd); } - len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES); + if (vaddr) { + uint64_t iova; - if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) { - log_debug("failed to map vaddr\n"); + if (iommu_translate_vaddr(__iommu_ctx(ctrl), vaddr, &iova)) { + log_debug("failed to translate vaddr\n"); + return -1; + } - pgunmap(cq->vaddr, len); - return -1; + cq->vaddr = vaddr; + cq->iova = iova; + } else { + size_t len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES); + + if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) { + log_debug("failed to map vaddr\n"); + + pgunmap(cq->vaddr, len); + return -1; + } } return 0; @@ -181,8 +192,8 @@ void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq) memset(cq, 0x0, sizeof(*cq)); } -static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, - struct nvme_cq *cq, unsigned long UNUSED flags) +int nvme_configure_sq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize, + struct nvme_cq *cq, unsigned long UNUSED flags) { struct nvme_sq *sq = &ctrl->sq[qid]; uint64_t cap; @@ -251,13 +262,25 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, rq->rq_next = &sq->rqs[i - 1]; } - len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES); - if (len < 0) - goto free_sq_rqs; + if (vaddr) { + uint64_t iova; - if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) { - log_debug("failed to map vaddr\n"); - goto unmap_sq; + if (iommu_translate_vaddr(__iommu_ctx(ctrl), vaddr, &iova)) { + log_debug("failed to translate vaddr\n"); + return -1; + } + + sq->vaddr = vaddr; + sq->iova = iova; + } else { + len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES); + if (len < 0) + goto free_sq_rqs; + + if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) { + log_debug("failed to map vaddr\n"); + goto unmap_sq; + } } return 0; @@ -309,12 +332,12 @@ int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags) struct nvme_cq *cq = &ctrl->cq[NVME_AQ]; struct nvme_sq *sq = &ctrl->sq[NVME_AQ]; - if (nvme_configure_cq(ctrl, NVME_AQ, NVME_AQ_QSIZE, 0)) { + if (nvme_configure_cq(ctrl, NULL, NVME_AQ, NVME_AQ_QSIZE, 0)) { log_debug("failed to configure admin completion queue\n"); return -1; } - if (nvme_configure_sq(ctrl, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) { + if (nvme_configure_sq(ctrl, NULL, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) { log_debug("failed to configure admin submission queue\n"); goto discard_cq; } @@ -349,7 +372,7 @@ int nvme_create_iocq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) uint16_t qflags = NVME_Q_PC; uint16_t iv = 0; - if (nvme_configure_cq(ctrl, qid, qsize, vector)) { + if (nvme_configure_cq(ctrl, NULL, qid, qsize, vector)) { log_debug("could not configure io completion queue\n"); return -1; } @@ -391,7 +414,7 @@ int nvme_create_iosq(struct nvme_ctrl *ctrl, int qid, int qsize, struct nvme_cq struct nvme_sq *sq = &ctrl->sq[qid]; union nvme_cmd cmd; - if (nvme_configure_sq(ctrl, qid, qsize, cq, flags)) { + if (nvme_configure_sq(ctrl, NULL, qid, qsize, cq, flags)) { log_debug("could not configure io submission queue\n"); return -1; }