nvme: export nvme_configure_[sq|cq]

There are two reasons why we have these two functions in public.

(1) high-level APIs that using these two are not considering interrupt
event support (e.g., MSI-X) for the completion.  For example,
nvme_create_io[sq|cq]() APIs call __admin() internal helper to issue an
admin command simply and it waits for the CQ entry by spinning on CQ by
default even if device enables the interrupt and application thread
might have fetched the cq entry while racing with this API.  To prevent
this conflicts, application should be able to choose how to reap the
CQ entry, not just relying on the libvfn high-level API.

(2) SQ and CQ are major driver context and they can be located in the
device memory such as CMB.  In this case, application might want to
initialize sq and cq instances with a queue memory which application
might have already allocated one.

To configure sq and cq instances much more flexible, this patch exported
them to public by adding @vaddr argument to give callers have a chance
to configure their own queue address space.  @vaddr can be given as NULL
and they will allocate one by default as it was, otherwise it will just
initialize sq and cq instances with the given @vaddr.

Signed-off-by: Minwoo Im <minwoo.im@samsung.com>

include/vfn/nvme/ctrl.h

@@ -265,6 +265,43 @@ int nvme_reset(struct nvme_ctrl *ctrl);
  */
 int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags);
 
+/**
+ * nvme_configure_sq - Initialize a SQ instance whose qid is given @qid
+ * @ctrl: See &struct nvme_ctrl
+ * @vaddr: Queue virtual address (can be NULL)
+ * @qid: Queue identifier
+ * @qsize: Queue size
+ * @cq: Associated I/O Completion Queue
+ * @flags: See &enum nvme_create_iosq_flags
+ *
+ * Initialize SQ instance attributes and doorbell.  It does not issue Create
+ * I/O SQ admin commmand to controller, instead it just initializes SQ
+ * instance.  If @vaddr is given as ``NULL``, it will allocate one, otherwise
+ * @vaddr should already be mapped to iommu.
+ *
+ * Return: ``0`` on success, ``-1`` on error and set ``errno``.
+ */
+int nvme_configure_sq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize,
+		      struct nvme_cq *cq, unsigned long flags);
+
+/**
+ * nvme_configure_cq - Initialize a CQ instance whose qid is given @qid
+ * @ctrl: See &struct nvme_ctrl
+ * @vaddr: Queue virtual address (can be NULL)
+ * @qid: Queue identifier
+ * @qsize: Queue size
+ * @vector: interrupt vector
+ *
+ * Initialize CQ instance attributes and doorbell.  It does not issue Create
+ * I/O CQ admin commmand to controller, instead it just initializes CQ
+ * instance.  If @vaddr is given as ``NULL``, it will allocate one, otherwise
+ * @vaddr should already be mapped to iommu.
+ *
+ * Return: ``0`` on success, ``-1`` on error and set ``errno``.
+ */
+int nvme_configure_cq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize,
+		      int vector);
+
 /**
  * nvme_enable - Enable controller
  * @ctrl: Controller to enable

src/nvme/core.c

@@ -110,12 +110,11 @@ int nvme_del_ctrl(struct nvme_ctrl *ctrl)
 	return 0;
 }
 
-static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector)
+int nvme_configure_cq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize, int vector)
 {
 	struct nvme_cq *cq = &ctrl->cq[qid];
 	uint64_t cap;
 	uint8_t dstrd;
-	size_t len;
 
 	cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP));
 	dstrd = NVME_FIELD_GET(cap, CAP_DSTRD);
@@ -149,13 +148,25 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec
 		cq->dbbuf.eventidx = cqhdbl(ctrl->dbbuf.eventidxs, qid, dstrd);
 	}
 
-	len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES);
+	if (vaddr) {
+		uint64_t iova;
 
-	if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) {
-		log_debug("failed to map vaddr\n");
+		if (iommu_translate_vaddr(__iommu_ctx(ctrl), vaddr, &iova)) {
+			log_debug("failed to translate vaddr\n");
+			return -1;
+		}
 
-		pgunmap(cq->vaddr, len);
-		return -1;
+		cq->vaddr = vaddr;
+		cq->iova = iova;
+	} else {
+		size_t len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES);
+
+		if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) {
+			log_debug("failed to map vaddr\n");
+
+			pgunmap(cq->vaddr, len);
+			return -1;
+		}
 	}
 
 	return 0;
@@ -181,8 +192,8 @@ void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq)
 	memset(cq, 0x0, sizeof(*cq));
 }
 
-static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize,
-			     struct nvme_cq *cq, unsigned long UNUSED flags)
+int nvme_configure_sq(struct nvme_ctrl *ctrl, void *vaddr, int qid, int qsize,
+		      struct nvme_cq *cq, unsigned long UNUSED flags)
 {
 	struct nvme_sq *sq = &ctrl->sq[qid];
 	uint64_t cap;
@@ -251,13 +262,25 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize,
 			rq->rq_next = &sq->rqs[i - 1];
 	}
 
-	len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES);
-	if (len < 0)
-		goto free_sq_rqs;
+	if (vaddr) {
+		uint64_t iova;
 
-	if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) {
-		log_debug("failed to map vaddr\n");
-		goto unmap_sq;
+		if (iommu_translate_vaddr(__iommu_ctx(ctrl), vaddr, &iova)) {
+			log_debug("failed to translate vaddr\n");
+			return -1;
+		}
+
+		sq->vaddr = vaddr;
+		sq->iova = iova;
+	} else {
+		len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES);
+		if (len < 0)
+			goto free_sq_rqs;
+
+		if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) {
+			log_debug("failed to map vaddr\n");
+			goto unmap_sq;
+		}
 	}
 
 	return 0;
@@ -309,12 +332,12 @@ int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags)
 	struct nvme_cq *cq = &ctrl->cq[NVME_AQ];
 	struct nvme_sq *sq = &ctrl->sq[NVME_AQ];
 
-	if (nvme_configure_cq(ctrl, NVME_AQ, NVME_AQ_QSIZE, 0)) {
+	if (nvme_configure_cq(ctrl, NULL, NVME_AQ, NVME_AQ_QSIZE, 0)) {
 		log_debug("failed to configure admin completion queue\n");
 		return -1;
 	}
 
-	if (nvme_configure_sq(ctrl, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) {
+	if (nvme_configure_sq(ctrl, NULL, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) {
 		log_debug("failed to configure admin submission queue\n");
 		goto discard_cq;
 	}
@@ -349,7 +372,7 @@ int nvme_create_iocq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector)
 	uint16_t qflags = NVME_Q_PC;
 	uint16_t iv = 0;
 
-	if (nvme_configure_cq(ctrl, qid, qsize, vector)) {
+	if (nvme_configure_cq(ctrl, NULL, qid, qsize, vector)) {
 		log_debug("could not configure io completion queue\n");
 		return -1;
 	}
@@ -391,7 +414,7 @@ int nvme_create_iosq(struct nvme_ctrl *ctrl, int qid, int qsize, struct nvme_cq
 	struct nvme_sq *sq = &ctrl->sq[qid];
 	union nvme_cmd cmd;
 
-	if (nvme_configure_sq(ctrl, qid, qsize, cq, flags)) {
+	if (nvme_configure_sq(ctrl, NULL, qid, qsize, cq, flags)) {
 		log_debug("could not configure io submission queue\n");
 		return -1;
 	}