From 0b4694bcc6e087cf297157b4adc337460ed025a5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 14 Nov 2018 09:38:28 -0700 Subject: [PATCH 1776/2944] nvme: provide optimized poll function for separate poll queues to #28991349 commit dabcefab45d36ecb5a22f16577bb0f298876a22d upstream If we have separate poll queues, we know that they aren't using interrupts. Hence we don't need to disable interrupts around finding completions. Provide a separate set of blk_mq_ops for such devices. Signed-off-by: Jens Axboe Signed-off-by: Xiaoguang Wang Reviewed-by: Joseph Qi --- drivers/nvme/host/pci.c | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 2b23428..0283d9c 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1112,6 +1112,23 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) return __nvme_poll(nvmeq, tag); } +static int nvme_poll_noirq(struct blk_mq_hw_ctx *hctx, unsigned int tag) +{ + struct nvme_queue *nvmeq = hctx->driver_data; + u16 start, end; + bool found; + + if (!nvme_cqe_pending(nvmeq)) + return 0; + + spin_lock(&nvmeq->cq_lock); + found = nvme_process_cq(nvmeq, &start, &end, tag); + spin_unlock(&nvmeq->cq_lock); + + nvme_complete_cqes(nvmeq, start, end); + return found; +} + static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl) { struct nvme_dev *dev = to_nvme_dev(ctrl); @@ -1607,16 +1624,24 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) .timeout = nvme_timeout, }; +#define NVME_SHARED_MQ_OPS \ + .queue_rq = nvme_queue_rq, \ + .commit_rqs = nvme_commit_rqs, \ + .rq_flags_to_type = nvme_rq_flags_to_type, \ + .complete = nvme_pci_complete_rq, \ + .init_hctx = nvme_init_hctx, \ + .init_request = nvme_init_request, \ + .map_queues = nvme_pci_map_queues, \ + .timeout = nvme_timeout \ + static const struct blk_mq_ops nvme_mq_ops = { - .queue_rq = nvme_queue_rq, - .commit_rqs = nvme_commit_rqs, - .rq_flags_to_type = nvme_rq_flags_to_type, - .complete = nvme_pci_complete_rq, - .init_hctx = nvme_init_hctx, - .init_request = nvme_init_request, - .map_queues = nvme_pci_map_queues, - .timeout = nvme_timeout, - .poll = nvme_poll, + NVME_SHARED_MQ_OPS, + .poll = nvme_poll, +}; + +static const struct blk_mq_ops nvme_mq_poll_noirq_ops = { + NVME_SHARED_MQ_OPS, + .poll = nvme_poll_noirq, }; static void nvme_dev_remove_admin(struct nvme_dev *dev) @@ -2300,7 +2325,11 @@ static int nvme_dev_add(struct nvme_dev *dev) int ret; if (!dev->ctrl.tagset) { - dev->tagset.ops = &nvme_mq_ops; + if (!dev->io_queues[NVMEQ_TYPE_POLL]) + dev->tagset.ops = &nvme_mq_ops; + else + dev->tagset.ops = &nvme_mq_poll_noirq_ops; + dev->tagset.nr_hw_queues = dev->online_queues - 1; dev->tagset.nr_maps = NVMEQ_TYPE_NR; dev->tagset.timeout = NVME_IO_TIMEOUT; -- 1.8.3.1