From 48f9a1c0d29cd4f7d83e57742eae016cc3f5ad5e Mon Sep 17 00:00:00 2001 From: Jeffle Xu Date: Fri, 25 Sep 2020 19:04:00 +0800 Subject: [PATCH 2285/2944] alinux: dm: add support for IO polling Design of cookie is initially constrained as a per-bio concept. It dosn't work well when bio-split needed, and it is really an issue when adding support for iopoll for dm devices. The current algorithm implementation is simple. The returned cookie of dm device is actually not used since it is just the cookie of one of the cloned bios. Polling of dm device is actually polling on all hardware queues (in poll mode) of all underlying target devices. Signed-off-by: Jeffle Xu Reviewed-by: Joseph Qi --- drivers/md/dm-core.h | 1 + drivers/md/dm-table.c | 31 +++++++++++++++++++ drivers/md/dm.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index 7e426e4..34721a5 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -150,4 +150,5 @@ static inline bool dm_message_test_buffer_overflow(char *result, unsigned maxlen extern wait_queue_head_t dm_global_eventq; void dm_issue_global_event(void); +int dm_io_poll(struct request_queue *q, blk_qc_t cookie, bool spin); #endif diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 980a3c3..a3adb3a 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1932,6 +1932,31 @@ static bool dm_table_requires_stable_pages(struct dm_table *t) return false; } +static int device_supports_poll(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) +{ + struct request_queue *q = bdev_get_queue(dev->bdev); + + return q && test_bit(QUEUE_FLAG_POLL, &q->queue_flags); +} + +static bool dm_table_supports_poll(struct dm_table *t) +{ + struct dm_target *ti; + unsigned int i; + + /* Ensure that all targets support iopoll. */ + for (i = 0; i < dm_table_get_num_targets(t); i++) { + ti = dm_table_get_target(t, i); + + if (!ti->type->iterate_devices || + !ti->type->iterate_devices(ti, device_supports_poll, NULL)) + return false; + } + + return true; +} + void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, struct queue_limits *limits) { @@ -2011,6 +2036,12 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, */ if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random)) blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q); + + if (dm_table_supports_poll(t)) { + q->poll_fn = dm_io_poll; + blk_queue_flag_set(QUEUE_FLAG_POLL, q); + } + } unsigned int dm_table_get_num_targets(struct dm_table *t) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index b6dd74e..13150f1 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1765,6 +1765,88 @@ static int dm_any_congested(void *congested_data, int bdi_bits) return r; } +static int do_dm_io_poll(struct request_queue *q, blk_qc_t cookie); + +static int dm_poll_one_dev(struct request_queue *q, blk_qc_t cookie) +{ + /* + * Iterate polling on all polling queues for mq device, while + * calling .poll_fn() recursively if the underlying device is + * actually another dm device on the next level of the device + * stack. + * Be noted that @q can't be a single-queue (sq) device here. + * q->queue_flags will not be set QUEUE_FLAG_POLL during device + * initialization, and thus dm_io_poll() won't be called if one + * of the underlying devices is actually a sq device. + */ + if (q->mq_ops) { + struct blk_mq_hw_ctx *hctx; + int i, ret = 0; + + if (!percpu_ref_tryget(&q->q_usage_counter)) + return 0; + + queue_for_each_poll_hw_ctx(q, hctx, i) + ret += q->mq_ops->poll(hctx, -1); + + percpu_ref_put(&q->q_usage_counter); + return ret; + } else + return do_dm_io_poll(q, cookie); +} + +static int do_dm_io_poll(struct request_queue *q, blk_qc_t cookie) +{ + struct mapped_device *md = q->queuedata; + struct dm_table *table; + struct dm_dev_internal *dd; + int srcu_idx; + int ret = 0; + + table = dm_get_live_table(md, &srcu_idx); + if (!table) + goto out; + + list_for_each_entry(dd, dm_table_get_devices(table), list) + ret += dm_poll_one_dev(bdev_get_queue(dd->dm_dev->bdev), cookie); +out: + dm_put_live_table(md, srcu_idx); + return ret; +} + +int dm_io_poll(struct request_queue *q, blk_qc_t cookie, bool spin) +{ + long state; + + /* + * Hybrid polling is not supported for dm device currently. + * And the following code is just a duplicate copy from + * blk_mq_poll(). + */ + state = current->state; + while (!need_resched()) { + int ret; + + ret = do_dm_io_poll(q, cookie); + if (ret > 0) { + set_current_state(TASK_RUNNING); + return ret; + } + + if (signal_pending_state(state, current)) + set_current_state(TASK_RUNNING); + + if (current->state == TASK_RUNNING) + return 1; + if (ret < 0 || !spin) + break; + cpu_relax(); + } + + __set_current_state(TASK_RUNNING); + return 0; +} + /*----------------------------------------------------------------- * An IDR is used to keep track of allocated minor numbers. *---------------------------------------------------------------*/ -- 1.8.3.1