From 9895c8dede32f6909ab04be60229a53a6a7f073a Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Wed, 23 Dec 2020 14:11:20 +0800 Subject: [PATCH 2483/2944] io_uring: hold uring_lock while completing failed polled io in io_wq_submit_work() fix #32050414 commit c07e6719511e77c4b289f62bfe96423eb6ea061d linux-block io_iopoll_complete() does not hold completion_lock to complete polled io, so in io_wq_submit_work(), we can not call io_req_complete() directly, to complete polled io, otherwise there maybe concurrent access to cqring, defer_list, etc, which is not safe. Commit dad1b1242fd5 ("io_uring: always let io_iopoll_complete() complete polled io") has fixed this issue, but Pavel reported that IOPOLL apart from rw can do buf reg/unreg requests( IORING_OP_PROVIDE_BUFFERS or IORING_OP_REMOVE_BUFFERS), so the fix is not good. Given that io_iopoll_complete() is always called under uring_lock, so here for polled io, we can also get uring_lock to fix this issue. Fixes: dad1b1242fd5 ("io_uring: always let io_iopoll_complete() complete polled io") Cc: # 5.5+ Signed-off-by: Xiaoguang Wang Reviewed-by: Pavel Begunkov [axboe: don't deref 'req' after completing it'] Signed-off-by: Jens Axboe Reviewed-by: Joseph Qi --- fs/io_uring.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b19873e..0510be2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5848,20 +5848,28 @@ static void io_wq_submit_work(struct io_wq_work **workptr) } if (ret) { + struct io_ring_ctx *lock_ctx = NULL; + + if (req->ctx->flags & IORING_SETUP_IOPOLL) + lock_ctx = req->ctx; + /* - * io_iopoll_complete() does not hold completion_lock to complete - * polled io, so here for polled io, just mark it done and still let - * io_iopoll_complete() complete it. + * io_iopoll_complete() does not hold completion_lock to + * complete polled io, so here for polled io, we can not call + * io_req_complete() directly, otherwise there maybe concurrent + * access to cqring, defer_list, etc, which is not safe. Given + * that io_iopoll_complete() is always called under uring_lock, + * so here for polled io, we also get uring_lock to complete it. */ - if (req->ctx->flags & IORING_SETUP_IOPOLL) { - struct kiocb *kiocb = &req->rw.kiocb; + if (lock_ctx) + mutex_lock(&lock_ctx->uring_lock); - kiocb_done(kiocb, ret); - } else { - req_set_fail_links(req); - io_cqring_add_event(req, ret); - io_put_req(req); - } + req_set_fail_links(req); + io_cqring_add_event(req, ret); + io_put_req(req); + + if (lock_ctx) + mutex_unlock(&lock_ctx->uring_lock); } io_steal_work(req, workptr); -- 1.8.3.1