From ffb1d1ec0bbcc38c03c505c0a9a83c2543c9da87 Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Fri, 11 Dec 2020 22:38:56 +0800 Subject: [PATCH 2446/2944] io_uring: fix IOPOLL -EAGAIN retries to #31739384 commit eefdf30f3dcb5c1d47bee2b3afdb9d4d05343ff3 upstream Comments from Xiaoguang Wang: If we process polled io in io-wq, we still may get -EAGAIN error due to request starvation, in this case, current codes call kiocb_done() against req, which will set req's iopoll_completed to be 1 and req's result to -EAGAIN, then if there is another context call io_do_iopoll, we'll re-queue this req to io-wq, though previous context that calls kiocb_done() is still running. Retrying a request more than once at the same time by io-wq can lead to weird results in the io-wq handling (and other spots). See 56450c20fe10 ("io_uring: clear req->result on IOPOLL re-issue") commit message. This normally isn't hit, as polling is mostly done on NVMe with deep queue depths. But if we do run into request starvation, we need to ensure that retries are properly serialized. Reported-by: Andres Freund Signed-off-by: Jens Axboe Signed-off-by: Xiaoguang Wang Acked-by: Joseph Qi --- fs/io_uring.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4edd079..5b036a3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1139,7 +1139,7 @@ static inline void io_prep_async_work(struct io_kiocb *req, io_req_init_async(req); if (req->flags & REQ_F_ISREG) { - if (def->hash_reg_file) + if (def->hash_reg_file || (req->ctx->flags & IORING_SETUP_IOPOLL)) io_wq_hash_work(&req->work, file_inode(req->file)); } else { if (def->unbound_nonreg_file) @@ -2937,6 +2937,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock) struct kiocb *kiocb = &req->rw.kiocb; struct iov_iter iter; size_t iov_count; + unsigned long nr_segs; ssize_t io_size, ret; bool no_async; @@ -2944,6 +2945,9 @@ static int io_read(struct io_kiocb *req, bool force_nonblock) if (ret < 0) return ret; + iov_count = iov_iter_count(&iter); + nr_segs = iter.nr_segs; + /* Ensure we clear previously set non-block flag */ if (!force_nonblock) kiocb->ki_flags &= ~IOCB_NOWAIT; @@ -2961,21 +2965,22 @@ static int io_read(struct io_kiocb *req, bool force_nonblock) if (no_async) goto copy_iov; - iov_count = iov_iter_count(&iter); ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count); if (!ret) { - unsigned long nr_segs = iter.nr_segs; ssize_t ret2 = 0; ret2 = io_iter_do_read(req, &iter); /* Catch -EAGAIN return for forced non-blocking submission */ if (!force_nonblock || (ret2 != -EAGAIN && ret2 != -EIO)) { + /* IOPOLL retry should happen for io-wq threads */ + if ((req->ctx->flags & IORING_SETUP_IOPOLL) && (ret2 == -EAGAIN)) + goto copy_iov; kiocb_done(kiocb, ret2); } else { +copy_iov: iter.count = iov_count; iter.nr_segs = nr_segs; -copy_iov: ret = io_setup_async_rw(req, io_size, iovec, inline_vecs, &iter); if (ret) @@ -3044,12 +3049,16 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) struct kiocb *kiocb = &req->rw.kiocb; struct iov_iter iter; size_t iov_count; + unsigned long nr_segs; ssize_t ret, io_size; ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock); if (ret < 0) return ret; + iov_count = iov_iter_count(&iter); + nr_segs = iter.nr_segs; + /* Ensure we clear previously set non-block flag */ if (!force_nonblock) req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT; @@ -3071,10 +3080,8 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) (req->flags & REQ_F_ISREG)) goto copy_iov; - iov_count = iov_iter_count(&iter); ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count); if (!ret) { - unsigned long nr_segs = iter.nr_segs; ssize_t ret2; /* @@ -3112,11 +3119,14 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) ret2 = -EAGAIN; if (!force_nonblock || ret2 != -EAGAIN) { + /* IOPOLL retry should happen for io-wq threads */ + if ((req->ctx->flags & IORING_SETUP_IOPOLL) && (ret2 == -EAGAIN)) + goto copy_iov; kiocb_done(kiocb, ret2); } else { +copy_iov: iter.count = iov_count; iter.nr_segs = nr_segs; -copy_iov: ret = io_setup_async_rw(req, io_size, iovec, inline_vecs, &iter); if (ret) -- 1.8.3.1