From f4e3d2b8e6c7a119916417c9950cc36fd6abce32 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Nov 2019 13:05:32 -0700 Subject: [PATCH 1039/2944] io_uring: close lookup gap for dependent next work to #26323578 commit b76da70fc3759df13e0991706451f1a2e06ba19e upstream. When we find new work to process within the work handler, we queue the linked timeout before we have issued the new work. This can be problematic for very short timeouts, as we have a window where the new work isn't visible. Allow the work handler to store a callback function for this in the work item, and flag it with IO_WQ_WORK_CB if the caller has done so. If that is set, then io-wq will call the callback when it has setup the new work item. Reported-by: Pavel Begunkov Signed-off-by: Jens Axboe Signed-off-by: Joseph Qi Acked-by: Xiaoguang Wang --- fs/io-wq.c | 3 +++ fs/io-wq.h | 6 +++++- fs/io_uring.c | 16 ++++++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 49ab908..7051273 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -435,6 +435,9 @@ static void io_worker_handle_work(struct io_worker *worker) worker->cur_work = work; spin_unlock_irq(&worker->lock); + if (work->flags & IO_WQ_WORK_CB) + work->func(&work); + if ((work->flags & IO_WQ_WORK_NEEDS_FILES) && current->files != work->files) { task_lock(current); diff --git a/fs/io-wq.h b/fs/io-wq.h index 6a97a46..ce955eb 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -11,6 +11,7 @@ enum { IO_WQ_WORK_NEEDS_FILES = 16, IO_WQ_WORK_UNBOUND = 32, IO_WQ_WORK_INTERNAL = 64, + IO_WQ_WORK_CB = 128, IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ }; @@ -22,7 +23,10 @@ enum io_wq_cancel { }; struct io_wq_work { - struct list_head list; + union { + struct list_head list; + void *data; + }; void (*func)(struct io_wq_work **); unsigned flags; struct files_struct *files; diff --git a/fs/io_uring.c b/fs/io_uring.c index aeee433..3d7b611 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2681,6 +2681,15 @@ static int __io_submit_sqe(struct io_kiocb *req, struct io_kiocb **nxt, return 0; } +static void io_link_work_cb(struct io_wq_work **workptr) +{ + struct io_wq_work *work = *workptr; + struct io_kiocb *link = work->data; + + io_queue_linked_timeout(link); + work->func = io_wq_submit_work; +} + static void io_wq_submit_work(struct io_wq_work **workptr) { struct io_wq_work *work = *workptr; @@ -2727,8 +2736,11 @@ static void io_wq_submit_work(struct io_wq_work **workptr) io_prep_async_work(nxt, &link); *workptr = &nxt->work; - if (link) - io_queue_linked_timeout(link); + if (link) { + nxt->work.flags |= IO_WQ_WORK_CB; + nxt->work.func = io_link_work_cb; + nxt->work.data = link; + } } } -- 1.8.3.1