From 8a7898262f509af316d424fcd06bd7a8eb5e1ffa Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Sat, 26 Dec 2020 14:36:05 +0800 Subject: [PATCH 2486/2944] io_uring: allow non-fixed files with SQPOLL task #32115874 commit 28cea78af44918b920306df150afbd116bd94301 upstream Commets from Xiaoguang Wang: Make some code changes according to 4.19 code base. The restriction of needing fixed files for SQPOLL is problematic, and prevents/inhibits several valid uses cases. With the referenced files_struct that we have now, it's trivially supportable. Treat ->files like we do the mm for the SQPOLL thread - grab a reference to it (and assign it), and drop it when we're done. This feature is exposed as IORING_FEAT_SQPOLL_NONFIXED. Signed-off-by: Jens Axboe Signed-off-by: Xiaoguang Wang Reviewed-by: Joseph Qi --- fs/io_uring.c | 123 ++++++++++++++++++++++++++++++++++-------- include/uapi/linux/io_uring.h | 1 + 2 files changed, 103 insertions(+), 21 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0510be2..cf2eb65 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -272,6 +272,11 @@ struct io_ring_ctx { struct io_wq *io_wq; struct task_struct *sqo_thread; /* if using sq thread polling */ struct mm_struct *sqo_mm; + /* + * For SQPOLL usage - we hold a reference to the parent task, so we + * have access to the ->files + */ + struct task_struct *sqo_task; wait_queue_head_t *sqo_wait; wait_queue_head_t __sqo_wait; @@ -2038,14 +2043,51 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res) __io_cqring_add_event(req, res, cflags); } -static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx) +static void io_sq_thread_drop_mm_files(struct io_ring_ctx *ctx) { + struct files_struct *files = current->files; struct mm_struct *mm = current->mm; if (mm) { unuse_mm(mm); mmput(mm); } + + if (files) { + struct nsproxy *nsproxy = current->nsproxy; + + task_lock(current); + current->files = NULL; + current->nsproxy = NULL; + task_unlock(current); + put_files_struct(files); + put_nsproxy(nsproxy); + } +} + +static int __io_sq_thread_acquire_files(struct io_ring_ctx *ctx) +{ + if (!current->files) { + struct files_struct *files; + struct nsproxy *nsproxy; + + task_lock(ctx->sqo_task); + files = ctx->sqo_task->files; + if (!files) { + task_unlock(ctx->sqo_task); + return -EOWNERDEAD; + } + atomic_inc(&files->count); + get_nsproxy(ctx->sqo_task->nsproxy); + nsproxy = ctx->sqo_task->nsproxy; + task_unlock(ctx->sqo_task); + + task_lock(current); + current->files = files; + current->nsproxy = nsproxy; + task_unlock(current); + } + return 0; } static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) @@ -2059,12 +2101,24 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) return 0; } -static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx, - struct io_kiocb *req) +static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, + struct io_kiocb *req) { - if (!io_op_defs[req->opcode].needs_mm) - return 0; - return __io_sq_thread_acquire_mm(ctx); + int ret; + + if (io_op_defs[req->opcode].needs_mm) { + ret = __io_sq_thread_acquire_mm(ctx); + if (unlikely(ret)) + return ret; + } + + if (io_op_defs[req->opcode].needs_file) { + ret = __io_sq_thread_acquire_files(ctx); + if (unlikely(ret)) + return ret; + } + + return 0; } #ifdef CONFIG_BLOCK @@ -2123,7 +2177,7 @@ static void io_rw_resubmit(struct callback_head *cb) __set_current_state(TASK_RUNNING); - err = io_sq_thread_acquire_mm(ctx, req); + err = io_sq_thread_acquire_mm_files(ctx, req); if (io_resubmit_prep(req, err)) { refcount_inc(&req->refs); @@ -4728,7 +4782,7 @@ static void io_async_task_func(struct callback_head *cb) if (!canceled) { __set_current_state(TASK_RUNNING); - if (io_sq_thread_acquire_mm(ctx, req)) { + if (io_sq_thread_acquire_mm_files(ctx, req)) { io_cqring_add_event(req, -EFAULT); goto end_req; } @@ -5923,13 +5977,7 @@ static int io_file_get(struct io_submit_state *state, struct io_kiocb *req, static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req, int fd) { - bool fixed; - - fixed = (req->flags & REQ_F_FIXED_FILE) != 0; - if (unlikely(!fixed && io_async_submit(req->ctx))) - return -EBADF; - - return io_file_get(state, req, fd, &req->file, fixed); + return io_file_get(state, req, fd, &req->file, req->flags & REQ_F_FIXED_FILE); } static int io_grab_files(struct io_kiocb *req) @@ -6334,7 +6382,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; - if (unlikely(io_sq_thread_acquire_mm(ctx, req))) + if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) return -EFAULT; sqe_flags = READ_ONCE(sqe->flags); @@ -6458,12 +6506,19 @@ static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) static int io_sq_thread(void *data) { struct io_ring_ctx *ctx = data; + struct files_struct *old_files = current->files; + struct nsproxy *old_nsproxy = current->nsproxy; const struct cred *old_cred; mm_segment_t old_fs; DEFINE_WAIT(wait); unsigned long timeout; int ret = 0; + task_lock(current); + current->files = NULL; + current->nsproxy = NULL; + task_unlock(current); + complete(&ctx->sq_thread_comp); old_fs = get_fs(); @@ -6498,7 +6553,7 @@ static int io_sq_thread(void *data) * adding ourselves to the waitqueue, as the unuse/drop * may sleep. */ - io_sq_thread_drop_mm(ctx); + io_sq_thread_drop_mm_files(ctx); /* * We're polling. If we're within the defined idle @@ -6570,9 +6625,14 @@ static int io_sq_thread(void *data) task_work_run(); set_fs(old_fs); - io_sq_thread_drop_mm(ctx); + io_sq_thread_drop_mm_files(ctx); revert_creds(old_cred); + task_lock(current); + current->files = old_files; + current->nsproxy = old_nsproxy; + task_unlock(current); + kthread_parkme(); return 0; @@ -6629,11 +6689,18 @@ static int io_sq_thread_percpu(void *data) { struct sq_thread_percpu *t = data; struct io_ring_ctx *ctx; + struct files_struct *old_files = current->files; + struct nsproxy *old_nsproxy = current->nsproxy; const struct cred *old_cred = NULL; mm_segment_t old_fs; DEFINE_WAIT(wait); unsigned long timeout = 0; + task_lock(current); + current->files = NULL; + current->nsproxy = NULL; + task_unlock(current); + old_fs = get_fs(); set_fs(USER_DS); while (!kthread_should_stop()) { @@ -6684,7 +6751,7 @@ static int io_sq_thread_percpu(void *data) if (sqt_spin) timeout = jiffies + t->sq_thread_idle; else - io_sq_thread_drop_mm(ctx); + io_sq_thread_drop_mm_files(ctx); continue; } @@ -6719,10 +6786,15 @@ static int io_sq_thread_percpu(void *data) task_work_run(); set_fs(old_fs); - io_sq_thread_drop_mm(ctx); + io_sq_thread_drop_mm_files(ctx); if (old_cred) revert_creds(old_cred); + task_lock(current); + current->files = old_files; + current->nsproxy = old_nsproxy; + task_unlock(current); + kthread_parkme(); return 0; } @@ -7992,6 +8064,11 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->sqo_mm) mmdrop(ctx->sqo_mm); + if (ctx->sqo_task) { + put_task_struct(ctx->sqo_task); + ctx->sqo_task = NULL; + } + io_iopoll_reap_events(ctx); io_sqe_buffer_unregister(ctx); io_sqe_files_unregister(ctx); @@ -8667,6 +8744,9 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->user = user; ctx->creds = get_current_cred(); + get_task_struct(current); + ctx->sqo_task = current; + ret = io_allocate_scq_urings(ctx, p); if (ret) goto err; @@ -8696,7 +8776,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | - IORING_FEAT_POLL_32BITS | IORING_FEAT_EXT_ARG; + IORING_FEAT_POLL_32BITS | IORING_FEAT_EXT_ARG | + IORING_FEAT_SQPOLL_NONFIXED; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 56495b0..f9a3221 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -254,6 +254,7 @@ struct io_uring_params { #define IORING_FEAT_CUR_PERSONALITY (1U << 4) #define IORING_FEAT_FAST_POLL (1U << 5) #define IORING_FEAT_POLL_32BITS (1U << 6) +#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) #define IORING_FEAT_EXT_ARG (1U << 8) /* -- 1.8.3.1