summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
authorPavel Begunkov <asml.silence@gmail.com>2023-01-23 17:37:19 +0300
committerJens Axboe <axboe@kernel.dk>2023-01-30 01:17:41 +0300
commit50470fc5723ae0adb2f429a8b27ff6bf1a41913e (patch)
tree42773f2952bf87ce42ab1d4e4dfdb3fbb52daa32 /io_uring
parentcb6bf7f285c270d7808807128b5bce414e3f254a (diff)
downloadlinux-50470fc5723ae0adb2f429a8b27ff6bf1a41913e.tar.xz
io_uring: return normal tw run linking optimisation
io_submit_flush_completions() may produce new task_work items, so it's a good idea to recheck the task_work list after flushing completions. The optimisation is not new and was accidentially removed by f88262e60bb9 ("io_uring: lockless task list") Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/a7ed5ede84de190832cc33ebbcdd6e91cd90f5b6.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/io_uring.c9
1 files changed, 9 insertions, 0 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 55101013f3ee..9c92ca081c11 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1238,6 +1238,15 @@ void tctx_task_work(struct callback_head *cb)
loops++;
node = io_llist_xchg(&tctx->task_list, &fake);
count += handle_tw_list(node, &ctx, &uring_locked, &fake);
+
+ /* skip expensive cmpxchg if there are items in the list */
+ if (READ_ONCE(tctx->task_list.first) != &fake)
+ continue;
+ if (uring_locked && !wq_list_empty(&ctx->submit_state.compl_reqs)) {
+ io_submit_flush_completions(ctx);
+ if (READ_ONCE(tctx->task_list.first) != &fake)
+ continue;
+ }
node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL);
} while (node != &fake);