summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPavel Begunkov <asml.silence@gmail.com>2022-07-12 23:52:40 +0300
committerJens Axboe <axboe@kernel.dk>2022-07-25 03:41:06 +0300
commite58d498e81baa9fd8acf5132d8b2d4f829361f6b (patch)
treeaddc514ac82d4325ee3fa2dc2f0e7b0aa9ca97bb
parenteb4a299b2f95437af6183946c2a2e850621cefdb (diff)
downloadlinux-e58d498e81baa9fd8acf5132d8b2d4f829361f6b.tar.xz
io_uring: complete notifiers in tw
We need a task context to post CQEs but using wq is too expensive. Try to complete notifiers using task_work and fall back to wq if fails. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/089799ab665b10b78fdc614ae6d59fa7ef0d5f91.1657643355.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--io_uring/notif.c22
-rw-r--r--io_uring/notif.h3
2 files changed, 22 insertions, 3 deletions
diff --git a/io_uring/notif.c b/io_uring/notif.c
index b257db2120b4..aec74f88fc33 100644
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -13,6 +13,11 @@ static void __io_notif_complete_tw(struct callback_head *cb)
struct io_notif *notif = container_of(cb, struct io_notif, task_work);
struct io_ring_ctx *ctx = notif->ctx;
+ if (likely(notif->task)) {
+ io_put_task(notif->task, 1);
+ notif->task = NULL;
+ }
+
io_cq_lock(ctx);
io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true);
@@ -43,6 +48,14 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
if (!refcount_dec_and_test(&uarg->refcnt))
return;
+
+ if (likely(notif->task)) {
+ init_task_work(&notif->task_work, __io_notif_complete_tw);
+ if (likely(!task_work_add(notif->task, &notif->task_work,
+ TWA_SIGNAL)))
+ return;
+ }
+
INIT_WORK(&notif->commit_work, io_notif_complete_wq);
queue_work(system_unbound_wq, &notif->commit_work);
}
@@ -134,12 +147,15 @@ __cold int io_notif_unregister(struct io_ring_ctx *ctx)
for (i = 0; i < ctx->nr_notif_slots; i++) {
struct io_notif_slot *slot = &ctx->notif_slots[i];
- if (slot->notif)
- io_notif_slot_flush(slot);
+ if (!slot->notif)
+ continue;
+ if (WARN_ON_ONCE(slot->notif->task))
+ slot->notif->task = NULL;
+ io_notif_slot_flush(slot);
}
kvfree(ctx->notif_slots);
ctx->notif_slots = NULL;
ctx->nr_notif_slots = 0;
return 0;
-} \ No newline at end of file
+}
diff --git a/io_uring/notif.h b/io_uring/notif.h
index b23c9c0515bb..23ca7620fff9 100644
--- a/io_uring/notif.h
+++ b/io_uring/notif.h
@@ -11,6 +11,9 @@ struct io_notif {
struct ubuf_info uarg;
struct io_ring_ctx *ctx;
+ /* complete via tw if ->task is non-NULL, fallback to wq otherwise */
+ struct task_struct *task;
+
/* cqe->user_data, io_notif_slot::tag if not overridden */
u64 tag;
/* see struct io_notif_slot::seq */