diff options
author | Jens Axboe <axboe@kernel.dk> | 2024-01-04 22:21:08 +0300 |
---|---|---|
committer | Greg Kroah-Hartman <gregkh@linuxfoundation.org> | 2024-01-26 02:35:45 +0300 |
commit | 0241f4c2caad6f5ec50dc935eac7bdf7f4ee84d0 (patch) | |
tree | 05b937c998395c11a63d460854590c52b90d7950 | |
parent | c239b77ea4d6771e65476e31231b87d0b22fde0c (diff) | |
download | linux-0241f4c2caad6f5ec50dc935eac7bdf7f4ee84d0.tar.xz |
io_uring: ensure local task_work is run on wait timeout
commit 6ff1407e24e6fdfa4a16ba9ba551e3d253a26391 upstream.
A previous commit added an earlier break condition here, which is fine if
we're using non-local task_work as it'll be run on return to userspace.
However, if DEFER_TASKRUN is used, then we could be leaving local
task_work that is ready to process in the ctx list until next time that
we enter the kernel to wait for events.
Move the break condition to _after_ we have run task_work.
Cc: stable@vger.kernel.org
Fixes: 846072f16eed ("io_uring: mimimise io_cqring_wait_schedule")
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-rw-r--r-- | io_uring/io_uring.c | 14 |
1 files changed, 12 insertions, 2 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b406cc6e3955..1cc4c7b05949 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2630,8 +2630,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, __set_current_state(TASK_RUNNING); atomic_set(&ctx->cq_wait_nr, 0); - if (ret < 0) - break; /* * Run task_work after scheduling and before io_should_wake(). * If we got woken because of task_work being processed, run it @@ -2641,6 +2639,18 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, if (!llist_empty(&ctx->work_llist)) io_run_local_work(ctx); + /* + * Non-local task_work will be run on exit to userspace, but + * if we're using DEFER_TASKRUN, then we could have waited + * with a timeout for a number of requests. If the timeout + * hits, we could have some requests ready to process. Ensure + * this break is _after_ we have run task_work, to avoid + * deferring running potentially pending requests until the + * next time we wait for events. + */ + if (ret < 0) + break; + check_cq = READ_ONCE(ctx->check_cq); if (unlikely(check_cq)) { /* let the caller flush overflows, retry */ |