summaryrefslogtreecommitdiff
path: root/io_uring/tctx.h
diff options
context:
space:
mode:
authorPavel Begunkov <asml.silence@gmail.com>2022-06-20 17:27:35 +0300
committerJens Axboe <axboe@kernel.dk>2022-07-25 03:39:15 +0300
commit4a0fef62788b69df09267c8e3f3f11d4bb9d50e7 (patch)
treeac71025cad11239d4ecc1c31bb8c2af811954fb2 /io_uring/tctx.h
parentbce5d70cd64a5d48aff613334b8a5fac450b9753 (diff)
downloadlinux-4a0fef62788b69df09267c8e3f3f11d4bb9d50e7.tar.xz
io_uring: optimize io_uring_task layout
task_work bits of io_uring_task are split into two cache lines causing extra cache bouncing, place them into a separate cache line. Also move the most used submission path fields closer together, so there are hot. Cc: stable@vger.kernel.org # 5.15+ Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring/tctx.h')
-rw-r--r--io_uring/tctx.h34
1 files changed, 18 insertions, 16 deletions
diff --git a/io_uring/tctx.h b/io_uring/tctx.h
index dde82ce4d8e2..dead0ed00429 100644
--- a/io_uring/tctx.h
+++ b/io_uring/tctx.h
@@ -7,22 +7,24 @@
struct io_uring_task {
/* submission side */
- int cached_refs;
- struct xarray xa;
- struct wait_queue_head wait;
- const struct io_ring_ctx *last;
- struct io_wq *io_wq;
- struct percpu_counter inflight;
- atomic_t inflight_tracked;
- atomic_t in_idle;
-
- spinlock_t task_lock;
- struct io_wq_work_list task_list;
- struct io_wq_work_list prio_task_list;
- struct callback_head task_work;
- bool task_running;
-
- struct file *registered_rings[IO_RINGFD_REG_MAX];
+ int cached_refs;
+ const struct io_ring_ctx *last;
+ struct io_wq *io_wq;
+ struct file *registered_rings[IO_RINGFD_REG_MAX];
+
+ struct xarray xa;
+ struct wait_queue_head wait;
+ atomic_t in_idle;
+ atomic_t inflight_tracked;
+ struct percpu_counter inflight;
+
+ struct { /* task_work */
+ spinlock_t task_lock;
+ bool task_running;
+ struct io_wq_work_list task_list;
+ struct io_wq_work_list prio_task_list;
+ struct callback_head task_work;
+ } ____cacheline_aligned_in_smp;
};
struct io_tctx_node {