diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2022-06-17 11:48:05 +0300 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-07-25 03:39:14 +0300 |
| commit | b3659a65be70eb68d9fc9802c4ce81e0f943abfd (patch) | |
| tree | 4384da90a16b4e7940e3acac5afd3b3a01a72cb7 | |
| parent | e8c328c3913d381bf60f2aecdf350c04b5b7e67d (diff) | |
| download | linux-b3659a65be70eb68d9fc9802c4ce81e0f943abfd.tar.xz | |
io_uring: change ->cqe_cached invariant for CQE32
With IORING_SETUP_CQE32 ->cqe_cached doesn't store a real address but
rather an implicit offset into cqes. Store the real cqe pointer and
increment it accordingly if CQE32.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/1ee1838cba16bed96381a006950b36ba640d998c.1655455613.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | io_uring/io_uring.c | 15 | ||||
| -rw-r--r-- | io_uring/io_uring.h | 8 |
2 files changed, 12 insertions, 11 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 7acb94c180b8..0dbf6a74f9f3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -701,11 +701,8 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); - unsigned int shift = 0; unsigned int free, queued, len; - if (ctx->flags & IORING_SETUP_CQE32) - shift = 1; /* userspace may cheat modifying the tail, be safe and do min */ queued = min(__io_cqring_events(ctx), ctx->cq_entries); @@ -715,11 +712,19 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx) if (!len) return NULL; - ctx->cached_cq_tail++; + if (ctx->flags & IORING_SETUP_CQE32) { + off <<= 1; + len <<= 1; + } + ctx->cqe_cached = &rings->cqes[off]; ctx->cqe_sentinel = ctx->cqe_cached + len; + + ctx->cached_cq_tail++; ctx->cqe_cached++; - return &rings->cqes[off << shift]; + if (ctx->flags & IORING_SETUP_CQE32) + ctx->cqe_cached++; + return &rings->cqes[off]; } static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index dfb490e7cf45..558a860a93fc 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -22,14 +22,10 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) { struct io_uring_cqe *cqe = ctx->cqe_cached; - if (ctx->flags & IORING_SETUP_CQE32) { - unsigned int off = ctx->cqe_cached - ctx->rings->cqes; - - cqe += off; - } - ctx->cached_cq_tail++; ctx->cqe_cached++; + if (ctx->flags & IORING_SETUP_CQE32) + ctx->cqe_cached++; return cqe; } |