diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2023-03-31 00:53:51 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2023-04-24 20:37:17 +0300 |
commit | 487c20b016dc48230367a7be017f40313e53e3bd (patch) | |
tree | 29788f6f58adb0fc0b7d584268854386fef305c5 | |
parent | b9dff2195f8a5847fad801046b26955e05670d31 (diff) | |
download | linux-487c20b016dc48230367a7be017f40313e53e3bd.tar.xz |
iov: improve copy_iovec_from_user() code generation
Use the same pattern as the compat version of this code does: instead of
copying the whole array to a kernel buffer and then having a separate
phase of verifying it, just do it one entry at a time, verifying as you
go.
On Jens' /dev/zero readv() test this improves performance by ~6%.
[ This was obviously triggered by Jens' ITER_UBUF updates series ]
Reported-and-tested-by: Jens Axboe <axboe@kernel.dk>
Link: https://lore.kernel.org/all/de35d11d-bce7-e976-7372-1f2caf417103@kernel.dk/
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r-- | lib/iov_iter.c | 35 |
1 files changed, 26 insertions, 9 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 86a066aa9bcc..967fba189c5f 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1735,18 +1735,35 @@ uaccess_end: } static int copy_iovec_from_user(struct iovec *iov, - const struct iovec __user *uvec, unsigned long nr_segs) + const struct iovec __user *uiov, unsigned long nr_segs) { - unsigned long seg; + int ret = -EFAULT; - if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec))) + if (!user_access_begin(uiov, nr_segs * sizeof(*uiov))) return -EFAULT; - for (seg = 0; seg < nr_segs; seg++) { - if ((ssize_t)iov[seg].iov_len < 0) - return -EINVAL; - } - return 0; + do { + void __user *buf; + ssize_t len; + + unsafe_get_user(len, &uiov->iov_len, uaccess_end); + unsafe_get_user(buf, &uiov->iov_base, uaccess_end); + + /* check for size_t not fitting in ssize_t .. */ + if (unlikely(len < 0)) { + ret = -EINVAL; + goto uaccess_end; + } + iov->iov_base = buf; + iov->iov_len = len; + + uiov++; iov++; + } while (--nr_segs); + + ret = 0; +uaccess_end: + user_access_end(); + return ret; } struct iovec *iovec_from_user(const struct iovec __user *uvec, @@ -1771,7 +1788,7 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec, return ERR_PTR(-ENOMEM); } - if (compat) + if (unlikely(compat)) ret = copy_compat_iovec_from_user(iov, uvec, nr_segs); else ret = copy_iovec_from_user(iov, uvec, nr_segs); |