diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2023-04-16 05:31:34 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2023-04-19 03:05:28 +0300 |
commit | 577e6a7fd50d519c201d20968b6a027a6563dc4c (patch) | |
tree | ad8e4f8a75f78b627bf7ce28aabff48cac32bb54 /arch/x86/include/asm/uaccess_64.h | |
parent | 3639a535587d7aac449cdce9710dfdc97a3c8c8e (diff) | |
download | linux-577e6a7fd50d519c201d20968b6a027a6563dc4c.tar.xz |
x86: inline the 'rep movs' in user copies for the FSRM case
This does the same thing for the user copies as commit 0db7058e8e23
("x86/clear_user: Make it faster") did for clear_user(). In other
words, it inlines the "rep movs" case when X86_FEATURE_FSRM is set,
avoiding the function call entirely.
In order to do that, it makes the calling convention for the out-of-line
case ("copy_user_generic_unrolled") match the 'rep movs' calling
convention, although it does also end up clobbering a number of
additional registers.
Also, to simplify code sharing in the low-level assembly with the
__copy_user_nocache() function (that uses the normal C calling
convention), we end up with a kind of mixed return value for the
low-level asm code: it will return the result in both %rcx (to work as
an alternative for the 'rep movs' case), _and_ in %rax (for the nocache
case).
We could avoid this by wrapping __copy_user_nocache() callers in an
inline asm, but since the cost is just an extra register copy, it's
probably not worth it.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'arch/x86/include/asm/uaccess_64.h')
-rw-r--r-- | arch/x86/include/asm/uaccess_64.h | 23 |
1 files changed, 10 insertions, 13 deletions
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 339883729065..8cc918acbabc 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -18,29 +18,26 @@ /* Handles exceptions in both to and from, but doesn't do access_ok */ __must_check unsigned long -copy_user_fast_string(void *to, const void *from, unsigned len); -__must_check unsigned long copy_user_generic_unrolled(void *to, const void *from, unsigned len); static __always_inline __must_check unsigned long -copy_user_generic(void *to, const void *from, unsigned len) +copy_user_generic(void *to, const void *from, unsigned long len) { - unsigned ret; - stac(); /* * If CPU has FSRM feature, use 'rep movs'. * Otherwise, use copy_user_generic_unrolled. */ - alternative_call(copy_user_generic_unrolled, - copy_user_fast_string, - X86_FEATURE_FSRM, - ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), - "=d" (len)), - "1" (to), "2" (from), "3" (len) - : "memory", "rcx", "r8", "r9", "r10", "r11"); + asm volatile( + "1:\n\t" + ALTERNATIVE("rep movsb", + "call copy_user_generic_unrolled", ALT_NOT(X86_FEATURE_FSRM)) + "2:\n" + _ASM_EXTABLE_UA(1b, 2b) + :"+c" (len), "+D" (to), "+S" (from), ASM_CALL_CONSTRAINT + : : "memory", "rax", "rdx", "r8", "r9", "r10", "r11"); clac(); - return ret; + return len; } static __always_inline __must_check unsigned long |