summaryrefslogtreecommitdiff
path: root/arch/x86
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2023-04-15 23:39:15 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2023-04-19 03:05:28 +0300
commit3639a535587d7aac449cdce9710dfdc97a3c8c8e (patch)
treeec5bb3c10ed99a83143a66979a6c0af529ea6daa /arch/x86
parentd2c95f9d6802cc518d71d9795f4d9da54fb4e24d (diff)
downloadlinux-3639a535587d7aac449cdce9710dfdc97a3c8c8e.tar.xz
x86: move stac/clac from user copy routines into callers
This is preparatory work for inlining the 'rep movs' case, but also a cleanup. The __copy_user_nocache() function was mis-used by the rdma code to do uncached kernel copies that don't actually want user copies at all, and as a result doesn't want the stac/clac either. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/include/asm/uaccess_64.h8
-rw-r--r--arch/x86/lib/copy_user_64.S10
-rw-r--r--arch/x86/lib/usercopy_64.c6
3 files changed, 12 insertions, 12 deletions
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 59ea54af505e..339883729065 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -27,6 +27,7 @@ copy_user_generic(void *to, const void *from, unsigned len)
{
unsigned ret;
+ stac();
/*
* If CPU has FSRM feature, use 'rep movs'.
* Otherwise, use copy_user_generic_unrolled.
@@ -38,6 +39,7 @@ copy_user_generic(void *to, const void *from, unsigned len)
"=d" (len)),
"1" (to), "2" (from), "3" (len)
: "memory", "rcx", "r8", "r9", "r10", "r11");
+ clac();
return ret;
}
@@ -64,8 +66,12 @@ static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
unsigned size)
{
+ long ret;
kasan_check_write(dst, size);
- return __copy_user_nocache(dst, src, size, 0);
+ stac();
+ ret = __copy_user_nocache(dst, src, size, 0);
+ clac();
+ return ret;
}
static inline int
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index d0283bc7567d..818f2f728294 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -51,7 +51,6 @@
* eax uncopied bytes or 0 if successful.
*/
SYM_FUNC_START(copy_user_generic_unrolled)
- ASM_STAC
cmpl $8,%edx
jb .Lcopy_user_short_string_bytes
ALIGN_DESTINATION
@@ -123,15 +122,12 @@ EXPORT_SYMBOL(copy_user_generic_unrolled)
* eax uncopied bytes or 0 if successful.
*/
SYM_FUNC_START(copy_user_fast_string)
- ASM_STAC
movl %edx,%ecx
1: rep movsb
xorl %eax,%eax
- ASM_CLAC
RET
12: movl %ecx,%eax /* ecx is zerorest also */
- ASM_CLAC
RET
_ASM_EXTABLE_CPY(1b, 12b)
@@ -160,12 +156,10 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
movl %edx,%ecx
1: rep movsb
2: mov %ecx,%eax
- ASM_CLAC
RET
3:
movl %edx,%eax
- ASM_CLAC
RET
_ASM_EXTABLE_CPY(1b, 2b)
@@ -209,7 +203,6 @@ SYM_CODE_START_LOCAL(copy_user_short_string)
decl %ecx
jnz 21b
23: xor %eax,%eax
- ASM_CLAC
RET
40: leal (%rdx,%rcx,8),%edx
@@ -233,8 +226,6 @@ SYM_CODE_END(copy_user_short_string)
* - Require 4-byte alignment when size is 4 bytes.
*/
SYM_FUNC_START(__copy_user_nocache)
- ASM_STAC
-
/* If size is less than 8 bytes, go to 4-byte copy */
cmpl $8,%edx
jb .L_4b_nocache_copy_entry
@@ -327,7 +318,6 @@ SYM_FUNC_START(__copy_user_nocache)
/* Finished copying; fence the prior stores */
.L_finish_copy:
xorl %eax,%eax
- ASM_CLAC
sfence
RET
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 6c1f8ac5e721..15704c605a2b 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -45,7 +45,11 @@ EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
{
unsigned long flushed, dest = (unsigned long) dst;
- long rc = __copy_user_nocache(dst, src, size, 0);
+ long rc;
+
+ stac();
+ rc = __copy_user_nocache(dst, src, size, 0);
+ clac();
/*
* __copy_user_nocache() uses non-temporal stores for the bulk