From 6d5946274df1fff539a7eece458a43be733d1db8 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 30 May 2022 11:27:05 +0200 Subject: s390/gmap: voluntarily schedule during key setting With large and many guest with storage keys it is possible to create large latencies or stalls during initial key setting: rcu: INFO: rcu_sched self-detected stall on CPU rcu: 18-....: (2099 ticks this GP) idle=54e/1/0x4000000000000002 softirq=35598716/35598716 fqs=998 (t=2100 jiffies g=155867385 q=20879) Task dump for CPU 18: CPU 1/KVM R running task 0 1030947 256019 0x06000004 Call Trace: sched_show_task rcu_dump_cpu_stacks rcu_sched_clock_irq update_process_times tick_sched_handle tick_sched_timer __hrtimer_run_queues hrtimer_interrupt do_IRQ ext_int_handler ptep_zap_key The mmap lock is held during the page walking but since this is a semaphore scheduling is still possible. Same for the kvm srcu. To minimize overhead do this on every segment table entry or large page. Signed-off-by: Christian Borntraeger Reviewed-by: Alexander Gordeev Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220530092706.11637-2-borntraeger@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Heiko Carstens --- arch/s390/mm/gmap.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch/s390/mm') diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index af03cacf34ec..f3694f6df8b7 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2601,6 +2601,18 @@ static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr, return 0; } +/* + * Give a chance to schedule after setting a key to 256 pages. + * We only hold the mm lock, which is a rwsem and the kvm srcu. + * Both can sleep. + */ +static int __s390_enable_skey_pmd(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + cond_resched(); + return 0; +} + static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, unsigned long hmask, unsigned long next, struct mm_walk *walk) @@ -2623,12 +2635,14 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, end = start + HPAGE_SIZE - 1; __storage_key_init_range(start, end); set_bit(PG_arch_1, &page->flags); + cond_resched(); return 0; } static const struct mm_walk_ops enable_skey_walk_ops = { .hugetlb_entry = __s390_enable_skey_hugetlb, .pte_entry = __s390_enable_skey_pte, + .pmd_entry = __s390_enable_skey_pmd, }; int s390_enable_skey(void) -- cgit v1.2.3 From 3ae11dbcfac906a8c3a480e98660a823130dc16a Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 30 May 2022 11:27:06 +0200 Subject: s390/mm: use non-quiescing sske for KVM switch to keyed guest The switch to a keyed guest does not require a classic sske as the other guest CPUs are not accessing the key before the switch is complete. By using the NQ SSKE things are faster especially with multiple guests. Signed-off-by: Christian Borntraeger Suggested-by: Janis Schoetterl-Glausch Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220530092706.11637-3-borntraeger@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Heiko Carstens --- arch/s390/mm/pgtable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/s390/mm') diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 697df02362af..4909dcd762e8 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -748,7 +748,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT; ptev = pte_val(*ptep); if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) - page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); + page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0); pgste_set_unlock(ptep, pgste); preempt_enable(); } -- cgit v1.2.3 From 454ede3f1424c36bc124f8d7ed4b3ea654654be1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 1 May 2022 21:26:06 +0200 Subject: s390/uaccess: use exception handler to zero result on get_user() failure Historically the uaccess code pre-initializes the result of get_user() (and now also __get_kernel_nofault()) to zero and uses the result as input parameter for inline assemblies. This is different to what most, if not all, other architectures are doing, which set the result to zero within the exception handler in case of a fault. Use the new extable mechanism and handle zeroing of the result within the exception handler in case of a fault. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/asm-extable.h | 91 +++++++++++++++++++++++++------------ arch/s390/include/asm/uaccess.h | 72 ++++++++++++++++++----------- arch/s390/mm/extable.c | 39 ++++++++++++++-- 3 files changed, 143 insertions(+), 59 deletions(-) (limited to 'arch/s390/mm') diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index f24d9591aaed..b74f1070ddb2 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -3,12 +3,24 @@ #define __ASM_EXTABLE_H #include +#include #include -#define EX_TYPE_NONE 0 -#define EX_TYPE_FIXUP 1 -#define EX_TYPE_BPF 2 -#define EX_TYPE_UACCESS 3 +#define EX_TYPE_NONE 0 +#define EX_TYPE_FIXUP 1 +#define EX_TYPE_BPF 2 +#define EX_TYPE_UA_STORE 3 +#define EX_TYPE_UA_LOAD_MEM 4 +#define EX_TYPE_UA_LOAD_REG 5 + +#define EX_DATA_REG_ERR_SHIFT 0 +#define EX_DATA_REG_ERR GENMASK(3, 0) + +#define EX_DATA_REG_ADDR_SHIFT 4 +#define EX_DATA_REG_ADDR GENMASK(7, 4) + +#define EX_DATA_LEN_SHIFT 8 +#define EX_DATA_LEN GENMASK(11, 8) #define __EX_TABLE(_section, _fault, _target, _type) \ stringify_in_c(.section _section,"a";) \ @@ -19,35 +31,58 @@ stringify_in_c(.short 0;) \ stringify_in_c(.previous) -#define __EX_TABLE_UA(_section, _fault, _target, _type, _reg) \ - stringify_in_c(.section _section,"a";) \ - stringify_in_c(.align 4;) \ - stringify_in_c(.long (_fault) - .;) \ - stringify_in_c(.long (_target) - .;) \ - stringify_in_c(.short (_type);) \ - stringify_in_c(.macro extable_reg reg;) \ - stringify_in_c(.set .Lfound, 0;) \ - stringify_in_c(.set .Lregnr, 0;) \ - stringify_in_c(.irp rs,r0,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12,r13,r14,r15;) \ - stringify_in_c(.ifc "\reg", "%%\rs";) \ - stringify_in_c(.set .Lfound, 1;) \ - stringify_in_c(.short .Lregnr;) \ - stringify_in_c(.endif;) \ - stringify_in_c(.set .Lregnr, .Lregnr+1;) \ - stringify_in_c(.endr;) \ - stringify_in_c(.ifne (.Lfound != 1);) \ - stringify_in_c(.error "extable_reg: bad register argument";) \ - stringify_in_c(.endif;) \ - stringify_in_c(.endm;) \ - stringify_in_c(extable_reg _reg;) \ - stringify_in_c(.purgem extable_reg;) \ +#define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\ + stringify_in_c(.section _section,"a";) \ + stringify_in_c(.align 4;) \ + stringify_in_c(.long (_fault) - .;) \ + stringify_in_c(.long (_target) - .;) \ + stringify_in_c(.short (_type);) \ + stringify_in_c(.macro extable_reg regerr, regaddr;) \ + stringify_in_c(.set .Lfound, 0;) \ + stringify_in_c(.set .Lcurr, 0;) \ + stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \ + stringify_in_c( .ifc "\regerr", "%%r\rs";) \ + stringify_in_c( .set .Lfound, 1;) \ + stringify_in_c( .set .Lregerr, .Lcurr;) \ + stringify_in_c( .endif;) \ + stringify_in_c( .set .Lcurr, .Lcurr+1;) \ + stringify_in_c(.endr;) \ + stringify_in_c(.ifne (.Lfound != 1);) \ + stringify_in_c( .error "extable_reg: bad register argument1";) \ + stringify_in_c(.endif;) \ + stringify_in_c(.set .Lfound, 0;) \ + stringify_in_c(.set .Lcurr, 0;) \ + stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \ + stringify_in_c( .ifc "\regaddr", "%%r\rs";) \ + stringify_in_c( .set .Lfound, 1;) \ + stringify_in_c( .set .Lregaddr, .Lcurr;) \ + stringify_in_c( .endif;) \ + stringify_in_c( .set .Lcurr, .Lcurr+1;) \ + stringify_in_c(.endr;) \ + stringify_in_c(.ifne (.Lfound != 1);) \ + stringify_in_c( .error "extable_reg: bad register argument2";) \ + stringify_in_c(.endif;) \ + stringify_in_c(.short .Lregerr << EX_DATA_REG_ERR_SHIFT | \ + .Lregaddr << EX_DATA_REG_ADDR_SHIFT | \ + _len << EX_DATA_LEN_SHIFT;) \ + stringify_in_c(.endm;) \ + stringify_in_c(extable_reg _regerr,_regaddr;) \ + stringify_in_c(.purgem extable_reg;) \ stringify_in_c(.previous) #define EX_TABLE(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP) + #define EX_TABLE_AMODE31(_fault, _target) \ __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP) -#define EX_TABLE_UA(_fault, _target, _reg) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UACCESS, _reg) + +#define EX_TABLE_UA_STORE(_fault, _target, _regerr) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) + +#define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) + +#define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 5ffb88b3fdee..2d8d9e89d48f 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -80,8 +80,12 @@ union oac { }; }; -#define __put_get_user_asm(to, from, size, oac_spec) \ +#define __put_user_asm(to, from, size) \ ({ \ + union oac __oac_spec = { \ + .oac1.as = PSW_BITS_AS_SECONDARY, \ + .oac1.a = 1, \ + }; \ int __rc; \ \ asm volatile( \ @@ -89,26 +93,15 @@ union oac { "0: mvcos %[_to],%[_from],%[_size]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%[rc]) EX_TABLE_UA(1b,2b,%[rc]) \ + EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ + EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ : [rc] "=&d" (__rc), [_to] "+Q" (*(to)) \ : [_size] "d" (size), [_from] "Q" (*(from)), \ - [spec] "d" (oac_spec.val) \ + [spec] "d" (__oac_spec.val) \ : "cc", "0"); \ __rc; \ }) -#define __put_user_asm(to, from, size) \ - __put_get_user_asm(to, from, size, ((union oac) { \ - .oac1.as = PSW_BITS_AS_SECONDARY, \ - .oac1.a = 1 \ - })) - -#define __get_user_asm(to, from, size) \ - __put_get_user_asm(to, from, size, ((union oac) { \ - .oac2.as = PSW_BITS_AS_SECONDARY, \ - .oac2.a = 1 \ - })) \ - static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) { int rc; @@ -141,6 +134,29 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon return rc; } +#define __get_user_asm(to, from, size) \ +({ \ + union oac __oac_spec = { \ + .oac2.as = PSW_BITS_AS_SECONDARY, \ + .oac2.a = 1, \ + }; \ + int __rc; \ + \ + asm volatile( \ + " lr 0,%[spec]\n" \ + "0: mvcos 0(%[_to]),%[_from],%[_size]\n" \ + "1: xr %[rc],%[rc]\n" \ + "2:\n" \ + EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize]) \ + EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize]) \ + : [rc] "=&d" (__rc), "=Q" (*(to)) \ + : [_size] "d" (size), [_from] "Q" (*(from)), \ + [spec] "d" (__oac_spec.val), [_to] "a" (to), \ + [_ksize] "K" (size) \ + : "cc", "0"); \ + __rc; \ +}) + static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) { int rc; @@ -210,28 +226,28 @@ static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsign __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: { \ - unsigned char __x = 0; \ + unsigned char __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 2: { \ - unsigned short __x = 0; \ + unsigned short __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 4: { \ - unsigned int __x = 0; \ + unsigned int __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 8: { \ - unsigned long long __x = 0; \ + unsigned long __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ @@ -281,7 +297,8 @@ int __noreturn __put_kernel_bad(void); "0: " insn " %[_val],%[_to]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ + EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ + EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \ : [_val] "d" (val) \ : "cc"); \ @@ -290,7 +307,7 @@ int __noreturn __put_kernel_bad(void); #define __put_kernel_nofault(dst, src, type, err_label) \ do { \ - u64 __x = (u64)(*((type *)(src))); \ + unsigned long __x = (unsigned long)(*((type *)(src))); \ int __pk_err; \ \ switch (sizeof(type)) { \ @@ -324,8 +341,9 @@ int __noreturn __get_kernel_bad(void); "0: " insn " %[_val],%[_from]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ - : [rc] "=d" (__rc), [_val] "+d" (val) \ + EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val]) \ + EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val]) \ + : [rc] "=d" (__rc), [_val] "=d" (val) \ : [_from] "Q" (*(from)) \ : "cc"); \ __rc; \ @@ -337,28 +355,28 @@ do { \ \ switch (sizeof(type)) { \ case 1: { \ - u8 __x = 0; \ + unsigned char __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "ic"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 2: { \ - u16 __x = 0; \ + unsigned short __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "lh"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 4: { \ - u32 __x = 0; \ + unsigned int __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "l"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 8: { \ - u64 __x = 0; \ + unsigned long __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "lg"); \ *((type *)(dst)) = (type)__x; \ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 8ac8ad2474a0..1e4d2187541a 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include +#include #include #include #include @@ -24,9 +26,34 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex, struct pt_r return true; } -static bool ex_handler_uaccess(const struct exception_table_entry *ex, struct pt_regs *regs) +static bool ex_handler_ua_store(const struct exception_table_entry *ex, struct pt_regs *regs) { - regs->gprs[ex->data] = -EFAULT; + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + + regs->gprs[reg_err] = -EFAULT; + regs->psw.addr = extable_fixup(ex); + return true; +} + +static bool ex_handler_ua_load_mem(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + size_t len = FIELD_GET(EX_DATA_LEN, ex->data); + + regs->gprs[reg_err] = -EFAULT; + memset((void *)regs->gprs[reg_addr], 0, len); + regs->psw.addr = extable_fixup(ex); + return true; +} + +static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_zero = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + + regs->gprs[reg_err] = -EFAULT; + regs->gprs[reg_zero] = 0; regs->psw.addr = extable_fixup(ex); return true; } @@ -43,8 +70,12 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_fixup(ex, regs); case EX_TYPE_BPF: return ex_handler_bpf(ex, regs); - case EX_TYPE_UACCESS: - return ex_handler_uaccess(ex, regs); + case EX_TYPE_UA_STORE: + return ex_handler_ua_store(ex, regs); + case EX_TYPE_UA_LOAD_MEM: + return ex_handler_ua_load_mem(ex, regs); + case EX_TYPE_UA_LOAD_REG: + return ex_handler_ua_load_reg(ex, regs); } panic("invalid exception table entry"); } -- cgit v1.2.3